From 3510f9c3484dfb709691780afed3e0662bc889ec Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Wed, 22 Jul 2026 11:29:26 +0300 Subject: [PATCH] =?UTF-8?q?feat:=20prosediff=20=E2=80=94=20word-level=20pr?= =?UTF-8?q?ose=20diff=20over=20markdown=20block=20structure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Phase 0 de-risk gate. Segments a document into blocks with goldmark (headings, paragraphs, list items, code fences, table rows, block quotes, frontmatter), aligns the two block sequences with Myers over content hashes, and diffs word-by-word inside modified prose blocks and line-by-line inside modified code fences. Whitespace and line wrapping alone produce no diff in prose, and always do in code — that split is the whole point. Moves are detected by verbatim anchor and grown over their neighbours, so a relocated section does not explode into add+remove; a move that also bridges one edited block is recognised, a section rewritten while moving is not, and that limit is pinned by a test rather than papered over. SPIKE.md reports the verdict against twelve real revisions of docs/DESIGN.md: rewrapping the whole 1139-line document produces 1563 changed lines for git and zero changes here; 77% of real prose modifications read as small edits; 13% shred and want a two-column fallback in the web layer, which BlockChange.Similarity already gates. Verdict: the approach works, build the review UI on it. --- prosediff/SPIKE.md | 304 +++++++++++++++++++++ prosediff/align.go | 397 +++++++++++++++++++++++++++ prosediff/cmd/prosediff/main.go | 55 ++++ prosediff/myers.go | 234 ++++++++++++++++ prosediff/myers_test.go | 126 +++++++++ prosediff/prosediff.go | 140 ++++++++++ prosediff/prosediff_test.go | 417 +++++++++++++++++++++++++++++ prosediff/render.go | 216 +++++++++++++++ prosediff/segment.go | 457 ++++++++++++++++++++++++++++++++ prosediff/segment_test.go | 150 +++++++++++ prosediff/token.go | 273 +++++++++++++++++++ prosediff/token_test.go | 140 ++++++++++ 12 files changed, 2909 insertions(+) create mode 100644 prosediff/SPIKE.md create mode 100644 prosediff/align.go create mode 100644 prosediff/cmd/prosediff/main.go create mode 100644 prosediff/myers.go create mode 100644 prosediff/myers_test.go create mode 100644 prosediff/prosediff.go create mode 100644 prosediff/prosediff_test.go create mode 100644 prosediff/render.go create mode 100644 prosediff/segment.go create mode 100644 prosediff/segment_test.go create mode 100644 prosediff/token.go create mode 100644 prosediff/token_test.go diff --git a/prosediff/SPIKE.md b/prosediff/SPIKE.md new file mode 100644 index 0000000000000000000000000000000000000000..3a5b26285f9100adf47384ad4f331df3372c18de --- /dev/null +++ b/prosediff/SPIKE.md @@ -0,0 +1,304 @@ +# Phase 0 spike: does a prose diff read well enough to review AI-written specs in a browser? + +Status: **run, with a verdict.** + +Corpus: this repository's own `docs/DESIGN.md` — twelve revisions, `be096fc` +… `db7d6c0`, 336 → 1139 lines of heavily reworked English prose with genuine +reflow, written by an agent and repeatedly corrected by review. Extracted with +`git show :docs/DESIGN.md`. Every number and excerpt below is real output +from `prosediff/cmd/prosediff`; revisions are pinned to SHAs rather than `HEAD` +because other agents were committing to this branch while the spike ran. + +## Verdict + +**Yes. Build the review UI on this.** The failure mode the design feared — "a +one-word edit shows as a whole-paragraph replace" — does not occur at all, and +the common cases are not merely tolerable but pleasant to read. Roughly 4 in 5 +real modifications read as what they are: a substitution, an added clause, a +deleted sentence. + +There is one real weak spot — paragraphs that were substantively rewritten +shred into interleaved fragments (13% of real prose modifications) — but it is +a *presentation* problem with a fix the data structure already supports, not a +defect in the approach. Details and the recommended fix below. + +Two limitations to carry into Phase 4, both honest rather than papered over: + +- **Move detection needs a verbatim anchor.** A section that moved *and* was + rewritten throughout still shows as add+remove. A section that moved with an + edit or two inside it is handled correctly. +- **A changed-blocks-only view loses structural context.** The reviewer needs + the heading a change lives under; the plain renderer prints a + `@@ heading › path @@` banner as a stand-in and the web UI will need + something better. + +## The headline measurement + +Rewrapping the whole document from ~80 to 58 columns, changing not one word: + +| | line differ | prosediff | +|---|---|---| +| `db7d6c0` vs. rewrapped `db7d6c0` | **1563 changed lines** | **0 changes** — 307 blocks, all equal | + +That is the entire premise of the package, and it holds on a real 1139-line +document containing paragraphs, nested lists, block quotes, tables, YAML and +ASCII-art code fences. It is not an artefact of the test fixtures; the +equivalent synthetic check is `TestWholeDocumentReflowIsSilent`. + +## The eleven real commit pairs + +Adjacent commits are what a review actually looks like, so these matter more +than the endpoints. `git --numstat` on the same pair for comparison: + +``` +be096fc→c945f38 95 equal, 1 modified, 27 added, 0 removed, 0 moved | git +86 -1 +c945f38→72f5894 113 equal, 8 modified, 11 added, 2 removed, 0 moved | git +76 -26 +72f5894→fc49924 117 equal, 13 modified, 32 added, 2 removed, 0 moved | git +165 -46 +fc49924→1e7e4b1 130 equal, 23 modified, 60 added, 9 removed, 0 moved | git +296 -89 +1e7e4b1→f0dcfb4 210 equal, 2 modified, 6 added, 1 removed, 0 moved | git +35 -19 +f0dcfb4→12e043e 185 equal, 18 modified, 32 added, 15 removed, 0 moved | git +125 -101 +12e043e→a733623 223 equal, 5 modified, 53 added, 7 removed, 0 moved | git +242 -49 +a733623→ba61af7 280 equal, 1 modified, 1 added, 0 removed, 0 moved | git +11 -1 +ba61af7→7c82786 280 equal, 2 modified, 6 added, 0 removed, 0 moved | git +28 -4 +7c82786→d148a66 283 equal, 4 modified, 9 added, 1 removed, 0 moved | git +42 -11 +d148a66→db7d6c0 294 equal, 2 modified, 11 added, 0 removed, 0 moved | git +48 -4 +``` + +The equal counts are the important column: in every pair the aligner keeps the +untouched bulk of the document silent, including across the two commits that +rewrote the largest sections (`fc49924→1e7e4b1`, `f0dcfb4→12e043e`). No +content-specific heuristics or tuning were needed to get that. + +Zero moves across the whole history is a property of this corpus, not a broken +detector: the document grew by insertion and rewriting rather than relocation, +and where sections were reorganised they were also rewritten, which removes the +verbatim anchor moves are keyed on. Move detection is therefore exercised +against constructed moves below. + +## Does a reflowed paragraph with a small edit show as a small edit? + +Yes. This is the strongest result. Appending one clause and rewrapping the +paragraph (a 7-line change to `git`): + +``` +@@ … › Merge model: no text merge, ever @@ +~ L404 paragraph modified + **Verified constraint:** `go-git` v5.19.1 implements only `FastForwardMerge` + (`repository.go:1800`; anything else returns `ErrUnsupportedMergeStrategy`). There is no + three-way merge available in-process{+, and none is planned upstream+}. + +307 blocks: 306 equal, 1 modified, 0 added, 0 removed, 0 moved (+6/-0 words) +``` + +Real commits look the same. From `7c82786` (agent-token rework): + +``` +~ L546 paragraph modified + [-Per-space (or per-project) tokens with a role — `reader` / `proposer` / `writer` — + and-]{+One agent token (no roles, no per-space scopes — see above) plus+} a **required + agent identity string**. Every commit records it in a way that survives clone: +``` + +and from the most recent pair, `d148a66→db7d6c0`: + +``` +~ L435→465 paragraph modified + **Deletion and rename need an explicit surface.** The write plane is whole-document `PUT`, + which gives an agent no way to express "delete this" or "move this". Two coherent answers; + **the second is the v1 [-recommendation-]{+decision+}**: +``` + +A line differ shows each of these as several deleted and several inserted +lines, with no indication of which words actually moved. + +Across all 69 prose modifications in the eleven pairs: + +| shape of the inline diff | count | reads as | +|---|---|---| +| ≤ 2 changed runs | 27 (39%) | one clean substitution — ideal | +| 3–6 changed runs | 26 (38%) | a few separate edits in one paragraph — fine | +| 7–8 changed runs | 7 (10%) | busy, still followable | +| ≥ 9 changed runs | 9 (13%) | shredded — see below | + +Similarity of paired blocks: min 0.41, p10 0.46, p25 0.63, median 0.76, +p75 0.86. Ten further modifications were code fences, diffed line-by-line. + +## Where it is bad: substantively rewritten paragraphs shred + +The worst case in the corpus (`c945f38→72f5894`, similarity 0.56, 18 changed +runs): + +``` + **[-Verified that opting out is-]{+Opting out, if we ever do, is verified+} + safe.** `api.sr.ht` federates *every* config section ending in `.sr.ht` [-— + it does not consult an-]{+with no+} allow-list, [-so our section does get + added as a federated service pointed at `-]{+pointing at `api-origin` (else + `+}origin{+`) `+} + "/query"`[-, which we will not serve-]. `updateSchema` + (`api.sr.ht/main.go`) [-handles this explicitly:-]{+logs and **skips**+} + services that are offline or [-have-]{+serve+} an invalid schema [-are + logged and **skipped**-], and `BuildSchema` runs over [-only-] the healthy + ones[-. So the effect is-]{+only — so a non-GraphQL service costs+} one + `Unable to update service [-"spec.sr.ht"-]` log line, not a broken gateway. +``` + +This is genuinely unpleasant, and it is *not* fixable by tuning the word +differ. The paragraph really was rewritten sentence by sentence, and +interleaving two rewritten sentences is inherently hard to read no matter how +the runs are computed. Absorbing tiny surviving fragments into the surrounding +change (the package does this — `smallEqualRun`) helps, but cannot rescue a +70%-rewritten paragraph. + +**The fix belongs in the web layer, and the data already carries the signal.** +Every block with ≥ 9 changed runs has similarity ≤ 0.73; every block above 0.75 +either reads cleanly or is a handful of clearly separate small edits. So +`BlockChange.Similarity` is a usable switch: render the inline word diff above +~0.75 and fall back to a two-column *old block / new block* view below it, +which is how a human wants to read a rewritten paragraph anyway. That is a +Phase 4 rendering decision, not a redesign — the field is exported for exactly +this. + +Worth noting what is *not* happening even in the bad case: the surrounding 200+ +blocks stay silent. The shredding is confined to the paragraph that was +actually rewritten, so the reviewer's cost is bounded and local. + +## Does a moved section explode into add+remove noise? + +No, for moves with any verbatim content. Moving the whole "Open risks" section +(6 blocks) ahead of "Repo layout" — `git` reports 34 changed lines: + +``` +@@ … › GraphQL: a read schema at our own `/query` in Phase 2 @@ +> L836 h2 moved here (was line 1065) +@@ … › Open risks @@ +> L838 list item moved here (was line 1067) +> L841 list item moved here (was line 1070) +… +< L1065 h2 moved away (now line 836) +< L1067 list item moved away (now line 838) +… + +313 blocks: 301 equal, 0 modified, 0 added, 0 removed, 6 moved (+0/-0 words) +``` + +Zero adds, zero removes. The same move with one word changed inside it also +holds together — the move group bridges a single edited block, so the reviewer +sees the section relocate *and* the one thing that actually changed in it: + +``` +> L836 h2 moved here (was line 1065) +~ L1067→838 list item modified (moved from line 1067) + **Prose diff quality (highest).** Everything downstream of the [-browser-review-]{+browser + review+} decision assumes it reads well. No fallback is designed; Phase 0 exists to find + out early rather than late. +> L841 list item moved here (was line 1070) + +313 blocks: 301 equal, 1 modified, 0 added, 0 removed, 5 moved (+2/-1 words) +``` + +**The honest limitation:** every move must be anchored by at least one block +that is byte-identical after normalization. A section that moved and was +rewritten throughout falls through as a delete plus an insert; +`TestMovedAndEditedIsNotAMove` pins that behaviour so it cannot regress +silently. This is deliberate. Matching moves by similarity alone would claim +relationships between blocks that merely share boilerplate, and a confidently +wrong "moved from line 1065" costs a reviewer more than an honest add+remove. + +## How do heavily-edited tables behave? + +Well, because **each table row is its own block**. From `ba61af7`, one edited +row of the reuse-inventory table with the other rows silent: + +``` +~ L574 table_row modified + | `sr-ht-core` (fork) | config, crypto, auth, database, server, **gqlgen scaffolding + + `webhooks`** | Pinned to `[-git.srht.bigb.es/~-]{+sourcecraft.dev/+}bigbes/[-core-go` via + `replace`, as in-]{+sr-ht-core v0.0.0-20260718185800-dd418a200152`, byte-identical to+} + both siblings{+. **No `replace` directive** — see the note below+}. Never `go get -u`. … +``` + +`f0dcfb4→12e043e` adds six rows to the decisions table; they render as six +clean `+ table_row` entries and the surrounding rows do not move. A new table +appears as a `+ table_header` followed by `+ table_row` entries, which reads +correctly. + +Two caveats: + +- **The alignment row is invisible.** goldmark folds `|---|---|` into the table + node, so changing it to `|:--|--:|` reports **no change at all** — measured, + not assumed. Acceptable (it is presentation, not content) but a real blind + spot to record. +- **Word diffing can straddle a cell boundary** — the + `git.srht.bigb.es/~` → `sourcecraft.dev/` fragment above splits mid-path + because the row is one token stream rather than a list of cells. Legible, but + cell-aware diffing would be better and is cheap to add later: the block + already records its column count. + +## How do heavily-edited lists behave? + +Well, and this is the second-strongest result. Each list item is its own block, +so editing one item leaves its siblings silent, adding an item is an insert +rather than a whole-list replace, and nesting depth is part of block identity — +re-indenting an item is reported (as a modification with no word changes, +flagged `StructureOnly`) rather than hidden. + +Real evidence: `1e7e4b1→f0dcfb4` adds two list items and removes one, and the +render is exactly two `+ list item` blocks and one `- list item`, each shown in +full, with the rest of their lists silent. `d148a66→db7d6c0` adds a five-row +table and six paragraphs inside existing sections without disturbing one +neighbouring block. + +## Smaller findings + +- **Renaming a heading does not dirty its section.** `HeadingPath` is context + for the reviewer, never part of a block's hash. Including it would have made + a section rename rewrite every block underneath — a trap worth naming, + because the comment-anchoring design in `DESIGN.md` wants that same tuple. +- **A soft line break is a space, and the differ is right to say so.** An early + rewrap run broke `logout-everywhere` across two lines; prosediff flagged it, + because in rendered markdown `logout-\neverywhere` really is + "logout- everywhere". Correct behaviour, initially mistaken for a bug. +- **Two-word blocks below the similarity floor become add+remove.** + `## Old name` → `## New name` scores 0.5, under the 0.6 floor that applies to + short blocks, so it renders as a removal plus an addition rather than a word + diff. For a two-word heading both readings are equally clear, so the floor + stays where it is rather than being tuned to a fixture + (`TestShortBlockRenameFallsBackToAddRemove`). +- **Within one changed region, entries are emitted old-side first.** A block + inserted at line 433 can therefore print after a modification at line 465. + The plain renderer accepts this; the web UI should order the new-side entries + by new-document position. +- **The heading-path banner flaps** when a section heading is itself added or + deleted: the blocks around it briefly display the old path. Cosmetic, and it + disappears once blocks are rendered structurally under their headings. +- **Fast enough to ignore.** The widest pair (336 vs 1139 lines, 307 blocks, + 211 of them added) compares in **12 ms** in-process. Typical adjacent-commit + pairs are far cheaper. No caching needed on the review page. + +## What would have made this a negative verdict + +Recorded so the verdict is falsifiable rather than a vibe. The spike would have +failed if: + +- a pure rewrap had produced any change at all — it produced zero; +- the *median* modification had been shredded rather than roughly one in eight; +- the block aligner had needed content-specific heuristics to keep the "equal" + set stable across the eleven real commits — it needed none; +- code fences had had to be diffed as prose, or prose as lines, to make either + read acceptably — they did not. + +None of those happened, so the browser-review decision and Phase 4 stand. + +## Recommendations for Phase 4 + +1. **Switch presentation on `Similarity`.** Inline word diff at ≥ ~0.75; + two-column old/new block below it. This removes the one genuinely bad case. +2. Render `ChangeMoveOut` as a collapsed one-line marker and `ChangeMoveIn` in + full at the new position. +3. Show unchanged blocks collapsed behind an expander rather than hidden — the + `@@` banner is a poor substitute for structural context in a browser. +4. Order new-side entries by new-document position (see "smaller findings"). +5. Consider cell-aware diffing for table rows before shipping, using the column + count already on the block. +6. `Block.Hash` + `HeadingPath` + `Ordinal` are exactly the comment-anchor + tuple `DESIGN.md` specifies. Phase 5 can test anchor survival against this + same corpus rather than guessing. diff --git a/prosediff/align.go b/prosediff/align.go new file mode 100644 index 0000000000000000000000000000000000000000..361e59c987ce1995260871ed525627d8d6114c22 --- /dev/null +++ b/prosediff/align.go @@ -0,0 +1,397 @@ +package prosediff + +import "sort" + +// Tuning constants for block alignment. They are deliberately package-level +// and documented rather than hidden in the code, because they are the knobs +// that decide whether a review page reads well. +const ( + // modifyThreshold is the token similarity two blocks need before they + // are called "the same block, edited" rather than a delete plus an + // insert. Below it, presenting a word diff would be noise. + modifyThreshold = 0.40 + + // shortBlockTokens is the size below which a block is considered too + // small to judge by similarity alone. + shortBlockTokens = 6 + + // shortBlockThreshold applies instead of modifyThreshold when either + // block is short: pairing "yes" with "no" on a 0.4 score helps nobody. + shortBlockThreshold = 0.60 + + // moveMinTokens is the smallest block that may be reported as moved. + // Below it, identical content is far more likely to be coincidence + // (a repeated "```", a "- see above") than an actual move. + moveMinTokens = 5 + + // pairBudget caps the similarity matrix inside one changed region. Past + // it, only pairs within pairWindow of each other are considered; a + // region with hundreds of blocks on both sides has no readable pairing + // anyway. + pairBudget = 4096 + pairWindow = 8 +) + +// region is one changed stretch of the alignment: blocks deleted from the old +// revision and blocks inserted into the new one, adjacent in the edit script. +type region struct { + del []int // indices into old + ins []int // indices into nw +} + +// align turns two block sequences into the diff's change list. +// +// Three passes, in this order: +// 1. Myers over block hashes finds the unchanged skeleton. +// 2. Blocks left over from pass 1 that reappear verbatim elsewhere are +// moves, grown outwards over their neighbours. +// 3. Similar-enough leftovers inside one changed region are modifications. +// +// Anything still unmatched is a plain insert or delete. +func align(old, nw []Block) []BlockChange { + in := newInterner() + oldIDs := make([]int, len(old)) + for i, b := range old { + oldIDs[i] = in.id(b.Hash) + } + newIDs := make([]int, len(nw)) + for i, b := range nw { + newIDs[i] = in.id(b.Hash) + } + script := diffInts(oldIDs, newIDs) + + var ( + out []BlockChange + regions []region + // slots[i] is where region i's entries go in the output. + slots []int + ) + + i, j := 0, 0 + for k := 0; k < len(script); { + if script[k].op == OpEqual { + for n := 0; n < script[k].n; n++ { + o, w := old[i+n], nw[j+n] + out = append(out, BlockChange{Kind: ChangeEqual, Old: &o, New: &w}) + } + i += script[k].n + j += script[k].n + k++ + continue + } + var r region + for ; k < len(script) && script[k].op != OpEqual; k++ { + if script[k].op == OpDelete { + for n := 0; n < script[k].n; n++ { + r.del = append(r.del, i+n) + } + i += script[k].n + } else { + for n := 0; n < script[k].n; n++ { + r.ins = append(r.ins, j+n) + } + j += script[k].n + } + } + regions = append(regions, r) + slots = append(slots, len(out)) + out = append(out, BlockChange{}) // placeholder, expanded below + } + + moveOf, editedInMove := detectMoves(old, nw, regions) + + // Build each region's entries, then splice them in at their slot. + expanded := make([][]BlockChange, len(regions)) + for ri, r := range regions { + expanded[ri] = expandRegion(old, nw, r.del, r.ins, moveOf, editedInMove) + } + final := make([]BlockChange, 0, len(out)+len(regions)) + next := 0 + for idx := 0; idx < len(out); idx++ { + if next < len(slots) && slots[next] == idx { + final = append(final, expanded[next]...) + next++ + continue + } + final = append(final, out[idx]) + } + return final +} + +// movePair records that old block o and new block n are the same content in a +// different place. +type movePair struct { + oldIdx int + newIdx int +} + +// detectMoves matches leftover blocks across regions, seeding on exact hash +// equality and then growing each seed over its neighbours. +// +// Deliberate limitation: every move must be anchored by at least one block +// whose content is byte-identical after normalization. A block that moved and +// was edited is recognised only when it sits *between* two such anchors; a +// section that moved and was rewritten throughout falls through as a delete +// plus an insert. Matching moves by similarity alone would claim +// relationships between blocks that merely share boilerplate, and a wrong +// "moved from" costs a reviewer more than an honest add+remove. +func detectMoves(old, nw []Block, regions []region) (moves, edited map[int]movePair) { + freeOld := map[int]bool{} + freeNew := map[int]bool{} + byHash := map[string][]int{} + for _, r := range regions { + for _, oi := range r.del { + freeOld[oi] = true + if len(Tokenize(old[oi].Text)) >= moveMinTokens { + byHash[old[oi].Hash] = append(byHash[old[oi].Hash], oi) + } + } + for _, ni := range r.ins { + freeNew[ni] = true + } + } + + out := map[int]movePair{} + edited = map[int]movePair{} + pair := func(oi, ni int) { + p := movePair{oldIdx: oi, newIdx: ni} + out[oldKey(oi)] = p + out[newKey(ni)] = p + delete(freeOld, oi) + delete(freeNew, ni) + } + + // Seed: blocks big enough that identical content cannot be coincidence. + var seeds []movePair + for _, r := range regions { + for _, ni := range r.ins { + if len(Tokenize(nw[ni].Text)) < moveMinTokens { + continue + } + for _, oi := range byHash[nw[ni].Hash] { + if !freeOld[oi] || !freeNew[ni] { + continue + } + pair(oi, ni) + seeds = append(seeds, movePair{oi, ni}) + break + } + } + } + + // Grow each seed outwards while the neighbouring blocks are also + // unmatched and identical. This is what keeps a moved section's heading + // and its short trailing blocks attached to the move, instead of + // stranding them as a delete plus an insert either side of it. + // + // Growth also bridges a single edited block, but only when the block + // *past* it matches exactly — "a section was moved and one paragraph in + // it was touched" is common, while a lone similar block at the edge of a + // move is just as likely to be coincidence. + for _, s := range seeds { + for step := -1; step <= 1; step += 2 { + oi, ni := s.oldIdx+step, s.newIdx+step + for freeOld[oi] && freeNew[ni] { + if old[oi].Hash == nw[ni].Hash { + pair(oi, ni) + oi += step + ni += step + continue + } + if !bridgeable(old, nw, oi, ni, step, freeOld, freeNew, out) { + break + } + pair(oi, ni) + edited[oldKey(oi)] = movePair{oi, ni} + edited[newKey(ni)] = movePair{oi, ni} + oi += step + ni += step + } + } + } + return out, edited +} + +// bridgeable reports whether old[oi] and nw[ni] are an edited version of one +// another sitting inside a run of moved blocks. The anchor past the gap may +// be either still unclaimed or already paired to its counterpart by an +// earlier seed — both mean "the move continues on the far side". +func bridgeable(old, nw []Block, oi, ni, step int, freeOld, freeNew map[int]bool, moves map[int]movePair) bool { + if old[oi].Kind != nw[ni].Kind { + return false + } + no, nn := oi+step, ni+step + if no < 0 || no >= len(old) || nn < 0 || nn >= len(nw) { + return false + } + if old[no].Hash != nw[nn].Hash { + return false + } + anchored := freeOld[no] && freeNew[nn] + if p, ok := moves[oldKey(no)]; ok && p.newIdx == nn { + anchored = true + } + if !anchored { + return false + } + return blockSimilarity(old[oi], nw[ni]) >= thresholdFor(old[oi], nw[ni]) +} + +// Move bookkeeping keys old and new indices into one map without colliding. +func oldKey(i int) int { return i * 2 } +func newKey(i int) int { return i*2 + 1 } + +func expandRegion(old, nw []Block, del, ins []int, moveOf, editedInMove map[int]movePair) []BlockChange { + pairs := pairModified(old, nw, del, ins, moveOf) + pairedOld := map[int]int{} // old index -> new index + pairedNew := map[int]bool{} + for _, p := range pairs { + pairedOld[p.oldIdx] = p.newIdx + pairedNew[p.newIdx] = true + } + + var out []BlockChange + for _, oi := range del { + o := old[oi] + if mp, ok := moveOf[oldKey(oi)]; ok { + // A block that moved and was edited is announced here and + // shown in full at its new position, where the reviewer + // reads the section it now belongs to. + n := nw[mp.newIdx] + out = append(out, BlockChange{Kind: ChangeMoveOut, Old: &o, New: &n}) + continue + } + if ni, ok := pairedOld[oi]; ok { + n := nw[ni] + out = append(out, modifyChange(o, n)) + continue + } + out = append(out, BlockChange{Kind: ChangeDelete, Old: &o}) + } + for _, ni := range ins { + n := nw[ni] + if mp, ok := editedInMove[newKey(ni)]; ok { + c := modifyChange(old[mp.oldIdx], n) + c.Moved = true + out = append(out, c) + continue + } + if mp, ok := moveOf[newKey(ni)]; ok { + o := old[mp.oldIdx] + out = append(out, BlockChange{Kind: ChangeMoveIn, Old: &o, New: &n}) + continue + } + if pairedNew[ni] { + continue // already emitted as a modification + } + out = append(out, BlockChange{Kind: ChangeInsert, New: &n}) + } + return out +} + +func modifyChange(o, n Block) BlockChange { + c := BlockChange{Kind: ChangeModify, Old: &o, New: &n} + if o.Kind.Prose() { + c.Words = DiffWords(o.Text, n.Text) + c.StructureOnly = !hasChange(c.Words) + } else { + c.Lines = DiffLines(o.Lines, n.Lines) + c.StructureOnly = !hasChange(c.Lines) + } + c.Similarity = blockSimilarity(o, n) + return c +} + +func hasChange(spans []Span) bool { + for _, s := range spans { + if s.Op != OpEqual { + return true + } + } + return false +} + +// pairModified greedily matches the most similar delete/insert pairs left in +// one changed region, best first. +func pairModified(old, nw []Block, del, ins []int, moveOf map[int]movePair) []movePair { + var cand []int + for _, oi := range del { + if _, moved := moveOf[oldKey(oi)]; !moved { + cand = append(cand, oi) + } + } + var cins []int + for _, ni := range ins { + if _, moved := moveOf[newKey(ni)]; !moved { + cins = append(cins, ni) + } + } + if len(cand) == 0 || len(cins) == 0 { + return nil + } + windowed := len(cand)*len(cins) > pairBudget + + type scored struct { + p movePair + s float64 + } + var all []scored + for a, oi := range cand { + for b, ni := range cins { + if windowed && abs(a-b) > pairWindow { + continue + } + o, n := old[oi], nw[ni] + if o.Kind != n.Kind { + continue + } + s := blockSimilarity(o, n) + if s < thresholdFor(o, n) { + continue + } + all = append(all, scored{movePair{oi, ni}, s}) + } + } + sort.SliceStable(all, func(i, j int) bool { + if all[i].s != all[j].s { + return all[i].s > all[j].s + } + return all[i].p.oldIdx < all[j].p.oldIdx + }) + + usedOld := map[int]bool{} + usedNew := map[int]bool{} + var out []movePair + for _, c := range all { + if usedOld[c.p.oldIdx] || usedNew[c.p.newIdx] { + continue + } + usedOld[c.p.oldIdx] = true + usedNew[c.p.newIdx] = true + out = append(out, c.p) + } + return out +} + +func thresholdFor(a, b Block) float64 { + if len(Tokenize(a.Text)) < shortBlockTokens || len(Tokenize(b.Text)) < shortBlockTokens { + return shortBlockThreshold + } + return modifyThreshold +} + +// blockSimilarity is token similarity for prose and line similarity for code. +func blockSimilarity(a, b Block) float64 { + in := newInterner() + if a.Kind.Prose() { + return ratio(in.all(TokenTexts(Tokenize(a.Text))), in.all(TokenTexts(Tokenize(b.Text)))) + } + return ratio(in.all(a.Lines), in.all(b.Lines)) +} + +func abs(i int) int { + if i < 0 { + return -i + } + return i +} diff --git a/prosediff/cmd/prosediff/main.go b/prosediff/cmd/prosediff/main.go new file mode 100644 index 0000000000000000000000000000000000000000..406c8707500d1032b84894c39c55d3fb5e4ba864 --- /dev/null +++ b/prosediff/cmd/prosediff/main.go @@ -0,0 +1,55 @@ +// Command prosediff prints the prose diff between two markdown files. +// +// It exists so the differ can be judged the way a reviewer would judge it — +// by reading its output on real documents — without a web layer. +// +// prosediff old.md new.md +// prosediff -stats -width 100 old.md new.md +package main + +import ( + "flag" + "fmt" + "os" + + "sourcecraft.dev/bigbes/sr-ht-spec/prosediff" +) + +func main() { + var ( + width = flag.Int("width", 92, "wrap prose at this column; 0 disables wrapping") + context = flag.Int("context", 0, "unchanged blocks to show around each change") + showEqual = flag.Bool("all", false, "print unchanged blocks too") + statsOnly = flag.Bool("stats", false, "print only the summary line") + ) + flag.Parse() + if flag.NArg() != 2 { + fmt.Fprintln(os.Stderr, "usage: prosediff [flags] ") + flag.PrintDefaults() + os.Exit(2) + } + + oldSrc, err := os.ReadFile(flag.Arg(0)) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + newSrc, err := os.ReadFile(flag.Arg(1)) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + d := prosediff.Compare(oldSrc, newSrc) + if !*statsOnly { + fmt.Print(prosediff.RenderText(d, prosediff.RenderOptions{ + Width: *width, + Context: *context, + ShowEqual: *showEqual, + })) + } + s := d.Stats + fmt.Printf("\n%d blocks: %d equal, %d modified, %d added, %d removed, %d moved (+%d/-%d words)\n", + len(d.Changes), s.BlocksEqual, s.BlocksModified, s.BlocksInserted, + s.BlocksDeleted, s.BlocksMoved, s.WordsInserted, s.WordsDeleted) +} diff --git a/prosediff/myers.go b/prosediff/myers.go new file mode 100644 index 0000000000000000000000000000000000000000..9286c8dc8a6d6e3a1b1d2a29390e6596fc0361e0 --- /dev/null +++ b/prosediff/myers.go @@ -0,0 +1,234 @@ +package prosediff + +// Myers O(ND) sequence diff, used at three levels: blocks, words and code +// lines. Sequences are always interned to []int first, so equality is exact +// integer equality and there is no hash-collision risk. + +// Op is the role of a run in an edit script. +type Op uint8 + +const ( + // OpEqual marks content present, unchanged, in both revisions. + OpEqual Op = iota + // OpDelete marks content present only in the old revision. + OpDelete + // OpInsert marks content present only in the new revision. + OpInsert +) + +func (o Op) String() string { + switch o { + case OpEqual: + return "equal" + case OpDelete: + return "delete" + case OpInsert: + return "insert" + } + return "unknown" +} + +// edit is one run of an edit script: n consecutive elements sharing an Op. +type edit struct { + op Op + n int +} + +// interner maps arbitrary strings to dense ints so sequence comparison is +// integer equality. +type interner struct { + ids map[string]int +} + +func newInterner() *interner { return &interner{ids: make(map[string]int)} } + +func (in *interner) id(s string) int { + if v, ok := in.ids[s]; ok { + return v + } + v := len(in.ids) + 1 + in.ids[s] = v + return v +} + +func (in *interner) all(ss []string) []int { + out := make([]int, len(ss)) + for i, s := range ss { + out[i] = in.id(s) + } + return out +} + +// diffInts returns the edit script turning a into b, as a coalesced run list. +// The script is a minimal edit script (Myers), with common prefix and suffix +// trimmed first so the expensive part only runs over the changed middle. +func diffInts(a, b []int) []edit { + pre := 0 + for pre < len(a) && pre < len(b) && a[pre] == b[pre] { + pre++ + } + suf := 0 + for suf < len(a)-pre && suf < len(b)-pre && a[len(a)-1-suf] == b[len(b)-1-suf] { + suf++ + } + var out []edit + if pre > 0 { + out = append(out, edit{OpEqual, pre}) + } + out = append(out, myers(a[pre:len(a)-suf], b[pre:len(b)-suf])...) + if suf > 0 { + out = append(out, edit{OpEqual, suf}) + } + return coalesce(out) +} + +// vsnap is the V array of one Myers iteration, stored only over the diagonals +// [-d, d] that iteration can reach. +type vsnap struct { + d int + vals []int32 +} + +func (s vsnap) get(k int) int { return int(s.vals[k+s.d]) } + +func myers(a, b []int) []edit { + n, m := len(a), len(b) + switch { + case n == 0 && m == 0: + return nil + case n == 0: + return []edit{{OpInsert, m}} + case m == 0: + return []edit{{OpDelete, n}} + } + + maxD := n + m + v := make([]int32, 2*maxD+1) + off := maxD + trace := make([]vsnap, 0, 16) + + for d := 0; d <= maxD; d++ { + done := false + for k := -d; k <= d; k += 2 { + var x int + if k == -d || (k != d && v[off+k-1] < v[off+k+1]) { + x = int(v[off+k+1]) // move down: consume one element of b + } else { + x = int(v[off+k-1]) + 1 // move right: consume one element of a + } + y := x - k + for x < n && y < m && a[x] == b[y] { + x++ + y++ + } + v[off+k] = int32(x) + if x >= n && y >= m { + done = true + break + } + } + snap := vsnap{d: d, vals: make([]int32, 2*d+1)} + for k := -d; k <= d; k += 2 { + snap.vals[k+d] = v[off+k] + } + trace = append(trace, snap) + if done { + return backtrack(trace, n, m) + } + } + panic("prosediff: myers did not converge") +} + +// backtrack walks the saved V arrays from the end point back to the origin, +// emitting the edit script in reverse and then flipping it. +func backtrack(trace []vsnap, n, m int) []edit { + var rev []edit + push := func(op Op) { + if len(rev) > 0 && rev[len(rev)-1].op == op { + rev[len(rev)-1].n++ + return + } + rev = append(rev, edit{op, 1}) + } + + x, y := n, m + for d := len(trace) - 1; d > 0; d-- { + prev := trace[d-1] + k := x - y + var prevK int + if k == -d || (k != d && prev.get(k-1) < prev.get(k+1)) { + prevK = k + 1 + } else { + prevK = k - 1 + } + prevX := prev.get(prevK) + prevY := prevX - prevK + for x > prevX && y > prevY { + push(OpEqual) + x-- + y-- + } + if x == prevX { + push(OpInsert) + y-- + } else { + push(OpDelete) + x-- + } + x, y = prevX, prevY + } + for x > 0 && y > 0 { + push(OpEqual) + x-- + y-- + } + // d == 0 leaves at most one of x, y non-zero only when the other + // sequence was fully consumed on the diagonal, which the prefix trim + // already handled; keep the guard rather than assume. + for ; x > 0; x-- { + push(OpDelete) + } + for ; y > 0; y-- { + push(OpInsert) + } + + out := make([]edit, 0, len(rev)) + for i := len(rev) - 1; i >= 0; i-- { + out = append(out, rev[i]) + } + return coalesce(out) +} + +func coalesce(in []edit) []edit { + out := in[:0:0] + for _, e := range in { + if e.n == 0 { + continue + } + if len(out) > 0 && out[len(out)-1].op == e.op { + out[len(out)-1].n += e.n + continue + } + out = append(out, e) + } + return out +} + +// commonCount reports how many elements the edit script keeps equal. +func commonCount(script []edit) int { + n := 0 + for _, e := range script { + if e.op == OpEqual { + n += e.n + } + } + return n +} + +// ratio is the classic 2*common/(len(a)+len(b)) similarity in [0,1]. +func ratio(a, b []int) float64 { + if len(a) == 0 && len(b) == 0 { + return 1 + } + return 2 * float64(commonCount(diffInts(a, b))) / float64(len(a)+len(b)) +} diff --git a/prosediff/myers_test.go b/prosediff/myers_test.go new file mode 100644 index 0000000000000000000000000000000000000000..4449af79e221a02856b5f3dba800cc9751692fcc --- /dev/null +++ b/prosediff/myers_test.go @@ -0,0 +1,126 @@ +package prosediff + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDiffIntsScripts(t *testing.T) { + tests := []struct { + name string + a, b []int + want []edit + }{ + {"both empty", nil, nil, nil}, + {"identical", []int{1, 2, 3}, []int{1, 2, 3}, []edit{{OpEqual, 3}}}, + {"all inserted", nil, []int{1, 2}, []edit{{OpInsert, 2}}}, + {"all deleted", []int{1, 2}, nil, []edit{{OpDelete, 2}}}, + {"middle replaced", []int{1, 2, 3}, []int{1, 9, 3}, + []edit{{OpEqual, 1}, {OpDelete, 1}, {OpInsert, 1}, {OpEqual, 1}}}, + {"suffix appended", []int{1, 2}, []int{1, 2, 3}, []edit{{OpEqual, 2}, {OpInsert, 1}}}, + {"prefix removed", []int{0, 1, 2}, []int{1, 2}, []edit{{OpDelete, 1}, {OpEqual, 2}}}, + {"nothing in common", []int{1, 2}, []int{3, 4}, + []edit{{OpDelete, 2}, {OpInsert, 2}}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := diffInts(tc.a, tc.b) + assert.Equal(t, tc.want, got) + assertScriptApplies(t, tc.a, tc.b, got) + }) + } +} + +// TestDiffIntsMinimalRandom is the real correctness check: on random inputs +// the script must both reconstruct b from a and keep exactly as many elements +// equal as the longest common subsequence. +func TestDiffIntsMinimalRandom(t *testing.T) { + rng := rand.New(rand.NewSource(1)) + for iter := 0; iter < 400; iter++ { + a := randSeq(rng, rng.Intn(14), 5) + b := randSeq(rng, rng.Intn(14), 5) + script := diffInts(a, b) + assertScriptApplies(t, a, b, script) + require.Equal(t, lcsLen(a, b), commonCount(script), + "non-minimal script for a=%v b=%v: %v", a, b, script) + for i := 1; i < len(script); i++ { + require.NotEqual(t, script[i-1].op, script[i].op, "runs not coalesced: %v", script) + } + } +} + +func TestRatio(t *testing.T) { + assert.Equal(t, 1.0, ratio(nil, nil)) + assert.Equal(t, 1.0, ratio([]int{1, 2}, []int{1, 2})) + assert.Equal(t, 0.0, ratio([]int{1, 2}, []int{3, 4})) + assert.Equal(t, 0.5, ratio([]int{1, 2}, []int{1, 3})) +} + +func randSeq(rng *rand.Rand, n, alphabet int) []int { + if n == 0 { + return nil + } + out := make([]int, n) + for i := range out { + out[i] = rng.Intn(alphabet) + } + return out +} + +func assertScriptApplies(t *testing.T, a, b []int, script []edit) { + t.Helper() + var gotA, gotB []int + i, j := 0, 0 + for _, e := range script { + switch e.op { + case OpEqual: + require.LessOrEqual(t, i+e.n, len(a)) + require.LessOrEqual(t, j+e.n, len(b)) + for n := 0; n < e.n; n++ { + require.Equal(t, a[i+n], b[j+n], "equal run over differing elements") + } + gotA = append(gotA, a[i:i+e.n]...) + gotB = append(gotB, b[j:j+e.n]...) + i += e.n + j += e.n + case OpDelete: + gotA = append(gotA, a[i:i+e.n]...) + i += e.n + case OpInsert: + gotB = append(gotB, b[j:j+e.n]...) + j += e.n + } + } + require.Equal(t, len(a), i) + require.Equal(t, len(b), j) + require.Equal(t, a, sliceOrNil(gotA)) + require.Equal(t, b, sliceOrNil(gotB)) +} + +func sliceOrNil(s []int) []int { + if len(s) == 0 { + return nil + } + return s +} + +// lcsLen is the textbook O(nm) LCS, used only as the oracle. +func lcsLen(a, b []int) int { + dp := make([][]int, len(a)+1) + for i := range dp { + dp[i] = make([]int, len(b)+1) + } + for i := 1; i <= len(a); i++ { + for j := 1; j <= len(b); j++ { + if a[i-1] == b[j-1] { + dp[i][j] = dp[i-1][j-1] + 1 + continue + } + dp[i][j] = max(dp[i-1][j], dp[i][j-1]) + } + } + return dp[len(a)][len(b)] +} diff --git a/prosediff/prosediff.go b/prosediff/prosediff.go new file mode 100644 index 0000000000000000000000000000000000000000..e0651d609a161208b553f7144c44efdf9a47004a --- /dev/null +++ b/prosediff/prosediff.go @@ -0,0 +1,140 @@ +// Package prosediff diffs two revisions of a markdown document the way a +// human reads it: as blocks of prose, not as lines of text. +// +// Markdown reflows. A one-word edit rewraps a paragraph, and a line-oriented +// differ then reports the whole paragraph as replaced — useless for reviewing +// written text. This package instead: +// +// 1. segments each revision into blocks (headings, paragraphs, list items, +// code fences, table rows, block quotes) using goldmark's parser; +// 2. aligns the two block sequences, recognising unchanged, added, removed, +// modified and (exactly) moved blocks; +// 3. diffs word-by-word inside a modified prose block, and line-by-line +// inside a modified code fence. +// +// Whitespace and line wrapping alone never produce a diff in prose. They +// always do in code fences, which is the point of separating the two. +// +// The package renders plain text only. Rendering to HTML is the web layer's +// job; Diff is the data structure it consumes. +package prosediff + +// ChangeKind classifies what happened to one block. +type ChangeKind string + +const ( + // ChangeEqual: the block is present unchanged in both revisions. + ChangeEqual ChangeKind = "equal" + // ChangeInsert: the block exists only in the new revision. + ChangeInsert ChangeKind = "insert" + // ChangeDelete: the block exists only in the old revision. + ChangeDelete ChangeKind = "delete" + // ChangeModify: the same block, edited. Words (or Lines) carry the detail. + ChangeModify ChangeKind = "modify" + // ChangeMoveOut marks, at its old position, a block that moved elsewhere. + ChangeMoveOut ChangeKind = "move_out" + // ChangeMoveIn marks, at its new position, a block that moved from elsewhere. + ChangeMoveIn ChangeKind = "move_in" +) + +// BlockChange is one entry of a diff, in reading order: for each changed +// region, the old blocks first and then the new ones. +type BlockChange struct { + Kind ChangeKind + + // Old is the block in the old revision; nil for ChangeInsert. + Old *Block + // New is the block in the new revision; nil for ChangeDelete. + New *Block + + // Words is the inline edit script for a modified prose block. + Words []Span + // Lines is the line-oriented edit script for a modified code fence, + // frontmatter block or HTML block. + Lines []Span + + // Similarity is the token-level similarity that justified pairing a + // ChangeModify, in [0,1]. Zero for every other kind. + Similarity float64 + + // StructureOnly marks a modification whose content is untouched: only + // the heading level, list depth or quote depth changed. + StructureOnly bool + + // Moved marks a ChangeModify whose block also changed position: it sits + // inside a run of moved blocks. Its ChangeMoveOut counterpart appears at + // the old position. + Moved bool +} + +// Stats summarizes a diff, cheap enough for a listing page. +type Stats struct { + BlocksEqual int + BlocksInserted int + BlocksDeleted int + BlocksModified int + BlocksMoved int + WordsInserted int + WordsDeleted int +} + +// Changed reports whether the two revisions differ at all. +func (s Stats) Changed() bool { + return s.BlocksInserted+s.BlocksDeleted+s.BlocksModified+s.BlocksMoved > 0 +} + +// Diff is the whole comparison of two document revisions. +type Diff struct { + OldBlocks []Block + NewBlocks []Block + Changes []BlockChange + Stats Stats +} + +// Compare segments both revisions and aligns them. It has no failure mode: +// any byte slice is a parseable markdown document. +func Compare(oldSrc, newSrc []byte) *Diff { + old := Segment(oldSrc) + nw := Segment(newSrc) + d := &Diff{OldBlocks: old, NewBlocks: nw} + d.Changes = align(old, nw) + d.Stats = computeStats(d.Changes) + return d +} + +func computeStats(changes []BlockChange) Stats { + var s Stats + for _, c := range changes { + switch c.Kind { + case ChangeEqual: + s.BlocksEqual++ + case ChangeInsert: + s.BlocksInserted++ + s.WordsInserted += len(Tokenize(c.New.Text)) + case ChangeDelete: + s.BlocksDeleted++ + s.WordsDeleted += len(Tokenize(c.Old.Text)) + case ChangeModify: + s.BlocksModified++ + for _, sp := range c.Words { + switch sp.Op { + case OpInsert: + s.WordsInserted += len(Tokenize(sp.Text)) + case OpDelete: + s.WordsDeleted += len(Tokenize(sp.Text)) + } + } + for _, sp := range c.Lines { + switch sp.Op { + case OpInsert: + s.WordsInserted += len(Tokenize(sp.Text)) + case OpDelete: + s.WordsDeleted += len(Tokenize(sp.Text)) + } + } + case ChangeMoveIn: + s.BlocksMoved++ + } + } + return s +} diff --git a/prosediff/prosediff_test.go b/prosediff/prosediff_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d0bab7bcf48cf20f6f00dc881cbd7920baaa16c9 --- /dev/null +++ b/prosediff/prosediff_test.go @@ -0,0 +1,417 @@ +package prosediff + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// changeSummary is a compact, readable expectation: one string per change. +func changeSummary(d *Diff) []string { + var out []string + for _, c := range d.Changes { + if c.Kind == ChangeEqual { + continue + } + b := c.New + if b == nil { + b = c.Old + } + out = append(out, string(c.Kind)+" "+string(b.Kind)) + } + return out +} + +// inlineOf returns the word-level diff of the n-th modified block, rendered +// with markers, so tests can assert on what the reviewer would actually see. +func inlineOf(t *testing.T, d *Diff, n int) string { + t.Helper() + seen := 0 + for _, c := range d.Changes { + if c.Kind != ChangeModify { + continue + } + if seen != n { + seen++ + continue + } + var sb strings.Builder + for _, s := range c.Words { + if s.Space && sb.Len() > 0 { + sb.WriteByte(' ') + } + switch s.Op { + case OpEqual: + sb.WriteString(s.Text) + case OpDelete: + sb.WriteString("[-" + s.Text + "-]") + case OpInsert: + sb.WriteString("{+" + s.Text + "+}") + } + } + return sb.String() + } + t.Fatalf("no modified block #%d in diff", n) + return "" +} + +func TestCompare(t *testing.T) { + tests := []struct { + name string + old, new string + want []string // non-equal changes, "kind blockkind" + inline string // expected rendering of the first modification + stats func(t *testing.T, s Stats) + }{ + { + name: "pure reflow is a no-op", + old: "# Doc\n\nMarkdown reflows. A one-word edit renders as a whole-paragraph\n" + + "replace under a line-oriented differ, which makes reviewing agent\noutput miserable.\n", + new: "# Doc\n\nMarkdown reflows. A one-word edit renders as a\nwhole-paragraph replace under a line-oriented differ,\nwhich makes reviewing agent output miserable.\n", + want: nil, + stats: func(t *testing.T, s Stats) { + assert.False(t, s.Changed()) + assert.Equal(t, 2, s.BlocksEqual) + }, + }, + { + name: "single word edit inside a long reflowed paragraph", + old: "Markdown reflows. A one-word edit renders as a whole-paragraph replace\n" + + "under a line-oriented differ, which makes reviewing agent output\nmiserable.\n", + new: "Markdown reflows. A one-word edit renders as a\nwhole-paragraph replace under a line-oriented differ, which makes\n" + + "reviewing agent output unbearable.\n", + want: []string{"modify paragraph"}, + inline: "Markdown reflows. A one-word edit renders as a whole-paragraph replace under a line-oriented differ, which makes reviewing agent output [-miserable-]{+unbearable+}.", + stats: func(t *testing.T, s Stats) { + assert.Equal(t, 1, s.BlocksModified) + assert.Equal(t, 1, s.WordsInserted) + assert.Equal(t, 1, s.WordsDeleted) + }, + }, + { + name: "paragraph added", + old: "One.\n\nThree.\n", + new: "One.\n\nA brand new second paragraph goes here.\n\nThree.\n", + want: []string{"insert paragraph"}, + stats: func(t *testing.T, s Stats) { + assert.Equal(t, 1, s.BlocksInserted) + assert.Equal(t, 2, s.BlocksEqual) + }, + }, + { + name: "paragraph removed", + old: "One.\n\nA whole paragraph that is going away entirely.\n\nThree.\n", + new: "One.\n\nThree.\n", + want: []string{"delete paragraph"}, + stats: func(t *testing.T, s Stats) { + assert.Equal(t, 1, s.BlocksDeleted) + }, + }, + { + name: "heading text changed", + old: "## Prose diff, not line diff\n\nbody\n", + new: "## Prose diff, never line diff\n\nbody\n", + want: []string{"modify heading"}, + inline: "Prose diff, [-not-]{+never+} line diff", + }, + { + name: "heading level changed only", + old: "## The two hard parts\n\nbody\n", + new: "### The two hard parts\n\nbody\n", + want: []string{"modify heading"}, + }, + { + name: "list item edited, siblings untouched", + old: "- alpha stays exactly the same\n- beta gets a small correction here\n- gamma stays too\n", + new: "- alpha stays exactly the same\n- beta gets a large correction here\n- gamma stays too\n", + want: []string{"modify list_item"}, + inline: "beta gets a [-small-]{+large+} correction here", + stats: func(t *testing.T, s Stats) { + assert.Equal(t, 2, s.BlocksEqual) + assert.Equal(t, 1, s.BlocksModified) + }, + }, + { + name: "list item added", + old: "- alpha stays the same\n- gamma stays the same\n", + new: "- alpha stays the same\n- beta is entirely new here\n- gamma stays the same\n", + want: []string{"insert list_item"}, + }, + { + name: "list nesting change is structural", + old: "- alpha the first item\n- beta the second item\n", + new: "- alpha the first item\n - beta the second item\n", + want: []string{"modify list_item"}, + }, + { + name: "code fence line edited", + old: "```go\nx := 1\ny := 2\nz := 3\n```\n", + new: "```go\nx := 1\ny := 22\nz := 3\n```\n", + want: []string{"modify code"}, + }, + { + name: "code fence indentation matters", + old: "```py\nif x:\n y()\n```\n", + new: "```py\nif x:\n\ty()\n```\n", + want: []string{"modify code"}, + }, + { + name: "code fence added", + old: "Some prose here.\n", + new: "Some prose here.\n\n```sh\nmake build\n```\n", + want: []string{"insert code"}, + }, + { + name: "table row edited", + old: "| Decision | Choice |\n|---|---|\n| Review gate | Proposal-first |\n| Storage | Own bare git repos |\n", + new: "| Decision | Choice |\n|---|---|\n| Review gate | Proposal-first |\n| Storage | Own bare git repos, service-owned |\n", + want: []string{"modify table_row"}, + inline: "| Storage | Own bare git repos{+, service-owned+} |", + }, + { + name: "table row added", + old: "| a | b |\n|---|---|\n| one | two |\n", + new: "| a | b |\n|---|---|\n| one | two |\n| three | four |\n", + want: []string{"insert table_row"}, + }, + { + name: "block quote edited", + old: "> bot produces, human curates, bots consume.\n", + new: "> bot produces, human reviews, bots consume.\n", + want: []string{"modify paragraph"}, + inline: "bot produces, human [-curates-]{+reviews+}, bots consume.", + }, + { + name: "frontmatter edited", + old: "---\nid: SPEC-0007\nstatus: draft\n---\n\nBody text.\n", + new: "---\nid: SPEC-0007\nstatus: review\n---\n\nBody text.\n", + want: []string{"modify frontmatter"}, + }, + { + name: "unrelated replacement is not a modification", + old: "The quick brown fox jumps over the lazy dog.\n", + new: "Consistency and recovery is the section that follows.\n", + want: []string{"delete paragraph", "insert paragraph"}, + }, + { + name: "empty to content", + old: "", + new: "# New\n\nBody.\n", + want: []string{"insert heading", "insert paragraph"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + d := Compare([]byte(tc.old), []byte(tc.new)) + assert.Equal(t, tc.want, changeSummary(d)) + if tc.inline != "" { + assert.Equal(t, tc.inline, inlineOf(t, d, 0)) + } + if tc.stats != nil { + tc.stats(t, d.Stats) + } + // The renderer must survive every case. + assert.NotPanics(t, func() { RenderText(d, DefaultRenderOptions()) }) + }) + } +} + +// TestWholeDocumentReflowIsSilent is the design's central claim, checked over +// a document with every block kind in it. +func TestWholeDocumentReflowIsSilent(t *testing.T) { + src := `# Title + +A paragraph that is wrapped at some particular width and says several +things across more than one line of source text. + +- a list item long enough to be rewrapped by an editor at some point +- another list item + +> a block quote that also happens to be wrapped across two source +> lines here + +| a | b | +|---|---| +| 1 | 2 | + +` + "```go\nkeep := \"this exactly\"\n```\n" + + rewrapped := rewrapProse(src, 40) + require.NotEqual(t, src, rewrapped, "test fixture did not actually rewrap") + + d := Compare([]byte(src), []byte(rewrapped)) + assert.False(t, d.Stats.Changed(), "rewrapping produced: %s", RenderText(d, DefaultRenderOptions())) +} + +// rewrapProse rewraps paragraph, list and quote lines to width, leaving fenced +// code and table rows alone. +func rewrapProse(src string, width int) string { + var out []string + inFence := false + var para []string + var prefix string + + flush := func() { + if len(para) == 0 { + return + } + cont := prefix + if prefix == "- " { + cont = " " // continuation of a list item, not a new one + } + for i, l := range wrap(strings.Join(para, " "), width) { + if i == 0 { + out = append(out, prefix+l) + continue + } + out = append(out, cont+l) + } + para = nil + prefix = "" + } + + for _, line := range strings.Split(src, "\n") { + switch { + case strings.HasPrefix(line, "```"): + flush() + inFence = !inFence + out = append(out, line) + case inFence, strings.HasPrefix(line, "|"), strings.HasPrefix(line, "#"), strings.TrimSpace(line) == "": + flush() + out = append(out, line) + case strings.HasPrefix(line, "> "): + prefix = "> " + para = append(para, strings.TrimPrefix(line, "> ")) + case strings.HasPrefix(line, "- "): + flush() + prefix = "- " + para = append(para, strings.TrimPrefix(line, "- ")) + default: + para = append(para, strings.TrimSpace(line)) + } + } + flush() + return strings.Join(out, "\n") +} + +func TestMoveDetection(t *testing.T) { + a := "# Doc\n\n## Alpha\n\nThe alpha section body, long enough to be recognised.\n\n## Beta\n\nThe beta section body, also long enough to be recognised.\n" + b := "# Doc\n\n## Beta\n\nThe beta section body, also long enough to be recognised.\n\n## Alpha\n\nThe alpha section body, long enough to be recognised.\n" + + d := Compare([]byte(a), []byte(b)) + var kinds []ChangeKind + for _, c := range d.Changes { + if c.Kind != ChangeEqual { + kinds = append(kinds, c.Kind) + } + } + require.NotEmpty(t, kinds) + for _, k := range kinds { + assert.Contains(t, []ChangeKind{ChangeMoveIn, ChangeMoveOut}, k, + "a pure reorder should be moves only, got %v\n%s", kinds, RenderText(d, DefaultRenderOptions())) + } + assert.Equal(t, 2, d.Stats.BlocksMoved) +} + +// TestMoveGroupBridgesOneEditedBlock covers the common real case: a section +// is moved and one paragraph inside it is touched. The untouched blocks must +// stay moves and the touched one must be a modification, not four inserts. +func TestMoveGroupBridgesOneEditedBlock(t *testing.T) { + // Alpha is the section that moves; Beta is deliberately the larger one, + // so the alignment keeps Beta in place and Alpha is what has to be + // recognised as moved. + sectionA := "## Alpha\n\nAlpha intro paragraph, long enough to anchor a move.\n\n" + + "Alpha middle paragraph that will be edited slightly.\n\n" + + "Alpha closing paragraph, also long enough to anchor a move.\n" + sectionB := "## Beta\n\nBeta first paragraph, long enough to anchor a move too.\n\n" + + "Beta second paragraph, long enough to anchor a move too.\n\n" + + "Beta third paragraph, long enough to anchor a move too.\n\n" + + "Beta fourth paragraph, long enough to anchor a move too.\n\n" + + "Beta fifth paragraph, long enough to anchor a move too.\n" + + a := "# Doc\n\n" + sectionA + "\n" + sectionB + b := "# Doc\n\n" + sectionB + "\n" + + strings.Replace(sectionA, "edited slightly", "edited a little", 1) + + d := Compare([]byte(a), []byte(b)) + var modified []BlockChange + for _, c := range d.Changes { + switch c.Kind { + case ChangeModify: + modified = append(modified, c) + case ChangeInsert, ChangeDelete: + t.Fatalf("unexpected %s in a move+edit:\n%s", c.Kind, RenderText(d, DefaultRenderOptions())) + } + } + require.Len(t, modified, 1) + assert.True(t, modified[0].Moved, "the edited block should be marked as moved too") + assert.Equal(t, 3, d.Stats.BlocksMoved) +} + +// TestMovedAndEditedIsNotAMove documents the honest limitation: content that +// moved *and* changed, with no verbatim block left to anchor it, shows up as +// a delete plus an insert. +func TestMovedAndEditedIsNotAMove(t *testing.T) { + a := "## Alpha\n\nThe alpha body here.\n\n## Beta\n\nThe beta body here.\n" + b := "## Beta\n\nThe beta body here.\n\n## Alpha\n\nThe alpha body here, now with a tail.\n" + + d := Compare([]byte(a), []byte(b)) + assert.Equal(t, 0, d.Stats.BlocksMoved, + "edited-while-moved must not be claimed as a move:\n%s", RenderText(d, DefaultRenderOptions())) +} + +// TestHeadingRenameDoesNotDirtyItsSection: HeadingPath is context for the +// reviewer, never part of a block's identity. Including it would make +// renaming a section rewrite every block underneath it. +func TestHeadingRenameDoesNotDirtyItsSection(t *testing.T) { + old := "## The old section name\n\nFirst paragraph of the section.\n\nSecond paragraph of the section.\n" + nw := "## The new section name\n\nFirst paragraph of the section.\n\nSecond paragraph of the section.\n" + d := Compare([]byte(old), []byte(nw)) + assert.Equal(t, 2, d.Stats.BlocksEqual) + assert.Equal(t, 1, d.Stats.BlocksModified) +} + +// TestShortBlockRenameFallsBackToAddRemove pins the other side of the +// similarity floor: two blocks too short to judge are reported as a removal +// and an addition rather than paired on a coin-flip score. +func TestShortBlockRenameFallsBackToAddRemove(t *testing.T) { + d := Compare([]byte("## Old name\n\nbody\n"), []byte("## New name\n\nbody\n")) + assert.Equal(t, 0, d.Stats.BlocksModified) + assert.Equal(t, 1, d.Stats.BlocksInserted) + assert.Equal(t, 1, d.Stats.BlocksDeleted) +} + +func TestHeadingPathIsCarried(t *testing.T) { + src := "# Top\n\n## Middle\n\n### Leaf\n\nbody\n" + blocks := Segment([]byte(src)) + last := blocks[len(blocks)-1] + assert.Equal(t, []string{"Top", "Middle", "Leaf"}, last.HeadingPath) +} + +func TestRenderText(t *testing.T) { + d := Compare( + []byte("# Title\n\nThe quick brown fox jumps over the lazy dog every single day.\n"), + []byte("# Title\n\nThe quick red fox jumps over the lazy dog every single day.\n"), + ) + out := RenderText(d, DefaultRenderOptions()) + assert.Contains(t, out, "@@ Title @@") + assert.Contains(t, out, "[-brown-]") + assert.Contains(t, out, "{+red+}") + assert.NotContains(t, out, "\n L1 h1") // equal blocks hidden by default + + full := RenderText(d, RenderOptions{Width: 80, ShowEqual: true}) + assert.Contains(t, full, "Title") +} + +func TestRenderTextCodeIsLineOriented(t *testing.T) { + d := Compare( + []byte("```go\na := 1\nb := 2\n```\n"), + []byte("```go\na := 1\nb := 3\n```\n"), + ) + out := RenderText(d, DefaultRenderOptions()) + assert.Contains(t, out, " - b := 2") + assert.Contains(t, out, " + b := 3") +} diff --git a/prosediff/render.go b/prosediff/render.go new file mode 100644 index 0000000000000000000000000000000000000000..23e29f6048ac32c30040be10c5c45bfad10f710d --- /dev/null +++ b/prosediff/render.go @@ -0,0 +1,216 @@ +package prosediff + +import ( + "fmt" + "strings" + "unicode/utf8" +) + +// RenderOptions controls the plain-text renderer. It exists for tests and for +// reading a diff in a terminal; the web layer walks Diff.Changes itself. +type RenderOptions struct { + // Width wraps rendered prose. 0 means do not wrap. + Width int + // ShowEqual prints unchanged blocks too, instead of collapsing them. + ShowEqual bool + // Context is how many unchanged blocks to keep around a change when + // ShowEqual is false. 0 keeps none. + Context int + // Markers are the inline delete/insert brackets. Empty uses the + // defaults "[-", "-]", "{+", "+}". + DelOpen, DelClose, InsOpen, InsClose string +} + +// DefaultRenderOptions wraps at 80 columns and hides unchanged blocks. +func DefaultRenderOptions() RenderOptions { + return RenderOptions{Width: 80, Context: 0} +} + +func (o RenderOptions) markers() (string, string, string, string) { + d0, d1, i0, i1 := o.DelOpen, o.DelClose, o.InsOpen, o.InsClose + if d0 == "" && d1 == "" && i0 == "" && i1 == "" { + return "[-", "-]", "{+", "+}" + } + return d0, d1, i0, i1 +} + +// RenderText renders a diff as plain text. +func RenderText(d *Diff, opts RenderOptions) string { + var sb strings.Builder + keep := visible(d.Changes, opts) + + lastPath := "\x00" + skipped := 0 + flushSkipped := func() { + if skipped > 0 { + fmt.Fprintf(&sb, " … %d unchanged block(s)\n", skipped) + skipped = 0 + } + } + + for i, c := range d.Changes { + if !keep[i] { + if c.Kind == ChangeEqual { + skipped++ + } + continue + } + flushSkipped() + b := c.New + if b == nil { + b = c.Old + } + if p := strings.Join(b.HeadingPath, " › "); p != lastPath { + if p == "" { + p = "(document preamble)" + } + fmt.Fprintf(&sb, "\n@@ %s @@\n", p) + lastPath = p + } + sb.WriteString(renderChange(c, opts)) + } + flushSkipped() + return sb.String() +} + +func visible(changes []BlockChange, opts RenderOptions) []bool { + keep := make([]bool, len(changes)) + for i, c := range changes { + if c.Kind != ChangeEqual || opts.ShowEqual { + keep[i] = true + } + } + if opts.Context > 0 && !opts.ShowEqual { + orig := append([]bool(nil), keep...) + for i := range changes { + if !orig[i] { + continue + } + for j := i - opts.Context; j <= i+opts.Context; j++ { + if j >= 0 && j < len(keep) { + keep[j] = true + } + } + } + } + return keep +} + +func renderChange(c BlockChange, opts RenderOptions) string { + var sb strings.Builder + d0, d1, i0, i1 := opts.markers() + + switch c.Kind { + case ChangeEqual: + fmt.Fprintf(&sb, " %s %s\n", loc(c.Old, c.New), c.New.Label()) + writeBody(&sb, " ", c.New.Text, opts.Width, c.New.Kind.Prose()) + + case ChangeInsert: + fmt.Fprintf(&sb, "+ %s %s\n", loc(nil, c.New), c.New.Label()) + writeBody(&sb, "+ ", c.New.Text, opts.Width, c.New.Kind.Prose()) + + case ChangeDelete: + fmt.Fprintf(&sb, "- %s %s\n", loc(c.Old, nil), c.Old.Label()) + writeBody(&sb, "- ", c.Old.Text, opts.Width, c.Old.Kind.Prose()) + + case ChangeMoveOut: + fmt.Fprintf(&sb, "< %s %s moved away (now line %d)\n", + loc(c.Old, nil), c.Old.Label(), c.New.StartLine) + + case ChangeMoveIn: + fmt.Fprintf(&sb, "> %s %s moved here (was line %d)\n", + loc(nil, c.New), c.New.Label(), c.Old.StartLine) + + case ChangeModify: + note := "" + if c.StructureOnly { + note = fmt.Sprintf(" (structure only: %s → %s)", c.Old.Label(), c.New.Label()) + } + if c.Moved { + note += fmt.Sprintf(" (moved from line %d)", c.Old.StartLine) + } + fmt.Fprintf(&sb, "~ %s %s modified%s\n", loc(c.Old, c.New), c.New.Label(), note) + if len(c.Lines) > 0 { + for _, s := range c.Lines { + switch s.Op { + case OpEqual: + fmt.Fprintf(&sb, " %s\n", s.Text) + case OpDelete: + fmt.Fprintf(&sb, " - %s\n", s.Text) + case OpInsert: + fmt.Fprintf(&sb, " + %s\n", s.Text) + } + } + break + } + var body strings.Builder + for _, s := range c.Words { + if s.Space && body.Len() > 0 { + body.WriteByte(' ') + } + switch s.Op { + case OpEqual: + body.WriteString(s.Text) + case OpDelete: + body.WriteString(d0 + s.Text + d1) + case OpInsert: + body.WriteString(i0 + s.Text + i1) + } + } + writeBody(&sb, " ", body.String(), opts.Width, true) + } + return sb.String() +} + +func loc(old, nw *Block) string { + switch { + case old != nil && nw != nil: + if old.StartLine == nw.StartLine { + return fmt.Sprintf("L%d", old.StartLine) + } + return fmt.Sprintf("L%d→%d", old.StartLine, nw.StartLine) + case old != nil: + return fmt.Sprintf("L%d", old.StartLine) + case nw != nil: + return fmt.Sprintf("L%d", nw.StartLine) + } + return "L?" +} + +func writeBody(sb *strings.Builder, prefix, body string, width int, reflow bool) { + var lines []string + if reflow { + lines = wrap(body, width-len(prefix)) + } else { + lines = strings.Split(body, "\n") + } + for _, line := range lines { + sb.WriteString(prefix) + sb.WriteString(line) + sb.WriteByte('\n') + } +} + +// wrap reflows prose to width columns. Prose is always rewrapped: a diff that +// preserved the source wrapping would put the reviewer back where a line +// differ left them. +func wrap(s string, width int) []string { + fields := strings.Fields(s) + if len(fields) == 0 { + return []string{""} + } + if width <= 0 { + return []string{strings.Join(fields, " ")} + } + var out []string + line := fields[0] + for _, f := range fields[1:] { + if utf8.RuneCountInString(line)+1+utf8.RuneCountInString(f) > width { + out = append(out, line) + line = f + continue + } + line += " " + f + } + return append(out, line) +} diff --git a/prosediff/segment.go b/prosediff/segment.go new file mode 100644 index 0000000000000000000000000000000000000000..74db95e328792ddf91680d2f55e330ceaa6343ce --- /dev/null +++ b/prosediff/segment.go @@ -0,0 +1,457 @@ +package prosediff + +import ( + "fmt" + "hash/fnv" + "sort" + "strings" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/extension" + extast "github.com/yuin/goldmark/extension/ast" + "github.com/yuin/goldmark/text" +) + +// BlockKind is the structural role of a block. It is part of a block's +// identity: a paragraph that becomes a list item is not a modified paragraph. +type BlockKind string + +const ( + KindFrontmatter BlockKind = "frontmatter" + KindHeading BlockKind = "heading" + KindParagraph BlockKind = "paragraph" + KindListItem BlockKind = "list_item" + KindCode BlockKind = "code" + KindTableHeader BlockKind = "table_header" + KindTableRow BlockKind = "table_row" + KindThematicBreak BlockKind = "rule" + KindHTML BlockKind = "html" +) + +// Prose reports whether this kind diffs word-by-word. Everything else diffs +// line-by-line, which is the code-fence distinction the design calls for. +func (k BlockKind) Prose() bool { + switch k { + case KindCode, KindFrontmatter, KindHTML: + return false + } + return true +} + +// Block is one addressable unit of a document: a paragraph, a heading, a +// single list item, a whole code fence, one table row. +// +// The tuple (HeadingPath, Kind, Ordinal, Hash) is deliberately the anchor +// tuple the design names for inline comments; nothing here is diff-only. +type Block struct { + Kind BlockKind + Ordinal int // 0-based position in the document + Level int // heading level, or list nesting depth for list items + QuoteDepth int // blockquote nesting, 0 outside any quote + HeadingPath []string // enclosing headings, outermost first + Text string // source text of the block, as written + Lines []string // source lines; the diff unit for non-prose kinds + Info string // code fence language, list marker, table column count + StartLine int // 1-based, inclusive + EndLine int // 1-based, inclusive + Hash string // structure + normalized content +} + +// Empty reports whether the block carries no content at all. +func (b Block) Empty() bool { return strings.TrimSpace(b.Text) == "" } + +// Label is a short human description, used by the text renderer and usable +// as-is by a future web layer. +func (b Block) Label() string { + s := string(b.Kind) + switch b.Kind { + case KindHeading: + s = fmt.Sprintf("h%d", b.Level) + case KindListItem: + if b.Level > 1 { + s = fmt.Sprintf("list item (depth %d)", b.Level) + } else { + s = "list item" + } + case KindCode: + if b.Info != "" { + s = "code:" + b.Info + } + } + if b.QuoteDepth > 0 { + s = strings.Repeat("quoted ", b.QuoteDepth) + s + } + return s +} + +// Segment splits a markdown document into blocks in document order. +func Segment(src []byte) []Block { + body, fm, lineOffset := splitFrontmatter(src) + + s := &segmenter{src: body, lineStarts: lineStarts(body), lineOffset: lineOffset} + if fm != nil { + s.blocks = append(s.blocks, finishBlock(Block{ + Kind: KindFrontmatter, + Lines: fm, + Text: strings.Join(fm, "\n"), + StartLine: 1, + EndLine: len(fm), + })) + } + + md := goldmark.New(goldmark.WithExtensions(extension.Table)) + doc := md.Parser().Parse(text.NewReader(body)) + s.walk(doc, ctx{}) + + for i := range s.blocks { + s.blocks[i].Ordinal = i + } + return s.blocks +} + +// splitFrontmatter peels a leading YAML frontmatter fence off the document. +// goldmark would otherwise parse "---" as a thematic break and the keys as a +// paragraph, which diffs badly and is not what the block is. +func splitFrontmatter(src []byte) (body []byte, fm []string, lineOffset int) { + s := string(src) + if !strings.HasPrefix(s, "---\n") && s != "---" { + return src, nil, 0 + } + rest := s[4:] + end := strings.Index(rest, "\n---") + if end < 0 { + return src, nil, 0 + } + tail := rest[end+4:] + if tail != "" && !strings.HasPrefix(tail, "\n") { + return src, nil, 0 + } + fm = strings.Split(s[:end+8], "\n") + if tail != "" { + tail = tail[1:] + } + return []byte(tail), fm, len(fm) +} + +type ctx struct { + headings []string + quoteDepth int + listDepth int + marker string +} + +type segmenter struct { + src []byte + lineStarts []int + lineOffset int + blocks []Block + // headingStack holds (level, text) of the currently open headings. + headingStack []headingEntry +} + +type headingEntry struct { + level int + text string +} + +func (s *segmenter) walk(n ast.Node, c ctx) { + for child := n.FirstChild(); child != nil; child = child.NextSibling() { + s.node(child, c) + } +} + +func (s *segmenter) node(n ast.Node, c ctx) { + switch v := n.(type) { + case *ast.Heading: + txt := s.linesText(v.Lines()) + start, end := s.segLines(v.Lines()) + s.emit(Block{ + Kind: KindHeading, + Level: v.Level, + QuoteDepth: c.quoteDepth, + HeadingPath: s.currentPath(), + Text: txt, + Lines: splitLines(txt), + StartLine: start, + EndLine: end, + }) + s.pushHeading(v.Level, txt) + + case *ast.Paragraph, *ast.TextBlock: + lines := n.Lines() + txt := s.linesText(lines) + if strings.TrimSpace(txt) == "" { + return + } + start, end := s.segLines(lines) + kind := KindParagraph + level := 0 + if c.listDepth > 0 { + kind = KindListItem + level = c.listDepth + } + s.emit(Block{ + Kind: kind, + Level: level, + QuoteDepth: c.quoteDepth, + HeadingPath: s.currentPath(), + Text: txt, + Lines: splitLines(txt), + Info: c.marker, + StartLine: start, + EndLine: end, + }) + + case *ast.FencedCodeBlock: + txt := s.linesText(v.Lines()) + start, end := s.segLines(v.Lines()) + info := "" + if v.Info != nil { + seg := v.Info.Segment + info = string(seg.Value(s.src)) + } + s.emit(Block{ + Kind: KindCode, + QuoteDepth: c.quoteDepth, + Level: c.listDepth, + HeadingPath: s.currentPath(), + Text: txt, + Lines: splitLines(txt), + Info: info, + StartLine: start, + EndLine: end, + }) + + case *ast.CodeBlock: + txt := s.linesText(v.Lines()) + start, end := s.segLines(v.Lines()) + s.emit(Block{ + Kind: KindCode, + QuoteDepth: c.quoteDepth, + Level: c.listDepth, + HeadingPath: s.currentPath(), + Text: txt, + Lines: splitLines(txt), + Info: "indented", + StartLine: start, + EndLine: end, + }) + + case *ast.HTMLBlock: + txt := s.linesText(v.Lines()) + start, end := s.segLines(v.Lines()) + s.emit(Block{ + Kind: KindHTML, + QuoteDepth: c.quoteDepth, + HeadingPath: s.currentPath(), + Text: txt, + Lines: splitLines(txt), + StartLine: start, + EndLine: end, + }) + + case *ast.ThematicBreak: + line := s.lineOf(nodeStart(n)) + s.emit(Block{ + Kind: KindThematicBreak, + QuoteDepth: c.quoteDepth, + HeadingPath: s.currentPath(), + Text: "---", + Lines: []string{"---"}, + StartLine: line, + EndLine: line, + }) + + case *ast.List: + inner := c + inner.listDepth = c.listDepth + 1 + inner.marker = string(rune(v.Marker)) + if v.IsOrdered() { + inner.marker = "ordered" + } + s.walk(v, inner) + + case *ast.ListItem: + s.walk(v, c) + + case *ast.Blockquote: + inner := c + inner.quoteDepth = c.quoteDepth + 1 + s.walk(v, inner) + + case *extast.Table: + s.table(v, c) + + default: + // Containers we do not model explicitly still get descended into, + // so no content is silently dropped. + if n.Type() == ast.TypeBlock && n.HasChildren() { + s.walk(n, c) + } + } +} + +func (s *segmenter) table(t *extast.Table, c ctx) { + cols := len(t.Alignments) + for row := t.FirstChild(); row != nil; row = row.NextSibling() { + kind := KindTableRow + if row.Kind() == extast.KindTableHeader { + kind = KindTableHeader + } + start, stop := nodeSpan(row) + if start < 0 { + continue + } + txt := strings.TrimRight(s.sourceLineRange(start, stop), "\n") + line := s.lineOf(start) + s.emit(Block{ + Kind: kind, + QuoteDepth: c.quoteDepth, + HeadingPath: s.currentPath(), + Text: txt, + Lines: splitLines(txt), + Info: fmt.Sprintf("%d cols", cols), + StartLine: line, + EndLine: s.lineOf(stop), + }) + } +} + +func (s *segmenter) emit(b Block) { + s.blocks = append(s.blocks, finishBlock(b)) +} + +// finishBlock computes the identity hash: structure plus normalized content. +// Prose normalizes through the tokenizer (so wrapping does not count); code +// and frontmatter keep their lines verbatim (so whitespace does count). +func finishBlock(b Block) Block { + h := fnv.New64a() + fmt.Fprintf(h, "%s\x00%d\x00%d\x00", b.Kind, b.Level, b.QuoteDepth) + if b.Kind.Prose() { + h.Write([]byte(Normalize(b.Text))) + } else { + h.Write([]byte(b.Info)) + h.Write([]byte{0}) + h.Write([]byte(strings.Join(b.Lines, "\n"))) + } + b.Hash = fmt.Sprintf("%016x", h.Sum64()) + return b +} + +func (s *segmenter) pushHeading(level int, txt string) { + for len(s.headingStack) > 0 && s.headingStack[len(s.headingStack)-1].level >= level { + s.headingStack = s.headingStack[:len(s.headingStack)-1] + } + s.headingStack = append(s.headingStack, headingEntry{level: level, text: strings.TrimSpace(txt)}) +} + +func (s *segmenter) currentPath() []string { + if len(s.headingStack) == 0 { + return nil + } + out := make([]string, len(s.headingStack)) + for i, e := range s.headingStack { + out[i] = e.text + } + return out +} + +func (s *segmenter) linesText(segs *text.Segments) string { + if segs == nil || segs.Len() == 0 { + return "" + } + var sb strings.Builder + for i := 0; i < segs.Len(); i++ { + seg := segs.At(i) + sb.Write(seg.Value(s.src)) + } + return strings.TrimRight(sb.String(), "\n") +} + +func (s *segmenter) segLines(segs *text.Segments) (int, int) { + if segs == nil || segs.Len() == 0 { + return 0, 0 + } + return s.lineOf(segs.At(0).Start), s.lineOf(segs.At(segs.Len()-1).Stop - 1) +} + +// sourceLineRange expands a byte span to whole source lines, which is how a +// table row (whose AST node carries only inline segments) recovers the pipe +// syntax the reviewer actually wrote. +func (s *segmenter) sourceLineRange(start, stop int) string { + if start < 0 || stop > len(s.src) || start > stop { + return "" + } + for start > 0 && s.src[start-1] != '\n' { + start-- + } + for stop < len(s.src) && s.src[stop] != '\n' { + stop++ + } + return string(s.src[start:stop]) +} + +func (s *segmenter) lineOf(off int) int { + i := sort.SearchInts(s.lineStarts, off+1) - 1 + if i < 0 { + i = 0 + } + return i + 1 + s.lineOffset +} + +func lineStarts(src []byte) []int { + out := []int{0} + for i, c := range src { + if c == '\n' { + out = append(out, i+1) + } + } + return out +} + +func splitLines(s string) []string { + if s == "" { + return nil + } + return strings.Split(s, "\n") +} + +// nodeSpan returns the byte range covered by a node's descendant text +// segments, or (-1, -1) when the node carries none. +func nodeSpan(n ast.Node) (int, int) { + start, stop := -1, -1 + consider := func(a, b int) { + if start < 0 || a < start { + start = a + } + if b > stop { + stop = b + } + } + var visit func(ast.Node) + visit = func(n ast.Node) { + if t, ok := n.(*ast.Text); ok { + consider(t.Segment.Start, t.Segment.Stop) + } + // Lines() panics on inline nodes, so it is only asked of blocks. + if n.Type() == ast.TypeBlock { + if lines := n.Lines(); lines != nil && lines.Len() > 0 { + consider(lines.At(0).Start, lines.At(lines.Len()-1).Stop) + } + } + for c := n.FirstChild(); c != nil; c = c.NextSibling() { + visit(c) + } + } + visit(n) + return start, stop +} + +func nodeStart(n ast.Node) int { + start, _ := nodeSpan(n) + if start < 0 { + return 0 + } + return start +} diff --git a/prosediff/segment_test.go b/prosediff/segment_test.go new file mode 100644 index 0000000000000000000000000000000000000000..39a6a0d7cce71ae619c515af128ef930006e2e40 --- /dev/null +++ b/prosediff/segment_test.go @@ -0,0 +1,150 @@ +package prosediff + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type wantBlock struct { + kind BlockKind + level int + quote int + text string + path []string +} + +func TestSegment(t *testing.T) { + tests := []struct { + name string + src string + want []wantBlock + }{ + { + name: "headings and paragraphs", + src: "# Title\n\nFirst para\nwrapped over lines.\n\n## Sub\n\nSecond para.\n", + want: []wantBlock{ + {kind: KindHeading, level: 1, text: "Title"}, + {kind: KindParagraph, text: "First para\nwrapped over lines.", path: []string{"Title"}}, + {kind: KindHeading, level: 2, text: "Sub", path: []string{"Title"}}, + {kind: KindParagraph, text: "Second para.", path: []string{"Title", "Sub"}}, + }, + }, + { + name: "list items are separate blocks", + src: "- alpha\n- beta\n - nested\n", + want: []wantBlock{ + {kind: KindListItem, level: 1, text: "alpha"}, + {kind: KindListItem, level: 1, text: "beta"}, + {kind: KindListItem, level: 2, text: "nested"}, + }, + }, + { + name: "fenced code is one block", + src: "text\n\n```go\na := 1\nb := 2\n```\n", + want: []wantBlock{ + {kind: KindParagraph, text: "text"}, + {kind: KindCode, text: "a := 1\nb := 2"}, + }, + }, + { + name: "table rows are separate blocks", + src: "| a | b |\n|---|---|\n| 1 | 2 |\n| 3 | 4 |\n", + want: []wantBlock{ + {kind: KindTableHeader, text: "| a | b |"}, + {kind: KindTableRow, text: "| 1 | 2 |"}, + {kind: KindTableRow, text: "| 3 | 4 |"}, + }, + }, + { + name: "block quote carries depth", + src: "> quoted text\n> continues\n", + want: []wantBlock{ + {kind: KindParagraph, quote: 1, text: "quoted text\ncontinues"}, + }, + }, + { + name: "frontmatter is its own block", + src: "---\nid: SPEC-0007\ntitle: x\n---\n\nBody.\n", + want: []wantBlock{ + {kind: KindFrontmatter, text: "---\nid: SPEC-0007\ntitle: x\n---"}, + {kind: KindParagraph, text: "Body."}, + }, + }, + { + name: "empty document", + src: "", + want: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := Segment([]byte(tc.src)) + require.Len(t, got, len(tc.want)) + for i, w := range tc.want { + b := got[i] + assert.Equal(t, w.kind, b.Kind, "block %d kind", i) + assert.Equal(t, w.level, b.Level, "block %d level", i) + assert.Equal(t, w.quote, b.QuoteDepth, "block %d quote depth", i) + assert.Equal(t, w.text, b.Text, "block %d text", i) + assert.Equal(t, w.path, b.HeadingPath, "block %d heading path", i) + assert.Equal(t, i, b.Ordinal) + assert.NotEmpty(t, b.Hash) + } + }) + } +} + +func TestSegmentLineNumbers(t *testing.T) { + src := "# Title\n\npara one\n\n```\ncode\n```\n" + got := Segment([]byte(src)) + require.Len(t, got, 3) + assert.Equal(t, 1, got[0].StartLine) + assert.Equal(t, 3, got[1].StartLine) + assert.Equal(t, 6, got[2].StartLine) +} + +func TestSegmentFrontmatterLineOffset(t *testing.T) { + src := "---\nid: X\n---\n\n# Title\n" + got := Segment([]byte(src)) + require.Len(t, got, 2) + assert.Equal(t, KindFrontmatter, got[0].Kind) + assert.Equal(t, 5, got[1].StartLine) +} + +// TestHashIgnoresWrappingForProse and its code counterpart are the two halves +// of the design's "code fences diff as code, prose diffs as prose" rule. +func TestHashIgnoresWrappingForProse(t *testing.T) { + a := Segment([]byte("one two three four\nfive six\n")) + b := Segment([]byte("one two\nthree four five six\n")) + require.Len(t, a, 1) + require.Len(t, b, 1) + assert.Equal(t, a[0].Hash, b[0].Hash) +} + +func TestHashRespectsWhitespaceForCode(t *testing.T) { + a := Segment([]byte("```\nif x:\n y()\n```\n")) + b := Segment([]byte("```\nif x:\n\ty()\n```\n")) + require.Len(t, a, 1) + require.Len(t, b, 1) + assert.NotEqual(t, a[0].Hash, b[0].Hash) +} + +func TestHashRespectsStructure(t *testing.T) { + para := Segment([]byte("same words here\n")) + item := Segment([]byte("- same words here\n")) + require.Len(t, para, 1) + require.Len(t, item, 1) + assert.NotEqual(t, para[0].Hash, item[0].Hash) +} + +func TestKindProse(t *testing.T) { + for _, k := range []BlockKind{KindHeading, KindParagraph, KindListItem, KindTableRow, KindTableHeader, KindThematicBreak} { + assert.True(t, k.Prose(), "%s should diff as prose", k) + } + for _, k := range []BlockKind{KindCode, KindFrontmatter, KindHTML} { + assert.False(t, k.Prose(), "%s should diff as code", k) + } +} diff --git a/prosediff/token.go b/prosediff/token.go new file mode 100644 index 0000000000000000000000000000000000000000..e65c69cf647b8f058be9b80e813ad96ac08befb7 --- /dev/null +++ b/prosediff/token.go @@ -0,0 +1,273 @@ +package prosediff + +import ( + "strings" + "unicode" + "unicode/utf8" +) + +// Token is one unit of the word-level diff. +// +// Whitespace is *not* a token. It survives only as Space, which records that +// some run of whitespace separated this token from the previous one. That is +// what makes reflow invisible: a paragraph rewrapped from 72 to 80 columns +// produces byte-for-byte the same token stream, because "\n" and " " both +// collapse to Space=true. +type Token struct { + Text string + Space bool +} + +// Tokenize splits prose into words and punctuation. +// +// A word is a run of letters/digits, optionally joined by an internal +// apostrophe or hyphen ("don't", "word-level") or an internal dot between +// digits ("v5.19.1"). Every other non-space rune forms a token from its own +// repeated run, so "**" and "---" stay single tokens rather than exploding +// into markup noise. +func Tokenize(s string) []Token { + var out []Token + space := false + for i := 0; i < len(s); { + r, sz := utf8.DecodeRuneInString(s[i:]) + switch { + case unicode.IsSpace(r): + space = true + i += sz + case isWordRune(r): + j := i + for j < len(s) { + r2, sz2 := utf8.DecodeRuneInString(s[j:]) + if isWordRune(r2) { + j += sz2 + continue + } + if isJoinRune(r2) && continuesWord(s, j+sz2, r2) { + j += sz2 + continue + } + break + } + out = append(out, Token{Text: s[i:j], Space: space}) + space = false + i = j + default: + j := i + for j < len(s) { + r2, sz2 := utf8.DecodeRuneInString(s[j:]) + if r2 != r { + break + } + j += sz2 + } + out = append(out, Token{Text: s[i:j], Space: space}) + space = false + i = j + } + } + return out +} + +func isWordRune(r rune) bool { + return unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' +} + +func isJoinRune(r rune) bool { return r == '\'' || r == '-' || r == '.' || r == '’' } + +// continuesWord reports whether the join rune at the previous position is +// followed by more word material (and, for a dot, by a digit — so "end." does +// not swallow the sentence terminator). +func continuesWord(s string, next int, join rune) bool { + if next >= len(s) { + return false + } + r, _ := utf8.DecodeRuneInString(s[next:]) + if join == '.' { + return unicode.IsDigit(r) + } + return isWordRune(r) +} + +// TokenTexts drops the spacing, leaving the comparable content. +func TokenTexts(toks []Token) []string { + out := make([]string, len(toks)) + for i, t := range toks { + out[i] = t.Text + } + return out +} + +// Normalize collapses a block's source text to its token stream joined by +// single spaces. Two blocks with the same Normalize are the same prose, +// however they were wrapped. +func Normalize(s string) string { + return strings.Join(TokenTexts(Tokenize(s)), " ") +} + +// Span is a run of word-level tokens sharing one Op, ready to render. +// +// Space says whether to emit a separating space before this span when the +// spans are rendered in order. An insertion that directly replaces a deletion +// carries Space=false, because the deletion before it already carried the +// separator; otherwise every one-word substitution would render as +// "[-old-] {+new+}". +type Span struct { + Op Op + Text string // tokens joined with single spaces + Space bool +} + +// smallEqualRun is the character budget below which an unchanged run wedged +// between two changed runs is absorbed into the change. Without it, a rewrite +// that happens to keep a comma or an "a" produces shredded output like +// "[-x-]{+y+} , [-z-]{+w+}". +const smallEqualRun = 4 + +// DiffWords produces the inline edit script between two pieces of prose. +// Whitespace differences alone yield a single OpEqual span. +func DiffWords(oldText, newText string) []Span { + a := Tokenize(oldText) + b := Tokenize(newText) + in := newInterner() + script := diffInts(in.all(TokenTexts(a)), in.all(TokenTexts(b))) + script = absorbSmallEqualRuns(script, a) + return spans(deletesFirst(script), a, b) +} + +// absorbSmallEqualRuns rewrites tiny equal runs that sit between two changed +// runs into delete+insert, so the surrounding change reads as one edit. +func absorbSmallEqualRuns(script []edit, a []Token) []edit { + if len(script) < 3 { + return script + } + // Position of each run in a, needed to measure the equal run's length. + posA := make([]int, len(script)) + x := 0 + for i, e := range script { + posA[i] = x + if e.op != OpInsert { + x += e.n + } + } + out := make([]edit, 0, len(script)+4) + for i, e := range script { + if e.op != OpEqual || i == 0 || i == len(script)-1 { + out = append(out, e) + continue + } + n := 0 + for _, t := range a[posA[i] : posA[i]+e.n] { + n += len(t.Text) + } + if n >= smallEqualRun { + out = append(out, e) + continue + } + out = append(out, edit{OpDelete, e.n}, edit{OpInsert, e.n}) + } + return coalesce(out) +} + +// deletesFirst rewrites each changed region so all deletions precede all +// insertions, whichever order Myers happened to emit them in. +func deletesFirst(script []edit) []edit { + out := make([]edit, 0, len(script)) + for i := 0; i < len(script); { + if script[i].op == OpEqual { + out = append(out, script[i]) + i++ + continue + } + del, ins := 0, 0 + for ; i < len(script) && script[i].op != OpEqual; i++ { + if script[i].op == OpDelete { + del += script[i].n + } else { + ins += script[i].n + } + } + if del > 0 { + out = append(out, edit{OpDelete, del}) + } + if ins > 0 { + out = append(out, edit{OpInsert, ins}) + } + } + return out +} + +// spans walks an edit script and materializes it into renderable runs. +func spans(script []edit, a, b []Token) []Span { + var out []Span + emit := func(op Op, toks []Token) { + if len(toks) == 0 { + return + } + var sb strings.Builder + for i, t := range toks { + if i > 0 && t.Space { + sb.WriteByte(' ') + } + sb.WriteString(t.Text) + } + out = append(out, Span{Op: op, Text: sb.String(), Space: toks[0].Space}) + } + + i, j := 0, 0 + for k := 0; k < len(script); k++ { + e := script[k] + // An insertion directly replacing a deletion must not re-announce + // the whitespace the deletion already carried, or every one-word + // substitution renders as "[-old-] {+new+}". + if e.op == OpInsert && k > 0 && script[k-1].op == OpDelete { + emit(OpInsert, b[j:j+e.n]) + if len(out) > 0 { + out[len(out)-1].Space = false + } + j += e.n + continue + } + switch e.op { + case OpEqual: + emit(OpEqual, a[i:i+e.n]) + i += e.n + j += e.n + case OpDelete: + emit(OpDelete, a[i:i+e.n]) + i += e.n + case OpInsert: + emit(OpInsert, b[j:j+e.n]) + j += e.n + } + } + return out +} + +// DiffLines is the code-fence path: line-oriented, whitespace-significant. +func DiffLines(oldLines, newLines []string) []Span { + in := newInterner() + script := diffInts(in.all(oldLines), in.all(newLines)) + var out []Span + i, j := 0, 0 + for _, e := range script { + switch e.op { + case OpEqual: + for n := 0; n < e.n; n++ { + out = append(out, Span{Op: OpEqual, Text: oldLines[i+n]}) + } + i += e.n + j += e.n + case OpDelete: + for n := 0; n < e.n; n++ { + out = append(out, Span{Op: OpDelete, Text: oldLines[i+n]}) + } + i += e.n + case OpInsert: + for n := 0; n < e.n; n++ { + out = append(out, Span{Op: OpInsert, Text: newLines[j+n]}) + } + j += e.n + } + } + return out +} diff --git a/prosediff/token_test.go b/prosediff/token_test.go new file mode 100644 index 0000000000000000000000000000000000000000..439e388553b8db519ff8b21f610620d80a5d727f --- /dev/null +++ b/prosediff/token_test.go @@ -0,0 +1,140 @@ +package prosediff + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTokenize(t *testing.T) { + tests := []struct { + name string + in string + want []string + space []bool + }{ + {"plain words", "one two", []string{"one", "two"}, []bool{false, true}}, + {"punctuation splits", "end.", []string{"end", "."}, []bool{false, false}}, + {"apostrophe joins", "don't stop", []string{"don't", "stop"}, []bool{false, true}}, + {"hyphen joins", "word-level diff", []string{"word-level", "diff"}, []bool{false, true}}, + {"version numbers stay whole", "v5.19.1", []string{"v5.19.1"}, nil}, + {"sentence dot still splits", "done. next", []string{"done", ".", "next"}, nil}, + {"emphasis is one token", "**bold**", []string{"**", "bold", "**"}, nil}, + {"em dash stands alone", "a — b", []string{"a", "—", "b"}, []bool{false, true, true}}, + {"newline is just space", "a\nb", []string{"a", "b"}, []bool{false, true}}, + {"cyrillic is a word", "спецификация готова", []string{"спецификация", "готова"}, nil}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + toks := Tokenize(tc.in) + assert.Equal(t, tc.want, TokenTexts(toks)) + if tc.space != nil { + got := make([]bool, len(toks)) + for i, tk := range toks { + got[i] = tk.Space + } + assert.Equal(t, tc.space, got) + } + }) + } +} + +// TestNormalizeIgnoresWrapping is the load-bearing property of the whole +// package: rewrapping prose must not change its normalized form. +func TestNormalizeIgnoresWrapping(t *testing.T) { + narrow := "Markdown reflows. A one-word edit renders as a\nwhole-paragraph replace under a\nline-oriented differ." + wide := "Markdown reflows. A one-word edit renders as a whole-paragraph replace\nunder a line-oriented differ." + assert.Equal(t, Normalize(narrow), Normalize(wide)) + assert.Equal(t, Normalize(narrow), Normalize(strings.ReplaceAll(narrow, "\n", " "))) +} + +func TestDiffWords(t *testing.T) { + tests := []struct { + name string + old, new string + want []Span + }{ + { + name: "reflow only is a single equal span", + old: "the quick brown fox\njumps over the lazy dog", + new: "the quick\nbrown fox jumps over\nthe lazy dog", + want: []Span{{Op: OpEqual, Text: "the quick brown fox jumps over the lazy dog"}}, + }, + { + name: "one word replaced", + old: "a quick brown fox", + new: "a quick red fox", + want: []Span{ + {Op: OpEqual, Text: "a quick"}, + {Op: OpDelete, Text: "brown", Space: true}, + // Space is false: the deletion it replaces already carried it. + {Op: OpInsert, Text: "red"}, + {Op: OpEqual, Text: "fox", Space: true}, + }, + }, + { + name: "word appended", + old: "one two", + new: "one two three", + want: []Span{ + {Op: OpEqual, Text: "one two"}, + {Op: OpInsert, Text: "three", Space: true}, + }, + }, + { + name: "punctuation alone", + old: "yes, always", + new: "yes; always", + want: []Span{ + {Op: OpEqual, Text: "yes"}, + {Op: OpDelete, Text: ","}, + {Op: OpInsert, Text: ";"}, + {Op: OpEqual, Text: "always", Space: true}, + }, + }, + { + name: "identical text", + old: "nothing changed here", + new: "nothing changed here", + want: []Span{{Op: OpEqual, Text: "nothing changed here"}}, + }, + { + name: "emptied", + old: "gone", + new: "", + want: []Span{{Op: OpDelete, Text: "gone"}}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, DiffWords(tc.old, tc.new)) + }) + } +} + +// TestDiffWordsAbsorbsShredding checks the cleanup pass: a rewrite that +// coincidentally keeps a comma must not be reported as three tiny edits. +func TestDiffWordsAbsorbsShredding(t *testing.T) { + spans := DiffWords( + "bots produce, humans curate", + "agents emit, operators approve", + ) + require.Len(t, spans, 2) + assert.Equal(t, OpDelete, spans[0].Op) + assert.Equal(t, "bots produce, humans curate", spans[0].Text) + assert.Equal(t, OpInsert, spans[1].Op) + assert.Equal(t, "agents emit, operators approve", spans[1].Text) +} + +func TestDiffLinesIsWhitespaceSensitive(t *testing.T) { + old := []string{"func main() {", "\tfmt.Println(1)", "}"} + nw := []string{"func main() {", " fmt.Println(1)", "}"} + spans := DiffLines(old, nw) + require.Len(t, spans, 4) + assert.Equal(t, OpEqual, spans[0].Op) + assert.Equal(t, OpDelete, spans[1].Op) + assert.Equal(t, OpInsert, spans[2].Op) + assert.Equal(t, OpEqual, spans[3].Op) +}