package prosediff import ( "strings" "unicode" "unicode/utf8" ) // Token is one unit of the word-level diff. // // Whitespace is *not* a token. It survives only as Space, which records that // some run of whitespace separated this token from the previous one. That is // what makes reflow invisible: a paragraph rewrapped from 72 to 80 columns // produces byte-for-byte the same token stream, because "\n" and " " both // collapse to Space=true. type Token struct { Text string Space bool } // Tokenize splits prose into words and punctuation. // // A word is a run of letters/digits, optionally joined by an internal // apostrophe or hyphen ("don't", "word-level") or an internal dot between // digits ("v5.19.1"). Every other non-space rune forms a token from its own // repeated run, so "**" and "---" stay single tokens rather than exploding // into markup noise. func Tokenize(s string) []Token { var out []Token space := false for i := 0; i < len(s); { r, sz := utf8.DecodeRuneInString(s[i:]) switch { case unicode.IsSpace(r): space = true i += sz case isWordRune(r): j := i for j < len(s) { r2, sz2 := utf8.DecodeRuneInString(s[j:]) if isWordRune(r2) { j += sz2 continue } if isJoinRune(r2) && continuesWord(s, j+sz2, r2) { j += sz2 continue } break } out = append(out, Token{Text: s[i:j], Space: space}) space = false i = j default: j := i for j < len(s) { r2, sz2 := utf8.DecodeRuneInString(s[j:]) if r2 != r { break } j += sz2 } out = append(out, Token{Text: s[i:j], Space: space}) space = false i = j } } return out } func isWordRune(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' } func isJoinRune(r rune) bool { return r == '\'' || r == '-' || r == '.' || r == '’' } // continuesWord reports whether the join rune at the previous position is // followed by more word material (and, for a dot, by a digit — so "end." does // not swallow the sentence terminator). func continuesWord(s string, next int, join rune) bool { if next >= len(s) { return false } r, _ := utf8.DecodeRuneInString(s[next:]) if join == '.' { return unicode.IsDigit(r) } return isWordRune(r) } // TokenTexts drops the spacing, leaving the comparable content. func TokenTexts(toks []Token) []string { out := make([]string, len(toks)) for i, t := range toks { out[i] = t.Text } return out } // Normalize collapses a block's source text to its token stream joined by // single spaces. Two blocks with the same Normalize are the same prose, // however they were wrapped. func Normalize(s string) string { return strings.Join(TokenTexts(Tokenize(s)), " ") } // Span is a run of word-level tokens sharing one Op, ready to render. // // Space says whether to emit a separating space before this span when the // spans are rendered in order. An insertion that directly replaces a deletion // carries Space=false, because the deletion before it already carried the // separator; otherwise every one-word substitution would render as // "[-old-] {+new+}". type Span struct { Op Op Text string // tokens joined with single spaces Space bool } // smallEqualRun is the character budget below which an unchanged run wedged // between two changed runs is absorbed into the change. Without it, a rewrite // that happens to keep a comma or an "a" produces shredded output like // "[-x-]{+y+} , [-z-]{+w+}". const smallEqualRun = 4 // DiffWords produces the inline edit script between two pieces of prose. // Whitespace differences alone yield a single OpEqual span. func DiffWords(oldText, newText string) []Span { a := Tokenize(oldText) b := Tokenize(newText) in := newInterner() script := diffInts(in.all(TokenTexts(a)), in.all(TokenTexts(b))) script = absorbSmallEqualRuns(script, a) return spans(deletesFirst(script), a, b) } // absorbSmallEqualRuns rewrites tiny equal runs that sit between two changed // runs into delete+insert, so the surrounding change reads as one edit. func absorbSmallEqualRuns(script []edit, a []Token) []edit { if len(script) < 3 { return script } // Position of each run in a, needed to measure the equal run's length. posA := make([]int, len(script)) x := 0 for i, e := range script { posA[i] = x if e.op != OpInsert { x += e.n } } out := make([]edit, 0, len(script)+4) for i, e := range script { if e.op != OpEqual || i == 0 || i == len(script)-1 { out = append(out, e) continue } n := 0 for _, t := range a[posA[i] : posA[i]+e.n] { n += len(t.Text) } if n >= smallEqualRun { out = append(out, e) continue } out = append(out, edit{OpDelete, e.n}, edit{OpInsert, e.n}) } return coalesce(out) } // deletesFirst rewrites each changed region so all deletions precede all // insertions, whichever order Myers happened to emit them in. func deletesFirst(script []edit) []edit { out := make([]edit, 0, len(script)) for i := 0; i < len(script); { if script[i].op == OpEqual { out = append(out, script[i]) i++ continue } del, ins := 0, 0 for ; i < len(script) && script[i].op != OpEqual; i++ { if script[i].op == OpDelete { del += script[i].n } else { ins += script[i].n } } if del > 0 { out = append(out, edit{OpDelete, del}) } if ins > 0 { out = append(out, edit{OpInsert, ins}) } } return out } // spans walks an edit script and materializes it into renderable runs. func spans(script []edit, a, b []Token) []Span { var out []Span // space is passed in rather than read off toks[0] because an equal run // exists on both sides at once and the two sides can disagree about the // separator in front of it. A block that gains words at its head has // Space=false on the old side's first token — nothing precedes it there — // and Space=true on the new side's, where the inserted words do. Reading // only the old side emitted "{+two words+}the rest" with the words run // together; a renderer has no way to recover the separator, because the new // side's flag never reached it. emit := func(op Op, toks []Token, space bool) { if len(toks) == 0 { return } var sb strings.Builder for i, t := range toks { if i > 0 && t.Space { sb.WriteByte(' ') } sb.WriteString(t.Text) } out = append(out, Span{Op: op, Text: sb.String(), Space: space}) } i, j := 0, 0 for k := 0; k < len(script); k++ { e := script[k] // An insertion directly replacing a deletion must not re-announce // the whitespace the deletion already carried, or every one-word // substitution renders as "[-old-] {+new+}". if e.op == OpInsert && k > 0 && script[k-1].op == OpDelete { emit(OpInsert, b[j:j+e.n], false) j += e.n continue } switch e.op { case OpEqual: // Either side's separator is reason enough to emit one: the run is // rendered once, between whatever precedes it on the old side and // whatever precedes it on the new. emit(OpEqual, a[i:i+e.n], a[i].Space || b[j].Space) i += e.n j += e.n case OpDelete: emit(OpDelete, a[i:i+e.n], a[i].Space) i += e.n case OpInsert: emit(OpInsert, b[j:j+e.n], b[j].Space) j += e.n } } return out } // DiffLines is the code-fence path: line-oriented, whitespace-significant. func DiffLines(oldLines, newLines []string) []Span { in := newInterner() script := diffInts(in.all(oldLines), in.all(newLines)) var out []Span i, j := 0, 0 for _, e := range script { switch e.op { case OpEqual: for n := 0; n < e.n; n++ { out = append(out, Span{Op: OpEqual, Text: oldLines[i+n]}) } i += e.n j += e.n case OpDelete: for n := 0; n < e.n; n++ { out = append(out, Span{Op: OpDelete, Text: oldLines[i+n]}) } i += e.n case OpInsert: for n := 0; n < e.n; n++ { out = append(out, Span{Op: OpInsert, Text: newLines[j+n]}) } j += e.n } } return out }