package prosediff
import (
"strings"
"unicode"
"unicode/utf8"
)
// Token is one unit of the word-level diff.
//
// Whitespace is *not* a token. It survives only as Space, which records that
// some run of whitespace separated this token from the previous one. That is
// what makes reflow invisible: a paragraph rewrapped from 72 to 80 columns
// produces byte-for-byte the same token stream, because "\n" and " " both
// collapse to Space=true.
type Token struct {
Text string
Space bool
}
// Tokenize splits prose into words and punctuation.
//
// A word is a run of letters/digits, optionally joined by an internal
// apostrophe or hyphen ("don't", "word-level") or an internal dot between
// digits ("v5.19.1"). Every other non-space rune forms a token from its own
// repeated run, so "**" and "---" stay single tokens rather than exploding
// into markup noise.
func Tokenize(s string) []Token {
var out []Token
space := false
for i := 0; i < len(s); {
r, sz := utf8.DecodeRuneInString(s[i:])
switch {
case unicode.IsSpace(r):
space = true
i += sz
case isWordRune(r):
j := i
for j < len(s) {
r2, sz2 := utf8.DecodeRuneInString(s[j:])
if isWordRune(r2) {
j += sz2
continue
}
if isJoinRune(r2) && continuesWord(s, j+sz2, r2) {
j += sz2
continue
}
break
}
out = append(out, Token{Text: s[i:j], Space: space})
space = false
i = j
default:
j := i
for j < len(s) {
r2, sz2 := utf8.DecodeRuneInString(s[j:])
if r2 != r {
break
}
j += sz2
}
out = append(out, Token{Text: s[i:j], Space: space})
space = false
i = j
}
}
return out
}
func isWordRune(r rune) bool {
return unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_'
}
func isJoinRune(r rune) bool { return r == '\'' || r == '-' || r == '.' || r == '’' }
// continuesWord reports whether the join rune at the previous position is
// followed by more word material (and, for a dot, by a digit — so "end." does
// not swallow the sentence terminator).
func continuesWord(s string, next int, join rune) bool {
if next >= len(s) {
return false
}
r, _ := utf8.DecodeRuneInString(s[next:])
if join == '.' {
return unicode.IsDigit(r)
}
return isWordRune(r)
}
// TokenTexts drops the spacing, leaving the comparable content.
func TokenTexts(toks []Token) []string {
out := make([]string, len(toks))
for i, t := range toks {
out[i] = t.Text
}
return out
}
// Normalize collapses a block's source text to its token stream joined by
// single spaces. Two blocks with the same Normalize are the same prose,
// however they were wrapped.
func Normalize(s string) string {
return strings.Join(TokenTexts(Tokenize(s)), " ")
}
// Span is a run of word-level tokens sharing one Op, ready to render.
//
// Space says whether to emit a separating space before this span when the
// spans are rendered in order. An insertion that directly replaces a deletion
// carries Space=false, because the deletion before it already carried the
// separator; otherwise every one-word substitution would render as
// "[-old-] {+new+}".
type Span struct {
Op Op
Text string // tokens joined with single spaces
Space bool
}
// smallEqualRun is the character budget below which an unchanged run wedged
// between two changed runs is absorbed into the change. Without it, a rewrite
// that happens to keep a comma or an "a" produces shredded output like
// "[-x-]{+y+} , [-z-]{+w+}".
const smallEqualRun = 4
// DiffWords produces the inline edit script between two pieces of prose.
// Whitespace differences alone yield a single OpEqual span.
func DiffWords(oldText, newText string) []Span {
a := Tokenize(oldText)
b := Tokenize(newText)
in := newInterner()
script := diffInts(in.all(TokenTexts(a)), in.all(TokenTexts(b)))
script = absorbSmallEqualRuns(script, a)
return spans(deletesFirst(script), a, b)
}
// absorbSmallEqualRuns rewrites tiny equal runs that sit between two changed
// runs into delete+insert, so the surrounding change reads as one edit.
func absorbSmallEqualRuns(script []edit, a []Token) []edit {
if len(script) < 3 {
return script
}
// Position of each run in a, needed to measure the equal run's length.
posA := make([]int, len(script))
x := 0
for i, e := range script {
posA[i] = x
if e.op != OpInsert {
x += e.n
}
}
out := make([]edit, 0, len(script)+4)
for i, e := range script {
if e.op != OpEqual || i == 0 || i == len(script)-1 {
out = append(out, e)
continue
}
n := 0
for _, t := range a[posA[i] : posA[i]+e.n] {
n += len(t.Text)
}
if n >= smallEqualRun {
out = append(out, e)
continue
}
out = append(out, edit{OpDelete, e.n}, edit{OpInsert, e.n})
}
return coalesce(out)
}
// deletesFirst rewrites each changed region so all deletions precede all
// insertions, whichever order Myers happened to emit them in.
func deletesFirst(script []edit) []edit {
out := make([]edit, 0, len(script))
for i := 0; i < len(script); {
if script[i].op == OpEqual {
out = append(out, script[i])
i++
continue
}
del, ins := 0, 0
for ; i < len(script) && script[i].op != OpEqual; i++ {
if script[i].op == OpDelete {
del += script[i].n
} else {
ins += script[i].n
}
}
if del > 0 {
out = append(out, edit{OpDelete, del})
}
if ins > 0 {
out = append(out, edit{OpInsert, ins})
}
}
return out
}
// spans walks an edit script and materializes it into renderable runs.
func spans(script []edit, a, b []Token) []Span {
var out []Span
// space is passed in rather than read off toks[0] because an equal run
// exists on both sides at once and the two sides can disagree about the
// separator in front of it. A block that gains words at its head has
// Space=false on the old side's first token — nothing precedes it there —
// and Space=true on the new side's, where the inserted words do. Reading
// only the old side emitted "{+two words+}the rest" with the words run
// together; a renderer has no way to recover the separator, because the new
// side's flag never reached it.
emit := func(op Op, toks []Token, space bool) {
if len(toks) == 0 {
return
}
var sb strings.Builder
for i, t := range toks {
if i > 0 && t.Space {
sb.WriteByte(' ')
}
sb.WriteString(t.Text)
}
out = append(out, Span{Op: op, Text: sb.String(), Space: space})
}
i, j := 0, 0
for k := 0; k < len(script); k++ {
e := script[k]
// An insertion directly replacing a deletion must not re-announce
// the whitespace the deletion already carried, or every one-word
// substitution renders as "[-old-] {+new+}".
if e.op == OpInsert && k > 0 && script[k-1].op == OpDelete {
emit(OpInsert, b[j:j+e.n], false)
j += e.n
continue
}
switch e.op {
case OpEqual:
// Either side's separator is reason enough to emit one: the run is
// rendered once, between whatever precedes it on the old side and
// whatever precedes it on the new.
emit(OpEqual, a[i:i+e.n], a[i].Space || b[j].Space)
i += e.n
j += e.n
case OpDelete:
emit(OpDelete, a[i:i+e.n], a[i].Space)
i += e.n
case OpInsert:
emit(OpInsert, b[j:j+e.n], b[j].Space)
j += e.n
}
}
return out
}
// DiffLines is the code-fence path: line-oriented, whitespace-significant.
func DiffLines(oldLines, newLines []string) []Span {
in := newInterner()
script := diffInts(in.all(oldLines), in.all(newLines))
var out []Span
i, j := 0, 0
for _, e := range script {
switch e.op {
case OpEqual:
for n := 0; n < e.n; n++ {
out = append(out, Span{Op: OpEqual, Text: oldLines[i+n]})
}
i += e.n
j += e.n
case OpDelete:
for n := 0; n < e.n; n++ {
out = append(out, Span{Op: OpDelete, Text: oldLines[i+n]})
}
i += e.n
case OpInsert:
for n := 0; n < e.n; n++ {
out = append(out, Span{Op: OpInsert, Text: newLines[j+n]})
}
j += e.n
}
}
return out
}