package prosediff
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestTokenize(t *testing.T) {
tests := []struct {
name string
in string
want []string
space []bool
}{
{"plain words", "one two", []string{"one", "two"}, []bool{false, true}},
{"punctuation splits", "end.", []string{"end", "."}, []bool{false, false}},
{"apostrophe joins", "don't stop", []string{"don't", "stop"}, []bool{false, true}},
{"hyphen joins", "word-level diff", []string{"word-level", "diff"}, []bool{false, true}},
{"version numbers stay whole", "v5.19.1", []string{"v5.19.1"}, nil},
{"sentence dot still splits", "done. next", []string{"done", ".", "next"}, nil},
{"emphasis is one token", "**bold**", []string{"**", "bold", "**"}, nil},
{"em dash stands alone", "a — b", []string{"a", "—", "b"}, []bool{false, true, true}},
{"newline is just space", "a\nb", []string{"a", "b"}, []bool{false, true}},
{"cyrillic is a word", "спецификация готова", []string{"спецификация", "готова"}, nil},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
toks := Tokenize(tc.in)
assert.Equal(t, tc.want, TokenTexts(toks))
if tc.space != nil {
got := make([]bool, len(toks))
for i, tk := range toks {
got[i] = tk.Space
}
assert.Equal(t, tc.space, got)
}
})
}
}
// TestNormalizeIgnoresWrapping is the load-bearing property of the whole
// package: rewrapping prose must not change its normalized form.
func TestNormalizeIgnoresWrapping(t *testing.T) {
narrow := "Markdown reflows. A one-word edit renders as a\nwhole-paragraph replace under a\nline-oriented differ."
wide := "Markdown reflows. A one-word edit renders as a whole-paragraph replace\nunder a line-oriented differ."
assert.Equal(t, Normalize(narrow), Normalize(wide))
assert.Equal(t, Normalize(narrow), Normalize(strings.ReplaceAll(narrow, "\n", " ")))
}
func TestDiffWords(t *testing.T) {
tests := []struct {
name string
old, new string
want []Span
}{
{
name: "reflow only is a single equal span",
old: "the quick brown fox\njumps over the lazy dog",
new: "the quick\nbrown fox jumps over\nthe lazy dog",
want: []Span{{Op: OpEqual, Text: "the quick brown fox jumps over the lazy dog"}},
},
{
name: "one word replaced",
old: "a quick brown fox",
new: "a quick red fox",
want: []Span{
{Op: OpEqual, Text: "a quick"},
{Op: OpDelete, Text: "brown", Space: true},
// Space is false: the deletion it replaces already carried it.
{Op: OpInsert, Text: "red"},
{Op: OpEqual, Text: "fox", Space: true},
},
},
{
name: "word appended",
old: "one two",
new: "one two three",
want: []Span{
{Op: OpEqual, Text: "one two"},
{Op: OpInsert, Text: "three", Space: true},
},
},
{
name: "punctuation alone",
old: "yes, always",
new: "yes; always",
want: []Span{
{Op: OpEqual, Text: "yes"},
{Op: OpDelete, Text: ","},
{Op: OpInsert, Text: ";"},
{Op: OpEqual, Text: "always", Space: true},
},
},
{
name: "identical text",
old: "nothing changed here",
new: "nothing changed here",
want: []Span{{Op: OpEqual, Text: "nothing changed here"}},
},
{
name: "emptied",
old: "gone",
new: "",
want: []Span{{Op: OpDelete, Text: "gone"}},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
assert.Equal(t, tc.want, DiffWords(tc.old, tc.new))
})
}
}
// TestDiffWordsAbsorbsShredding checks the cleanup pass: a rewrite that
// coincidentally keeps a comma must not be reported as three tiny edits.
func TestDiffWordsAbsorbsShredding(t *testing.T) {
spans := DiffWords(
"bots produce, humans curate",
"agents emit, operators approve",
)
require.Len(t, spans, 2)
assert.Equal(t, OpDelete, spans[0].Op)
assert.Equal(t, "bots produce, humans curate", spans[0].Text)
assert.Equal(t, OpInsert, spans[1].Op)
assert.Equal(t, "agents emit, operators approve", spans[1].Text)
}
// TestEqualRunAfterAHeadInsertionKeepsItsSeparator checks the one place the two
// sides of a diff disagree about whitespace. An equal run exists in both
// revisions at once; when words are inserted before it, the old side's first
// token has nothing in front of it and the new side's has the insertion. Reading
// only the old side dropped the separator, and a renderer joining the spans
// wrote "{+two words+}the rest" with no space — a defect the reader would read
// as the author's, since nothing in the output says a space went missing.
func TestEqualRunAfterAHeadInsertionKeepsItsSeparator(t *testing.T) {
spans := DiffWords(
"the storage layer keeps every revision",
"in practice the storage layer keeps every revision",
)
require.Len(t, spans, 2)
assert.Equal(t, OpInsert, spans[0].Op)
assert.Equal(t, "in practice", spans[0].Text)
assert.False(t, spans[0].Space, "nothing precedes the first span")
assert.Equal(t, OpEqual, spans[1].Op)
assert.True(t, spans[1].Space, "the inserted words are separated from the rest")
}
// TestSubstitutionStillCarriesNoDoubleSeparator guards the rule the fix above
// must not undo: an insertion directly replacing a deletion inherits the
// deletion's separator rather than announcing its own.
func TestSubstitutionStillCarriesNoDoubleSeparator(t *testing.T) {
spans := DiffWords("the quick brown fox", "the quick red fox")
require.Len(t, spans, 4)
assert.Equal(t, OpDelete, spans[1].Op)
assert.True(t, spans[1].Space)
assert.Equal(t, OpInsert, spans[2].Op)
assert.False(t, spans[2].Space)
}
func TestDiffLinesIsWhitespaceSensitive(t *testing.T) {
old := []string{"func main() {", "\tfmt.Println(1)", "}"}
nw := []string{"func main() {", " fmt.Println(1)", "}"}
spans := DiffLines(old, nw)
require.Len(t, spans, 4)
assert.Equal(t, OpEqual, spans[0].Op)
assert.Equal(t, OpDelete, spans[1].Op)
assert.Equal(t, OpInsert, spans[2].Op)
assert.Equal(t, OpEqual, spans[3].Op)
}