package prosediff import ( "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestTokenize(t *testing.T) { tests := []struct { name string in string want []string space []bool }{ {"plain words", "one two", []string{"one", "two"}, []bool{false, true}}, {"punctuation splits", "end.", []string{"end", "."}, []bool{false, false}}, {"apostrophe joins", "don't stop", []string{"don't", "stop"}, []bool{false, true}}, {"hyphen joins", "word-level diff", []string{"word-level", "diff"}, []bool{false, true}}, {"version numbers stay whole", "v5.19.1", []string{"v5.19.1"}, nil}, {"sentence dot still splits", "done. next", []string{"done", ".", "next"}, nil}, {"emphasis is one token", "**bold**", []string{"**", "bold", "**"}, nil}, {"em dash stands alone", "a — b", []string{"a", "—", "b"}, []bool{false, true, true}}, {"newline is just space", "a\nb", []string{"a", "b"}, []bool{false, true}}, {"cyrillic is a word", "спецификация готова", []string{"спецификация", "готова"}, nil}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { toks := Tokenize(tc.in) assert.Equal(t, tc.want, TokenTexts(toks)) if tc.space != nil { got := make([]bool, len(toks)) for i, tk := range toks { got[i] = tk.Space } assert.Equal(t, tc.space, got) } }) } } // TestNormalizeIgnoresWrapping is the load-bearing property of the whole // package: rewrapping prose must not change its normalized form. func TestNormalizeIgnoresWrapping(t *testing.T) { narrow := "Markdown reflows. A one-word edit renders as a\nwhole-paragraph replace under a\nline-oriented differ." wide := "Markdown reflows. A one-word edit renders as a whole-paragraph replace\nunder a line-oriented differ." assert.Equal(t, Normalize(narrow), Normalize(wide)) assert.Equal(t, Normalize(narrow), Normalize(strings.ReplaceAll(narrow, "\n", " "))) } func TestDiffWords(t *testing.T) { tests := []struct { name string old, new string want []Span }{ { name: "reflow only is a single equal span", old: "the quick brown fox\njumps over the lazy dog", new: "the quick\nbrown fox jumps over\nthe lazy dog", want: []Span{{Op: OpEqual, Text: "the quick brown fox jumps over the lazy dog"}}, }, { name: "one word replaced", old: "a quick brown fox", new: "a quick red fox", want: []Span{ {Op: OpEqual, Text: "a quick"}, {Op: OpDelete, Text: "brown", Space: true}, // Space is false: the deletion it replaces already carried it. {Op: OpInsert, Text: "red"}, {Op: OpEqual, Text: "fox", Space: true}, }, }, { name: "word appended", old: "one two", new: "one two three", want: []Span{ {Op: OpEqual, Text: "one two"}, {Op: OpInsert, Text: "three", Space: true}, }, }, { name: "punctuation alone", old: "yes, always", new: "yes; always", want: []Span{ {Op: OpEqual, Text: "yes"}, {Op: OpDelete, Text: ","}, {Op: OpInsert, Text: ";"}, {Op: OpEqual, Text: "always", Space: true}, }, }, { name: "identical text", old: "nothing changed here", new: "nothing changed here", want: []Span{{Op: OpEqual, Text: "nothing changed here"}}, }, { name: "emptied", old: "gone", new: "", want: []Span{{Op: OpDelete, Text: "gone"}}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { assert.Equal(t, tc.want, DiffWords(tc.old, tc.new)) }) } } // TestDiffWordsAbsorbsShredding checks the cleanup pass: a rewrite that // coincidentally keeps a comma must not be reported as three tiny edits. func TestDiffWordsAbsorbsShredding(t *testing.T) { spans := DiffWords( "bots produce, humans curate", "agents emit, operators approve", ) require.Len(t, spans, 2) assert.Equal(t, OpDelete, spans[0].Op) assert.Equal(t, "bots produce, humans curate", spans[0].Text) assert.Equal(t, OpInsert, spans[1].Op) assert.Equal(t, "agents emit, operators approve", spans[1].Text) } func TestDiffLinesIsWhitespaceSensitive(t *testing.T) { old := []string{"func main() {", "\tfmt.Println(1)", "}"} nw := []string{"func main() {", " fmt.Println(1)", "}"} spans := DiffLines(old, nw) require.Len(t, spans, 4) assert.Equal(t, OpEqual, spans[0].Op) assert.Equal(t, OpDelete, spans[1].Op) assert.Equal(t, OpInsert, spans[2].Op) assert.Equal(t, OpEqual, spans[3].Op) }