~bigbes/sr-ht-spec

ref: 3a56ee617a14a82d7d2d430173c84474444bc111 sr-ht-spec/prosediff/token_test.go -rw-r--r-- 5.9 KiB
3a56ee61 — Eugene Blikh deps: bump sr-ht-ecore for the slog panic reporter 10 days ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
package prosediff

import (
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestTokenize(t *testing.T) {
	tests := []struct {
		name  string
		in    string
		want  []string
		space []bool
	}{
		{"plain words", "one two", []string{"one", "two"}, []bool{false, true}},
		{"punctuation splits", "end.", []string{"end", "."}, []bool{false, false}},
		{"apostrophe joins", "don't stop", []string{"don't", "stop"}, []bool{false, true}},
		{"hyphen joins", "word-level diff", []string{"word-level", "diff"}, []bool{false, true}},
		{"version numbers stay whole", "v5.19.1", []string{"v5.19.1"}, nil},
		{"sentence dot still splits", "done. next", []string{"done", ".", "next"}, nil},
		{"emphasis is one token", "**bold**", []string{"**", "bold", "**"}, nil},
		{"em dash stands alone", "a — b", []string{"a", "—", "b"}, []bool{false, true, true}},
		{"newline is just space", "a\nb", []string{"a", "b"}, []bool{false, true}},
		{"cyrillic is a word", "спецификация готова", []string{"спецификация", "готова"}, nil},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			toks := Tokenize(tc.in)
			assert.Equal(t, tc.want, TokenTexts(toks))
			if tc.space != nil {
				got := make([]bool, len(toks))
				for i, tk := range toks {
					got[i] = tk.Space
				}
				assert.Equal(t, tc.space, got)
			}
		})
	}
}

// TestNormalizeIgnoresWrapping is the load-bearing property of the whole
// package: rewrapping prose must not change its normalized form.
func TestNormalizeIgnoresWrapping(t *testing.T) {
	narrow := "Markdown reflows. A one-word edit renders as a\nwhole-paragraph replace under a\nline-oriented differ."
	wide := "Markdown reflows. A one-word edit renders as a whole-paragraph replace\nunder a line-oriented differ."
	assert.Equal(t, Normalize(narrow), Normalize(wide))
	assert.Equal(t, Normalize(narrow), Normalize(strings.ReplaceAll(narrow, "\n", "   ")))
}

func TestDiffWords(t *testing.T) {
	tests := []struct {
		name     string
		old, new string
		want     []Span
	}{
		{
			name: "reflow only is a single equal span",
			old:  "the quick brown fox\njumps over the lazy dog",
			new:  "the quick\nbrown fox jumps over\nthe lazy dog",
			want: []Span{{Op: OpEqual, Text: "the quick brown fox jumps over the lazy dog"}},
		},
		{
			name: "one word replaced",
			old:  "a quick brown fox",
			new:  "a quick red fox",
			want: []Span{
				{Op: OpEqual, Text: "a quick"},
				{Op: OpDelete, Text: "brown", Space: true},
				// Space is false: the deletion it replaces already carried it.
				{Op: OpInsert, Text: "red"},
				{Op: OpEqual, Text: "fox", Space: true},
			},
		},
		{
			name: "word appended",
			old:  "one two",
			new:  "one two three",
			want: []Span{
				{Op: OpEqual, Text: "one two"},
				{Op: OpInsert, Text: "three", Space: true},
			},
		},
		{
			name: "punctuation alone",
			old:  "yes, always",
			new:  "yes; always",
			want: []Span{
				{Op: OpEqual, Text: "yes"},
				{Op: OpDelete, Text: ","},
				{Op: OpInsert, Text: ";"},
				{Op: OpEqual, Text: "always", Space: true},
			},
		},
		{
			name: "identical text",
			old:  "nothing changed here",
			new:  "nothing changed here",
			want: []Span{{Op: OpEqual, Text: "nothing changed here"}},
		},
		{
			name: "emptied",
			old:  "gone",
			new:  "",
			want: []Span{{Op: OpDelete, Text: "gone"}},
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			assert.Equal(t, tc.want, DiffWords(tc.old, tc.new))
		})
	}
}

// TestDiffWordsAbsorbsShredding checks the cleanup pass: a rewrite that
// coincidentally keeps a comma must not be reported as three tiny edits.
func TestDiffWordsAbsorbsShredding(t *testing.T) {
	spans := DiffWords(
		"bots produce, humans curate",
		"agents emit, operators approve",
	)
	require.Len(t, spans, 2)
	assert.Equal(t, OpDelete, spans[0].Op)
	assert.Equal(t, "bots produce, humans curate", spans[0].Text)
	assert.Equal(t, OpInsert, spans[1].Op)
	assert.Equal(t, "agents emit, operators approve", spans[1].Text)
}

// TestEqualRunAfterAHeadInsertionKeepsItsSeparator checks the one place the two
// sides of a diff disagree about whitespace. An equal run exists in both
// revisions at once; when words are inserted before it, the old side's first
// token has nothing in front of it and the new side's has the insertion. Reading
// only the old side dropped the separator, and a renderer joining the spans
// wrote "{+two words+}the rest" with no space — a defect the reader would read
// as the author's, since nothing in the output says a space went missing.
func TestEqualRunAfterAHeadInsertionKeepsItsSeparator(t *testing.T) {
	spans := DiffWords(
		"the storage layer keeps every revision",
		"in practice the storage layer keeps every revision",
	)
	require.Len(t, spans, 2)
	assert.Equal(t, OpInsert, spans[0].Op)
	assert.Equal(t, "in practice", spans[0].Text)
	assert.False(t, spans[0].Space, "nothing precedes the first span")
	assert.Equal(t, OpEqual, spans[1].Op)
	assert.True(t, spans[1].Space, "the inserted words are separated from the rest")
}

// TestSubstitutionStillCarriesNoDoubleSeparator guards the rule the fix above
// must not undo: an insertion directly replacing a deletion inherits the
// deletion's separator rather than announcing its own.
func TestSubstitutionStillCarriesNoDoubleSeparator(t *testing.T) {
	spans := DiffWords("the quick brown fox", "the quick red fox")
	require.Len(t, spans, 4)
	assert.Equal(t, OpDelete, spans[1].Op)
	assert.True(t, spans[1].Space)
	assert.Equal(t, OpInsert, spans[2].Op)
	assert.False(t, spans[2].Space)
}

func TestDiffLinesIsWhitespaceSensitive(t *testing.T) {
	old := []string{"func main() {", "\tfmt.Println(1)", "}"}
	nw := []string{"func main() {", "    fmt.Println(1)", "}"}
	spans := DiffLines(old, nw)
	require.Len(t, spans, 4)
	assert.Equal(t, OpEqual, spans[0].Op)
	assert.Equal(t, OpDelete, spans[1].Op)
	assert.Equal(t, OpInsert, spans[2].Op)
	assert.Equal(t, OpEqual, spans[3].Op)
}