~bigbes/sr-ht-spec

ref: 2dc6b71739a6101f0c113df5ae27d2007b52dbd9 sr-ht-spec/prosediff/token_test.go -rw-r--r-- 4.4 KiB
2dc6b717 — Eugene Blikh chore(beads): enable Dolt auto-push to dolt.srht.bigb.es 24 days ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
package prosediff

import (
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestTokenize(t *testing.T) {
	tests := []struct {
		name  string
		in    string
		want  []string
		space []bool
	}{
		{"plain words", "one two", []string{"one", "two"}, []bool{false, true}},
		{"punctuation splits", "end.", []string{"end", "."}, []bool{false, false}},
		{"apostrophe joins", "don't stop", []string{"don't", "stop"}, []bool{false, true}},
		{"hyphen joins", "word-level diff", []string{"word-level", "diff"}, []bool{false, true}},
		{"version numbers stay whole", "v5.19.1", []string{"v5.19.1"}, nil},
		{"sentence dot still splits", "done. next", []string{"done", ".", "next"}, nil},
		{"emphasis is one token", "**bold**", []string{"**", "bold", "**"}, nil},
		{"em dash stands alone", "a — b", []string{"a", "—", "b"}, []bool{false, true, true}},
		{"newline is just space", "a\nb", []string{"a", "b"}, []bool{false, true}},
		{"cyrillic is a word", "спецификация готова", []string{"спецификация", "готова"}, nil},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			toks := Tokenize(tc.in)
			assert.Equal(t, tc.want, TokenTexts(toks))
			if tc.space != nil {
				got := make([]bool, len(toks))
				for i, tk := range toks {
					got[i] = tk.Space
				}
				assert.Equal(t, tc.space, got)
			}
		})
	}
}

// TestNormalizeIgnoresWrapping is the load-bearing property of the whole
// package: rewrapping prose must not change its normalized form.
func TestNormalizeIgnoresWrapping(t *testing.T) {
	narrow := "Markdown reflows. A one-word edit renders as a\nwhole-paragraph replace under a\nline-oriented differ."
	wide := "Markdown reflows. A one-word edit renders as a whole-paragraph replace\nunder a line-oriented differ."
	assert.Equal(t, Normalize(narrow), Normalize(wide))
	assert.Equal(t, Normalize(narrow), Normalize(strings.ReplaceAll(narrow, "\n", "   ")))
}

func TestDiffWords(t *testing.T) {
	tests := []struct {
		name     string
		old, new string
		want     []Span
	}{
		{
			name: "reflow only is a single equal span",
			old:  "the quick brown fox\njumps over the lazy dog",
			new:  "the quick\nbrown fox jumps over\nthe lazy dog",
			want: []Span{{Op: OpEqual, Text: "the quick brown fox jumps over the lazy dog"}},
		},
		{
			name: "one word replaced",
			old:  "a quick brown fox",
			new:  "a quick red fox",
			want: []Span{
				{Op: OpEqual, Text: "a quick"},
				{Op: OpDelete, Text: "brown", Space: true},
				// Space is false: the deletion it replaces already carried it.
				{Op: OpInsert, Text: "red"},
				{Op: OpEqual, Text: "fox", Space: true},
			},
		},
		{
			name: "word appended",
			old:  "one two",
			new:  "one two three",
			want: []Span{
				{Op: OpEqual, Text: "one two"},
				{Op: OpInsert, Text: "three", Space: true},
			},
		},
		{
			name: "punctuation alone",
			old:  "yes, always",
			new:  "yes; always",
			want: []Span{
				{Op: OpEqual, Text: "yes"},
				{Op: OpDelete, Text: ","},
				{Op: OpInsert, Text: ";"},
				{Op: OpEqual, Text: "always", Space: true},
			},
		},
		{
			name: "identical text",
			old:  "nothing changed here",
			new:  "nothing changed here",
			want: []Span{{Op: OpEqual, Text: "nothing changed here"}},
		},
		{
			name: "emptied",
			old:  "gone",
			new:  "",
			want: []Span{{Op: OpDelete, Text: "gone"}},
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			assert.Equal(t, tc.want, DiffWords(tc.old, tc.new))
		})
	}
}

// TestDiffWordsAbsorbsShredding checks the cleanup pass: a rewrite that
// coincidentally keeps a comma must not be reported as three tiny edits.
func TestDiffWordsAbsorbsShredding(t *testing.T) {
	spans := DiffWords(
		"bots produce, humans curate",
		"agents emit, operators approve",
	)
	require.Len(t, spans, 2)
	assert.Equal(t, OpDelete, spans[0].Op)
	assert.Equal(t, "bots produce, humans curate", spans[0].Text)
	assert.Equal(t, OpInsert, spans[1].Op)
	assert.Equal(t, "agents emit, operators approve", spans[1].Text)
}

func TestDiffLinesIsWhitespaceSensitive(t *testing.T) {
	old := []string{"func main() {", "\tfmt.Println(1)", "}"}
	nw := []string{"func main() {", "    fmt.Println(1)", "}"}
	spans := DiffLines(old, nw)
	require.Len(t, spans, 4)
	assert.Equal(t, OpEqual, spans[0].Op)
	assert.Equal(t, OpDelete, spans[1].Op)
	assert.Equal(t, OpInsert, spans[2].Op)
	assert.Equal(t, OpEqual, spans[3].Op)
}