~bigbes/sr-ht-spec

ref: 9b5827b659a4c0524e7c213f5c0834821d35e783 sr-ht-spec/prosediff/token.go -rw-r--r-- 7.7 KiB
9b5827b6 — Eugene Blikh bearer: draw the 401 arm from IsAuthFailure so the two cannot drift 9 days ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
package prosediff

import (
	"strings"
	"unicode"
	"unicode/utf8"
)

// Token is one unit of the word-level diff.
//
// Whitespace is *not* a token. It survives only as Space, which records that
// some run of whitespace separated this token from the previous one. That is
// what makes reflow invisible: a paragraph rewrapped from 72 to 80 columns
// produces byte-for-byte the same token stream, because "\n" and " " both
// collapse to Space=true.
type Token struct {
	Text  string
	Space bool
}

// Tokenize splits prose into words and punctuation.
//
// A word is a run of letters/digits, optionally joined by an internal
// apostrophe or hyphen ("don't", "word-level") or an internal dot between
// digits ("v5.19.1"). Every other non-space rune forms a token from its own
// repeated run, so "**" and "---" stay single tokens rather than exploding
// into markup noise.
func Tokenize(s string) []Token {
	var out []Token
	space := false
	for i := 0; i < len(s); {
		r, sz := utf8.DecodeRuneInString(s[i:])
		switch {
		case unicode.IsSpace(r):
			space = true
			i += sz
		case isWordRune(r):
			j := i
			for j < len(s) {
				r2, sz2 := utf8.DecodeRuneInString(s[j:])
				if isWordRune(r2) {
					j += sz2
					continue
				}
				if isJoinRune(r2) && continuesWord(s, j+sz2, r2) {
					j += sz2
					continue
				}
				break
			}
			out = append(out, Token{Text: s[i:j], Space: space})
			space = false
			i = j
		default:
			j := i
			for j < len(s) {
				r2, sz2 := utf8.DecodeRuneInString(s[j:])
				if r2 != r {
					break
				}
				j += sz2
			}
			out = append(out, Token{Text: s[i:j], Space: space})
			space = false
			i = j
		}
	}
	return out
}

func isWordRune(r rune) bool {
	return unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_'
}

func isJoinRune(r rune) bool { return r == '\'' || r == '-' || r == '.' || r == '’' }

// continuesWord reports whether the join rune at the previous position is
// followed by more word material (and, for a dot, by a digit — so "end." does
// not swallow the sentence terminator).
func continuesWord(s string, next int, join rune) bool {
	if next >= len(s) {
		return false
	}
	r, _ := utf8.DecodeRuneInString(s[next:])
	if join == '.' {
		return unicode.IsDigit(r)
	}
	return isWordRune(r)
}

// TokenTexts drops the spacing, leaving the comparable content.
func TokenTexts(toks []Token) []string {
	out := make([]string, len(toks))
	for i, t := range toks {
		out[i] = t.Text
	}
	return out
}

// Normalize collapses a block's source text to its token stream joined by
// single spaces. Two blocks with the same Normalize are the same prose,
// however they were wrapped.
func Normalize(s string) string {
	return strings.Join(TokenTexts(Tokenize(s)), " ")
}

// Span is a run of word-level tokens sharing one Op, ready to render.
//
// Space says whether to emit a separating space before this span when the
// spans are rendered in order. An insertion that directly replaces a deletion
// carries Space=false, because the deletion before it already carried the
// separator; otherwise every one-word substitution would render as
// "[-old-] {+new+}".
type Span struct {
	Op    Op
	Text  string // tokens joined with single spaces
	Space bool
}

// smallEqualRun is the character budget below which an unchanged run wedged
// between two changed runs is absorbed into the change. Without it, a rewrite
// that happens to keep a comma or an "a" produces shredded output like
// "[-x-]{+y+} , [-z-]{+w+}".
const smallEqualRun = 4

// DiffWords produces the inline edit script between two pieces of prose.
// Whitespace differences alone yield a single OpEqual span.
func DiffWords(oldText, newText string) []Span {
	a := Tokenize(oldText)
	b := Tokenize(newText)
	in := newInterner()
	script := diffInts(in.all(TokenTexts(a)), in.all(TokenTexts(b)))
	script = absorbSmallEqualRuns(script, a)
	return spans(deletesFirst(script), a, b)
}

// absorbSmallEqualRuns rewrites tiny equal runs that sit between two changed
// runs into delete+insert, so the surrounding change reads as one edit.
func absorbSmallEqualRuns(script []edit, a []Token) []edit {
	if len(script) < 3 {
		return script
	}
	// Position of each run in a, needed to measure the equal run's length.
	posA := make([]int, len(script))
	x := 0
	for i, e := range script {
		posA[i] = x
		if e.op != OpInsert {
			x += e.n
		}
	}
	out := make([]edit, 0, len(script)+4)
	for i, e := range script {
		if e.op != OpEqual || i == 0 || i == len(script)-1 {
			out = append(out, e)
			continue
		}
		n := 0
		for _, t := range a[posA[i] : posA[i]+e.n] {
			n += len(t.Text)
		}
		if n >= smallEqualRun {
			out = append(out, e)
			continue
		}
		out = append(out, edit{OpDelete, e.n}, edit{OpInsert, e.n})
	}
	return coalesce(out)
}

// deletesFirst rewrites each changed region so all deletions precede all
// insertions, whichever order Myers happened to emit them in.
func deletesFirst(script []edit) []edit {
	out := make([]edit, 0, len(script))
	for i := 0; i < len(script); {
		if script[i].op == OpEqual {
			out = append(out, script[i])
			i++
			continue
		}
		del, ins := 0, 0
		for ; i < len(script) && script[i].op != OpEqual; i++ {
			if script[i].op == OpDelete {
				del += script[i].n
			} else {
				ins += script[i].n
			}
		}
		if del > 0 {
			out = append(out, edit{OpDelete, del})
		}
		if ins > 0 {
			out = append(out, edit{OpInsert, ins})
		}
	}
	return out
}

// spans walks an edit script and materializes it into renderable runs.
func spans(script []edit, a, b []Token) []Span {
	var out []Span
	// space is passed in rather than read off toks[0] because an equal run
	// exists on both sides at once and the two sides can disagree about the
	// separator in front of it. A block that gains words at its head has
	// Space=false on the old side's first token — nothing precedes it there —
	// and Space=true on the new side's, where the inserted words do. Reading
	// only the old side emitted "{+two words+}the rest" with the words run
	// together; a renderer has no way to recover the separator, because the new
	// side's flag never reached it.
	emit := func(op Op, toks []Token, space bool) {
		if len(toks) == 0 {
			return
		}
		var sb strings.Builder
		for i, t := range toks {
			if i > 0 && t.Space {
				sb.WriteByte(' ')
			}
			sb.WriteString(t.Text)
		}
		out = append(out, Span{Op: op, Text: sb.String(), Space: space})
	}

	i, j := 0, 0
	for k := 0; k < len(script); k++ {
		e := script[k]
		// An insertion directly replacing a deletion must not re-announce
		// the whitespace the deletion already carried, or every one-word
		// substitution renders as "[-old-] {+new+}".
		if e.op == OpInsert && k > 0 && script[k-1].op == OpDelete {
			emit(OpInsert, b[j:j+e.n], false)
			j += e.n
			continue
		}
		switch e.op {
		case OpEqual:
			// Either side's separator is reason enough to emit one: the run is
			// rendered once, between whatever precedes it on the old side and
			// whatever precedes it on the new.
			emit(OpEqual, a[i:i+e.n], a[i].Space || b[j].Space)
			i += e.n
			j += e.n
		case OpDelete:
			emit(OpDelete, a[i:i+e.n], a[i].Space)
			i += e.n
		case OpInsert:
			emit(OpInsert, b[j:j+e.n], b[j].Space)
			j += e.n
		}
	}
	return out
}

// DiffLines is the code-fence path: line-oriented, whitespace-significant.
func DiffLines(oldLines, newLines []string) []Span {
	in := newInterner()
	script := diffInts(in.all(oldLines), in.all(newLines))
	var out []Span
	i, j := 0, 0
	for _, e := range script {
		switch e.op {
		case OpEqual:
			for n := 0; n < e.n; n++ {
				out = append(out, Span{Op: OpEqual, Text: oldLines[i+n]})
			}
			i += e.n
			j += e.n
		case OpDelete:
			for n := 0; n < e.n; n++ {
				out = append(out, Span{Op: OpDelete, Text: oldLines[i+n]})
			}
			i += e.n
		case OpInsert:
			for n := 0; n < e.n; n++ {
				out = append(out, Span{Op: OpInsert, Text: newLines[j+n]})
			}
			j += e.n
		}
	}
	return out
}