~bigbes/sr-ht-spec

ref: 394285f4cb998359c0a127454de4d32284a3ef22 sr-ht-spec/prosediff/align.go -rw-r--r-- 11.0 KiB
394285f4 — Eugene Blikh chore(beads): file spec-rsb and spec-ovo, the two admin commands 13 days ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
package prosediff

import "sort"

// Tuning constants for block alignment. They are deliberately package-level
// and documented rather than hidden in the code, because they are the knobs
// that decide whether a review page reads well.
const (
	// modifyThreshold is the token similarity two blocks need before they
	// are called "the same block, edited" rather than a delete plus an
	// insert. Below it, presenting a word diff would be noise.
	modifyThreshold = 0.40

	// shortBlockTokens is the size below which a block is considered too
	// small to judge by similarity alone.
	shortBlockTokens = 6

	// shortBlockThreshold applies instead of modifyThreshold when either
	// block is short: pairing "yes" with "no" on a 0.4 score helps nobody.
	shortBlockThreshold = 0.60

	// moveMinTokens is the smallest block that may be reported as moved.
	// Below it, identical content is far more likely to be coincidence
	// (a repeated "```", a "- see above") than an actual move.
	moveMinTokens = 5

	// pairBudget caps the similarity matrix inside one changed region. Past
	// it, only pairs within pairWindow of each other are considered; a
	// region with hundreds of blocks on both sides has no readable pairing
	// anyway.
	pairBudget = 4096
	pairWindow = 8
)

// region is one changed stretch of the alignment: blocks deleted from the old
// revision and blocks inserted into the new one, adjacent in the edit script.
type region struct {
	del []int // indices into old
	ins []int // indices into nw
}

// align turns two block sequences into the diff's change list.
//
// Three passes, in this order:
//  1. Myers over block hashes finds the unchanged skeleton.
//  2. Blocks left over from pass 1 that reappear verbatim elsewhere are
//     moves, grown outwards over their neighbours.
//  3. Similar-enough leftovers inside one changed region are modifications.
//
// Anything still unmatched is a plain insert or delete.
func align(old, nw []Block) []BlockChange {
	in := newInterner()
	oldIDs := make([]int, len(old))
	for i, b := range old {
		oldIDs[i] = in.id(b.Hash)
	}
	newIDs := make([]int, len(nw))
	for i, b := range nw {
		newIDs[i] = in.id(b.Hash)
	}
	script := diffInts(oldIDs, newIDs)

	var (
		out     []BlockChange
		regions []region
		// slots[i] is where region i's entries go in the output.
		slots []int
	)

	i, j := 0, 0
	for k := 0; k < len(script); {
		if script[k].op == OpEqual {
			for n := 0; n < script[k].n; n++ {
				o, w := old[i+n], nw[j+n]
				out = append(out, BlockChange{Kind: ChangeEqual, Old: &o, New: &w})
			}
			i += script[k].n
			j += script[k].n
			k++
			continue
		}
		var r region
		for ; k < len(script) && script[k].op != OpEqual; k++ {
			if script[k].op == OpDelete {
				for n := 0; n < script[k].n; n++ {
					r.del = append(r.del, i+n)
				}
				i += script[k].n
			} else {
				for n := 0; n < script[k].n; n++ {
					r.ins = append(r.ins, j+n)
				}
				j += script[k].n
			}
		}
		regions = append(regions, r)
		slots = append(slots, len(out))
		out = append(out, BlockChange{}) // placeholder, expanded below
	}

	moveOf, editedInMove := detectMoves(old, nw, regions)

	// Build each region's entries, then splice them in at their slot.
	expanded := make([][]BlockChange, len(regions))
	for ri, r := range regions {
		expanded[ri] = expandRegion(old, nw, r.del, r.ins, moveOf, editedInMove)
	}
	final := make([]BlockChange, 0, len(out)+len(regions))
	next := 0
	for idx := 0; idx < len(out); idx++ {
		if next < len(slots) && slots[next] == idx {
			final = append(final, expanded[next]...)
			next++
			continue
		}
		final = append(final, out[idx])
	}
	return final
}

// movePair records that old block o and new block n are the same content in a
// different place.
type movePair struct {
	oldIdx int
	newIdx int
}

// detectMoves matches leftover blocks across regions, seeding on exact hash
// equality and then growing each seed over its neighbours.
//
// Deliberate limitation: every move must be anchored by at least one block
// whose content is byte-identical after normalization. A block that moved and
// was edited is recognised only when it sits *between* two such anchors; a
// section that moved and was rewritten throughout falls through as a delete
// plus an insert. Matching moves by similarity alone would claim
// relationships between blocks that merely share boilerplate, and a wrong
// "moved from" costs a reviewer more than an honest add+remove.
func detectMoves(old, nw []Block, regions []region) (moves, edited map[int]movePair) {
	freeOld := map[int]bool{}
	freeNew := map[int]bool{}
	byHash := map[string][]int{}
	for _, r := range regions {
		for _, oi := range r.del {
			freeOld[oi] = true
			if len(Tokenize(old[oi].Text)) >= moveMinTokens {
				byHash[old[oi].Hash] = append(byHash[old[oi].Hash], oi)
			}
		}
		for _, ni := range r.ins {
			freeNew[ni] = true
		}
	}

	out := map[int]movePair{}
	edited = map[int]movePair{}
	pair := func(oi, ni int) {
		p := movePair{oldIdx: oi, newIdx: ni}
		out[oldKey(oi)] = p
		out[newKey(ni)] = p
		delete(freeOld, oi)
		delete(freeNew, ni)
	}

	// Seed: blocks big enough that identical content cannot be coincidence.
	var seeds []movePair
	for _, r := range regions {
		for _, ni := range r.ins {
			if len(Tokenize(nw[ni].Text)) < moveMinTokens {
				continue
			}
			for _, oi := range byHash[nw[ni].Hash] {
				if !freeOld[oi] || !freeNew[ni] {
					continue
				}
				pair(oi, ni)
				seeds = append(seeds, movePair{oi, ni})
				break
			}
		}
	}

	// Grow each seed outwards while the neighbouring blocks are also
	// unmatched and identical. This is what keeps a moved section's heading
	// and its short trailing blocks attached to the move, instead of
	// stranding them as a delete plus an insert either side of it.
	//
	// Growth also bridges a single edited block, but only when the block
	// *past* it matches exactly — "a section was moved and one paragraph in
	// it was touched" is common, while a lone similar block at the edge of a
	// move is just as likely to be coincidence.
	for _, s := range seeds {
		for step := -1; step <= 1; step += 2 {
			oi, ni := s.oldIdx+step, s.newIdx+step
			for freeOld[oi] && freeNew[ni] {
				if old[oi].Hash == nw[ni].Hash {
					pair(oi, ni)
					oi += step
					ni += step
					continue
				}
				if !bridgeable(old, nw, oi, ni, step, freeOld, freeNew, out) {
					break
				}
				pair(oi, ni)
				edited[oldKey(oi)] = movePair{oi, ni}
				edited[newKey(ni)] = movePair{oi, ni}
				oi += step
				ni += step
			}
		}
	}
	return out, edited
}

// bridgeable reports whether old[oi] and nw[ni] are an edited version of one
// another sitting inside a run of moved blocks. The anchor past the gap may
// be either still unclaimed or already paired to its counterpart by an
// earlier seed — both mean "the move continues on the far side".
func bridgeable(old, nw []Block, oi, ni, step int, freeOld, freeNew map[int]bool, moves map[int]movePair) bool {
	if old[oi].Kind != nw[ni].Kind {
		return false
	}
	no, nn := oi+step, ni+step
	if no < 0 || no >= len(old) || nn < 0 || nn >= len(nw) {
		return false
	}
	if old[no].Hash != nw[nn].Hash {
		return false
	}
	anchored := freeOld[no] && freeNew[nn]
	if p, ok := moves[oldKey(no)]; ok && p.newIdx == nn {
		anchored = true
	}
	if !anchored {
		return false
	}
	return blockSimilarity(old[oi], nw[ni]) >= thresholdFor(old[oi], nw[ni])
}

// Move bookkeeping keys old and new indices into one map without colliding.
func oldKey(i int) int { return i * 2 }
func newKey(i int) int { return i*2 + 1 }

func expandRegion(old, nw []Block, del, ins []int, moveOf, editedInMove map[int]movePair) []BlockChange {
	pairs := pairModified(old, nw, del, ins, moveOf)
	pairedOld := map[int]int{} // old index -> new index
	pairedNew := map[int]bool{}
	for _, p := range pairs {
		pairedOld[p.oldIdx] = p.newIdx
		pairedNew[p.newIdx] = true
	}

	var out []BlockChange
	for _, oi := range del {
		o := old[oi]
		if mp, ok := moveOf[oldKey(oi)]; ok {
			// A block that moved and was edited is announced here and
			// shown in full at its new position, where the reviewer
			// reads the section it now belongs to.
			n := nw[mp.newIdx]
			out = append(out, BlockChange{Kind: ChangeMoveOut, Old: &o, New: &n})
			continue
		}
		if ni, ok := pairedOld[oi]; ok {
			n := nw[ni]
			out = append(out, modifyChange(o, n))
			continue
		}
		out = append(out, BlockChange{Kind: ChangeDelete, Old: &o})
	}
	for _, ni := range ins {
		n := nw[ni]
		if mp, ok := editedInMove[newKey(ni)]; ok {
			c := modifyChange(old[mp.oldIdx], n)
			c.Moved = true
			out = append(out, c)
			continue
		}
		if mp, ok := moveOf[newKey(ni)]; ok {
			o := old[mp.oldIdx]
			out = append(out, BlockChange{Kind: ChangeMoveIn, Old: &o, New: &n})
			continue
		}
		if pairedNew[ni] {
			continue // already emitted as a modification
		}
		out = append(out, BlockChange{Kind: ChangeInsert, New: &n})
	}
	return out
}

func modifyChange(o, n Block) BlockChange {
	c := BlockChange{Kind: ChangeModify, Old: &o, New: &n}
	if o.Kind.Prose() {
		c.Words = DiffWords(o.Text, n.Text)
		c.StructureOnly = !hasChange(c.Words)
	} else {
		c.Lines = DiffLines(o.Lines, n.Lines)
		c.StructureOnly = !hasChange(c.Lines)
	}
	c.Similarity = blockSimilarity(o, n)
	return c
}

func hasChange(spans []Span) bool {
	for _, s := range spans {
		if s.Op != OpEqual {
			return true
		}
	}
	return false
}

// pairModified greedily matches the most similar delete/insert pairs left in
// one changed region, best first.
func pairModified(old, nw []Block, del, ins []int, moveOf map[int]movePair) []movePair {
	var cand []int
	for _, oi := range del {
		if _, moved := moveOf[oldKey(oi)]; !moved {
			cand = append(cand, oi)
		}
	}
	var cins []int
	for _, ni := range ins {
		if _, moved := moveOf[newKey(ni)]; !moved {
			cins = append(cins, ni)
		}
	}
	if len(cand) == 0 || len(cins) == 0 {
		return nil
	}
	windowed := len(cand)*len(cins) > pairBudget

	type scored struct {
		p movePair
		s float64
	}
	var all []scored
	for a, oi := range cand {
		for b, ni := range cins {
			if windowed && abs(a-b) > pairWindow {
				continue
			}
			o, n := old[oi], nw[ni]
			if o.Kind != n.Kind {
				continue
			}
			s := blockSimilarity(o, n)
			if s < thresholdFor(o, n) {
				continue
			}
			all = append(all, scored{movePair{oi, ni}, s})
		}
	}
	sort.SliceStable(all, func(i, j int) bool {
		if all[i].s != all[j].s {
			return all[i].s > all[j].s
		}
		return all[i].p.oldIdx < all[j].p.oldIdx
	})

	usedOld := map[int]bool{}
	usedNew := map[int]bool{}
	var out []movePair
	for _, c := range all {
		if usedOld[c.p.oldIdx] || usedNew[c.p.newIdx] {
			continue
		}
		usedOld[c.p.oldIdx] = true
		usedNew[c.p.newIdx] = true
		out = append(out, c.p)
	}
	return out
}

func thresholdFor(a, b Block) float64 {
	if len(Tokenize(a.Text)) < shortBlockTokens || len(Tokenize(b.Text)) < shortBlockTokens {
		return shortBlockThreshold
	}
	return modifyThreshold
}

// blockSimilarity is token similarity for prose and line similarity for code.
func blockSimilarity(a, b Block) float64 {
	in := newInterner()
	if a.Kind.Prose() {
		return ratio(in.all(TokenTexts(Tokenize(a.Text))), in.all(TokenTexts(Tokenize(b.Text))))
	}
	return ratio(in.all(a.Lines), in.all(b.Lines))
}

func abs(i int) int {
	if i < 0 {
		return -i
	}
	return i
}