~bigbes/sr-ht-spec

ref: 3cb1c03d8078d5748cc13a2e9bd7ba7d078e1b37 sr-ht-spec/prosediff/wordlines.go -rw-r--r-- 4.1 KiB
3cb1c03d — Eugene Blikh go.mod: take the shared libraries' current heads 2 days ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
package prosediff

import "strings"

// LineWords is one source line of a modified block, with the part of the
// block's word edit script that falls on it.
type LineWords struct {
	// Line is the 1-based source line number, in the revision this side of the
	// diff came from.
	Line int
	// Spans is the slice of the block's edit script covering this line. A span
	// that straddled a line break is split, so every span here belongs wholly
	// to this line.
	Spans []Span
}

// WordsByLine attributes a modified block's word-level edit script to the
// source lines it came from: the old side keeps equal and deleted runs, the new
// side keeps equal and inserted ones.
//
// A word diff has no line information in it, and that is deliberate. [Tokenize]
// drops whitespace — "\n" and " " both collapse to Token.Space — which is what
// makes a rewrapped paragraph produce a byte-identical token stream and
// therefore no diff at all. The cost of that property is this function: to draw
// a line-numbered diff, the line each word sat on has to be recovered rather
// than read off.
//
// It is recoverable because the script is ordered. The spans carrying equal and
// deleted text reproduce the old block's tokens in sequence, and the equal and
// inserted ones reproduce the new block's; walking each side in step with that
// side's re-tokenized lines says which line every token belongs to.
//
// ok is false when a side's lines do not tokenize to the same sequence length
// the script consumed. That means the block's Lines and Text disagree, and the
// caller should fall back to rendering the block as one old/new pair labelled
// by line range — a wrong line number is worse than an honest range, because it
// invites a comment onto text that was never there.
func WordsByLine(c BlockChange) (old, nw []LineWords, ok bool) {
	if c.Kind != ChangeModify || len(c.Words) == 0 || c.Old == nil || c.New == nil {
		return nil, nil, false
	}
	old, ok = spread(c.Old, c.Words, OpDelete)
	if !ok {
		return nil, nil, false
	}
	nw, ok = spread(c.New, c.Words, OpInsert)
	if !ok {
		return nil, nil, false
	}
	return old, nw, true
}

// spread walks one side of the edit script — the equal runs plus the runs of
// side (OpDelete for the old side, OpInsert for the new) — and re-cuts it along
// the block's own lines.
//
// The script supplies only the operation per token; the text and its spacing
// come from re-tokenizing each source line. Taking the text from the spans
// instead would drop separators: Span.Space is false on an insertion that
// directly replaces a deletion, because in a combined rendering the deletion
// before it already carried the space. Split onto one side that deletion is
// gone, and "delta CHANGED zeta" renders as "deltaCHANGED zeta". The line's own
// tokens carry the spacing that was actually written, so they are the authority.
func spread(blk *Block, script []Span, side Op) ([]LineWords, bool) {
	// One op per token of this side, in order.
	var ops []Op
	for _, sp := range script {
		if sp.Op != OpEqual && sp.Op != side {
			continue
		}
		for range Tokenize(sp.Text) {
			ops = append(ops, sp.Op)
		}
	}

	out := make([]LineWords, len(blk.Lines))
	at := 0
	for i, ln := range blk.Lines {
		out[i] = LineWords{Line: blk.StartLine + i}
		toks := Tokenize(ln)
		for j := 0; j < len(toks); {
			if at >= len(ops) {
				return nil, false // the lines hold more tokens than the script
			}
			// Group the run of following tokens sharing this token's op.
			op, k := ops[at], j
			for k < len(toks) && at+(k-j) < len(ops) && ops[at+(k-j)] == op {
				k++
			}
			out[i].Spans = append(out[i].Spans, Span{
				Op:    op,
				Text:  joinTokens(toks[j:k]),
				Space: toks[j].Space,
			})
			at += k - j
			j = k
		}
	}
	if at != len(ops) {
		return nil, false // the script holds more tokens than the lines
	}
	return out, true
}

// joinTokens rebuilds text from a run of tokens, restoring the single space
// each token records as having preceded it.
func joinTokens(toks []Token) string {
	var b strings.Builder
	for i, t := range toks {
		if i > 0 && t.Space {
			b.WriteByte(' ')
		}
		b.WriteString(t.Text)
	}
	return b.String()
}