~bigbes/sr-ht-spec

ref: 824788ab8269bd6c58de5848bc4545450fac7aaf sr-ht-spec/prosediff/bench_test.go -rw-r--r-- 6.6 KiB
824788ab — Eugene Blikh mcpsrv: mark /mcp uncacheable, fail closed on origin, split tool errors from faults 2 days ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
package prosediff

import (
	"fmt"
	"strings"
	"testing"
)

// --- what these measure --------------------------------------------------------
//
// Every proposal view runs Compare over two whole revisions of a specification
// document: both are segmented into blocks with goldmark, the two block
// sequences are aligned, and every block the alignment paired as modified is
// diffed again word by word (or line by line, inside a code fence). It is the
// most expensive thing this service does per request that is not I/O, and it
// grows with the square of the number of blocks in the worst case, so the size
// of the document is the number worth watching.
//
// The corpus below is a specification of the shape this service holds: numbered
// chapters, prose paragraphs, requirement lists, tables and fenced examples.
// The edited revision applies the four edits a proposal actually makes — a
// reworded sentence, an inserted requirement, a deleted paragraph and a moved
// chapter — so the alignment has one of every kind to recognise rather than a
// single wall of insertions.

// benchChapters is how many chapters the synthetic specification has. Around
// 40 blocks each, so the document is a few hundred blocks: the size at which
// the alignment, and not the segmentation, dominates.
const benchChapters = 24

// benchSpec renders a synthetic specification. `shift` rotates the chapter
// bodies by one so that a chapter appears at a different position in the two
// revisions, which is the move the alignment exists to recognise.
func benchSpec(edited bool) []byte {
	var b strings.Builder
	b.WriteString("---\nid: SPEC-1\ntitle: The synthetic specification\nstatus: draft\n---\n\n")
	b.WriteString("# The synthetic specification\n\n")

	order := make([]int, 0, benchChapters)
	for i := 0; i < benchChapters; i++ {
		order = append(order, i)
	}
	if edited {
		// One chapter moved: the last is read first. Exactly one, so the move
		// detection has something to find and the rest of the alignment is still
		// an ordinary walk.
		order = append(order[len(order)-1:], order[:len(order)-1]...)
	}

	for _, i := range order {
		fmt.Fprintf(&b, "## %d. Chapter %d\n\n", i+1, i)
		fmt.Fprintf(&b, "This chapter states what the service does with the %dth\n"+
			"kind of document, in prose hard-wrapped the way a specification is\n"+
			"written, so that a one-word edit rewraps the paragraph and a\n"+
			"line-oriented differ would report the whole of it as replaced.\n\n", i)

		if edited && i == benchChapters/3 {
			// A reworded sentence: same block, edited, so the word-level diff runs.
			b.WriteString("A revision **must** carry an id, and the id must be stable\n" +
				"across every revision of the document that carries it.\n\n")
		} else {
			b.WriteString("A revision must carry an id, and that id is stable across\n" +
				"every revision of the document that carries it.\n\n")
		}

		if !(edited && i == benchChapters/2) {
			// Deleted in the edited revision: one whole paragraph gone.
			fmt.Fprintf(&b, "The paragraph chapter %d loses when the edit lands. It is\n"+
				"here so the alignment has a deletion to recognise and not only\n"+
				"insertions.\n\n", i)
		}

		b.WriteString("Requirements:\n\n")
		b.WriteString("- the reader must be able to address a revision by its hash\n")
		b.WriteString("- the writer must not be able to rewrite an approved branch\n")
		b.WriteString("- a proposal must name the revision it was cut from\n")
		if edited && i == benchChapters/4 {
			b.WriteString("- a proposal must carry a summary of at most one paragraph\n")
		}
		b.WriteString("\n")

		b.WriteString("| field | required | note |\n")
		b.WriteString("| ----- | -------- | ---- |\n")
		b.WriteString("| id | yes | stable across revisions |\n")
		b.WriteString("| title | yes | shown in every listing |\n")
		b.WriteString("| status | no | draft when absent |\n\n")

		b.WriteString("```yaml\n")
		fmt.Fprintf(&b, "id: SPEC-1.%d\n", i)
		b.WriteString("title: the example this chapter is about\n")
		if edited && i == benchChapters/6 {
			// A code fence is diffed line by line, whitespace and all: the other
			// half of the split this package exists for.
			b.WriteString("status: approved\n")
		} else {
			b.WriteString("status: draft\n")
		}
		b.WriteString("```\n\n")

		b.WriteString("> A block quote, because a specification always has one.\n\n")
	}
	return []byte(b.String())
}

// BenchmarkCompare is the proposal view's whole diff: both revisions segmented,
// the block sequences aligned, and every paired block diffed word by word or
// line by line.
func BenchmarkCompare(b *testing.B) {
	oldSrc := benchSpec(false)
	newSrc := benchSpec(true)

	b.ReportAllocs()
	b.SetBytes(int64(len(oldSrc) + len(newSrc)))
	for b.Loop() {
		d := Compare(oldSrc, newSrc)
		// Asserted rather than assumed: an alignment that paired nothing would be
		// the fastest run here, and so would one that found no edit at all.
		if d.Stats.BlocksModified == 0 || d.Stats.BlocksDeleted == 0 ||
			d.Stats.BlocksInserted == 0 || d.Stats.BlocksMoved == 0 {
			b.Fatalf("the fixture lost a change kind: %+v", d.Stats)
		}
	}
}

// BenchmarkCompareUnchanged is the same document against itself — the common
// case in a long proposal, where most files a reviewer opens are untouched.
// What it measures is segmentation plus the alignment's cheap path.
func BenchmarkCompareUnchanged(b *testing.B) {
	src := benchSpec(false)

	b.ReportAllocs()
	b.SetBytes(int64(2 * len(src)))
	for b.Loop() {
		d := Compare(src, src)
		if d.Stats.BlocksModified != 0 || d.Stats.BlocksInserted != 0 ||
			d.Stats.BlocksDeleted != 0 {
			b.Fatalf("a document compared with itself reported edits: %+v", d.Stats)
		}
	}
}

// BenchmarkSegment is the parse half alone, so a regression can be attributed
// to segmentation or to alignment rather than to "the diff".
func BenchmarkSegment(b *testing.B) {
	src := benchSpec(false)

	b.ReportAllocs()
	b.SetBytes(int64(len(src)))
	for b.Loop() {
		if len(Segment(src)) == 0 {
			b.Fatal("the fixture segmented into no blocks")
		}
	}
}

// BenchmarkDiffWords is the inline word diff on its own: one modified paragraph
// against its edit, which is what runs once per modified block above.
func BenchmarkDiffWords(b *testing.B) {
	oldText := strings.Repeat("A revision must carry an id, and that id is stable "+
		"across every revision of the document that carries it. ", 12)
	newText := strings.Repeat("A revision must carry an identifier, and the id is "+
		"stable across each revision of the document carrying it. ", 12)

	b.ReportAllocs()
	b.SetBytes(int64(len(oldText) + len(newText)))
	for b.Loop() {
		if len(DiffWords(oldText, newText)) == 0 {
			b.Fatal("two different paragraphs produced no spans")
		}
	}
}