~bigbes/sr-ht-dolt

ref: 377c616a8a3b385f18a8d6509c7a6be3a31af2bd sr-ht-dolt/web/markdown.go -rw-r--r-- 14.9 KiB
377c616a — Eugene Blikh web: render memory bodies as markdown, and resolve their references 3 days ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
package web

import (
	"bytes"
	"html/template"
	"net/url"

	"github.com/yuin/goldmark"
	"github.com/yuin/goldmark/ast"
	"github.com/yuin/goldmark/extension"
	"github.com/yuin/goldmark/parser"
	"github.com/yuin/goldmark/renderer"
	"github.com/yuin/goldmark/renderer/html"
	"github.com/yuin/goldmark/text"
	"github.com/yuin/goldmark/util"

	"sourcecraft.dev/bigbes/sr-ht-dolt/beads"
)

// --- memories, rendered as the markdown they are -------------------------------
//
// A memory's value is markdown and always was: `bd remember` stores what was
// typed, and what is typed is the same prose the memory files carry — bold
// leaders, code spans, fenced blocks, numbered steps. The view used to print it
// as pre-wrapped paragraphs, which is readable but is the source and not the
// document: a bullet list stays a line starting with a hyphen, a recipe stays
// four spaces of indent, and `**Why:**` keeps its asterisks.
//
// Three things this rendering does that a stock markdown filter would not:
//
//  1. `[[slug]]` becomes a link to the memory it names, in whichever database
//     holds it (see the wikilink parser below). That is the whole point of the
//     notation, and memories reference each other across trackers constantly:
//     the mirroring workflow files a memory by its type, so a related memory is
//     as likely to be in another tracker as in this one.
//  2. An issue id in the prose links to the issue, through the same
//     cross-database index the beads detail pane uses.
//  3. Raw HTML is *escaped and shown*, not dropped. goldmark's safe default
//     omits it, and the memory corpus is full of `<placeholder>` spellings —
//     `SRHT_<NAME>_VER`, `~/data/home/<repo>` — that CommonMark reads as tags.
//     Omitting them would silently rewrite `SRHT_<NAME>_VER` to `SRHT__VER`,
//     which is worse than showing markup: it is showing a different fact.
//
// Safety is goldmark's default posture, kept: no unsafe HTML, no dangerous URL
// schemes in links, everything that came out of the database escaped on its way
// to the browser. The one thing marked template.HTML is the finished document
// this file produced.

// memoryLinkIndexKey carries the per-request link index into the parse. A
// goldmark.Markdown is stateless and shared; what varies per request is which
// databases this caller may browse, and that belongs in the parse context rather
// than in a second renderer built per page.
var memoryLinkIndexKey = parser.NewContextKey()

// memoryMarkdown is the shared renderer. goldmark's own parsers and renderers
// are safe for concurrent use — all per-conversion state lives in the context —
// so this is built once and never rebuilt.
var memoryMarkdown = goldmark.New(
	// GFM for the shapes the memories actually use: tables, strikethrough, task
	// lists, and linkify — a bare https://dolt.srht.bigb.es/~bigbes/<repo> in the
	// prose is a URL the reader wants to follow.
	goldmark.WithExtensions(extension.GFM),
	goldmark.WithParserOptions(
		// Ahead of the link parser (200), behind the task-list marker (0): "[[" is
		// a wikilink before it is a link label. Registered as an inline parser and
		// not as a text rewrite, so a "[[slug]]" written inside a code span is
		// left alone by construction — inline parsers do not run in there.
		parser.WithInlineParsers(util.Prioritized(wikilinkParser{}, 150)),
		parser.WithASTTransformers(util.Prioritized(issueLinkTransformer{}, 900)),
	),
	goldmark.WithRendererOptions(
		renderer.WithNodeRenderers(util.Prioritized(memoryNodeRenderer{}, 100)),
	),
)

// memoryLinks renders memory bodies for one request. It holds the link index —
// prefixes and memory slugs over the databases this caller may browse — and
// nothing else; a nil one still renders markdown, with every reference left as
// text, which is a whole answer and not a degraded one.
type memoryLinks struct {
	index *beads.PrefixIndex
}

// Body renders one memory's stored text as HTML.
//
// It is the only function here that produces template.HTML, and what it marks is
// the document goldmark built: every leaf that came out of the database is
// escaped by the renderer on its way in, including the slug inside a wikilink
// and the href built from it.
func (l *memoryLinks) Body(src string) template.HTML {
	var index *beads.PrefixIndex
	if l != nil {
		index = l.index
	}
	pc := parser.NewContext()
	if index != nil {
		pc.Set(memoryLinkIndexKey, index)
	}
	var buf bytes.Buffer
	if err := memoryMarkdown.Convert([]byte(src), &buf, parser.WithContext(pc)); err != nil {
		// Convert fails only on a write, which this buffer cannot do. The memory is
		// still shown, as the escaped text it was: a rendering that could not run is
		// not a reason to answer with a blank pane.
		return template.HTML(`<p class="mem-raw">` +
			template.HTMLEscapeString(src) + `</p>`)
	}
	return template.HTML(buf.String())
}

// memoryHref is the address of one memory in one database: the memory view
// narrowed to a single slug, which is the page a reference wants to land on.
func memoryHref(owner, name, slug string) string {
	return "/~" + url.PathEscape(owner) + "/" + url.PathEscape(name) +
		"/view/memory?key=" + url.QueryEscape(slug)
}

// --- [[slug]] ------------------------------------------------------------------

// wikilink is a resolved or unresolved memory reference. Href is empty when no
// database this caller may browse holds a memory under that slug — which is
// deliberately the same node as one nobody ever wrote, so the rendering cannot
// disclose the existence of a database the caller may not see.
type wikilink struct {
	ast.BaseInline
	Href string
}

var kindWikilink = ast.NewNodeKind("MemoryWikilink")

func (n *wikilink) Kind() ast.NodeKind { return kindWikilink }

func (n *wikilink) Dump(source []byte, level int) {
	ast.DumpHelper(n, source, level, map[string]string{"Href": n.Href}, nil)
}

// wikilinkParser turns "[[slug]]" into a wikilink node, resolving the slug
// against the request's index as it goes.
type wikilinkParser struct{}

func (wikilinkParser) Trigger() []byte { return []byte{'['} }

// Parse reads a wikilink out of the current line, or nothing at all: a "[[" with
// no closing "]]" on the same line, and anything whose slug is not slug-shaped,
// is left to the ordinary link parser and ends up as the text it was. Memory
// slugs are single-token keys — `bd remember --key` takes one — so a reference
// never spans a line.
func (wikilinkParser) Parse(_ ast.Node, block text.Reader, pc parser.Context) ast.Node {
	line, _ := block.PeekLine()
	if len(line) < 5 || line[0] != '[' || line[1] != '[' {
		return nil
	}
	end := bytes.Index(line, []byte("]]"))
	if end < 3 {
		return nil
	}
	slug := string(line[2:end])
	if !isMemorySlug(slug) {
		return nil
	}
	block.Advance(end + 2)

	node := &wikilink{}
	if index, ok := pc.Get(memoryLinkIndexKey).(*beads.PrefixIndex); ok {
		if d, found := index.LookupMemory(slug); found {
			node.Href = memoryHref(d.OwnerName, d.Name, slug)
		}
	}
	// The slug is carried as a string node rather than as a source segment: a
	// line the reader hands back can be padded — a list item's continuation
	// indent is synthesised, not sliced — and an offset into it is then not an
	// offset into the source. The renderer escapes a string node exactly as it
	// escapes every other leaf.
	node.AppendChild(node, ast.NewString([]byte(slug)))
	return node
}

// memorySlugMax bounds what this will treat as a slug. `bd remember --key` takes
// a short name; a "[[" followed by half a paragraph and a "]]" is prose that
// happens to contain brackets.
const memorySlugMax = 128

// isMemorySlug is the shape a memory key has: the characters `bd remember --key`
// and the memory files' `name:` field use, and no others. It is deliberately
// narrower than "anything without brackets" — a bracketed aside is not a
// reference, and the difference has to be decidable without asking the index,
// since a slug nobody holds must render the same way whether or not it is one.
func isMemorySlug(s string) bool {
	if s == "" || len(s) > memorySlugMax {
		return false
	}
	for _, r := range s {
		switch {
		case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
		case r == '-', r == '_', r == '.', r == '/':
		default:
			return false
		}
	}
	return true
}

// --- issue ids in memory prose -------------------------------------------------

// issueLinkTransformer links the issue ids in a memory's prose to the databases
// that own them, reusing the index this render already built for the wikilinks.
//
// It runs after inline parsing, over the text nodes only, and never descends
// into a code span, a link, an autolink or a wikilink: an id inside `code` is
// being shown rather than cited, and an id inside a link label would nest an
// anchor in an anchor.
type issueLinkTransformer struct{}

func (issueLinkTransformer) Transform(doc *ast.Document, reader text.Reader, pc parser.Context) {
	index, ok := pc.Get(memoryLinkIndexKey).(*beads.PrefixIndex)
	if !ok || index == nil {
		return
	}
	source := reader.Source()

	// Collected first and rewritten after: replacing a node during the walk that
	// found it is how a walk starts stepping over its own edits.
	var texts []*ast.Text
	_ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
		if !entering {
			return ast.WalkContinue, nil
		}
		switch n.Kind() {
		case ast.KindLink, ast.KindImage, ast.KindAutoLink, ast.KindCodeSpan,
			ast.KindRawHTML, ast.KindHTMLBlock, ast.KindCodeBlock,
			ast.KindFencedCodeBlock, kindWikilink:
			return ast.WalkSkipChildren, nil
		case ast.KindText:
			t := n.(*ast.Text)
			// A raw text node is a code span's content, and a padded one carries a
			// block indent its segment offsets do not describe. Neither can be cut
			// on byte offsets taken from the source.
			if !t.IsRaw() && t.Segment.Padding == 0 {
				texts = append(texts, t)
			}
		}
		return ast.WalkContinue, nil
	})

	for _, t := range texts {
		linkIssueIDs(t, source, index)
	}
}

// linkIssueIDs replaces one text node with the sequence of text and link nodes
// its ids imply. A node with no id in it is left exactly as it was.
func linkIssueIDs(t *ast.Text, source []byte, index *beads.PrefixIndex) {
	seg := t.Segment
	refs := index.Scan(string(source[seg.Start:seg.Stop]))
	if len(refs) == 0 {
		return
	}
	parent := t.Parent()
	if parent == nil {
		return
	}

	var nodes []ast.Node
	last := seg.Start
	for _, ref := range refs {
		start, stop := seg.Start+ref.Start, seg.Start+ref.End
		if start > last {
			nodes = append(nodes, ast.NewTextSegment(text.NewSegment(last, start)))
		}
		link := ast.NewLink()
		link.Destination = []byte(beadIssueHref(
			ref.Database.OwnerName, ref.Database.Name, ref.ID))
		link.AppendChild(link, ast.NewTextSegment(text.NewSegment(start, stop)))
		nodes = append(nodes, link)
		last = stop
	}

	// The tail carries the original node's line-break flags. When the id ran to
	// the end of the node the tail is empty and is kept anyway: dropping it drops
	// the newline, and the next line's first word would be glued to the id.
	tail := ast.NewTextSegment(text.NewSegment(last, seg.Stop))
	tail.SetSoftLineBreak(t.SoftLineBreak())
	tail.SetHardLineBreak(t.HardLineBreak())
	nodes = append(nodes, tail)

	for _, n := range nodes {
		parent.InsertBefore(parent, t, n)
	}
	parent.RemoveChild(parent, t)
}

// --- the three nodes this rendering does not leave to goldmark -----------------

// memoryNodeRenderer registers the wikilink renderer and replaces goldmark's
// handling of raw HTML.
type memoryNodeRenderer struct{}

func (memoryNodeRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) {
	reg.Register(kindWikilink, renderWikilink)
	reg.Register(ast.KindRawHTML, renderRawHTMLAsText)
	reg.Register(ast.KindHTMLBlock, renderHTMLBlockAsText)
	reg.Register(ast.KindImage, renderImageAsLink)
}

// renderWikilink writes the anchor, or the muted marker for a slug no database
// this caller may browse holds. The marker says only that the reference does not
// resolve *here*; it cannot say more without disclosing what it must not.
func renderWikilink(w util.BufWriter, _ []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
	n := node.(*wikilink)
	switch {
	case entering && n.Href != "":
		_, _ = w.WriteString(`<a class="mem-link" href="`)
		_, _ = w.Write(util.EscapeHTML(util.URLEscape([]byte(n.Href), true)))
		_, _ = w.WriteString(`">`)
	case entering:
		_, _ = w.WriteString(`<span class="mem-link-out" ` +
			`title="No memory with this slug in a tracker you can browse.">`)
	case n.Href != "":
		_, _ = w.WriteString(`</a>`)
	default:
		_, _ = w.WriteString(`</span>`)
	}
	return ast.WalkContinue, nil
}

// renderImageAsLink renders an image reference as a link to it rather than as an
// <img>.
//
// An <img> pointing at another host is a request that host makes on behalf of
// whoever opened the page, and a memory is prose one account wrote and another
// may read: an image in it would report the reader's address to a server the
// reader never chose to contact. The reference is kept and stays followable; it
// simply is not fetched by opening the page.
func renderImageAsLink(w util.BufWriter, _ []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
	n := node.(*ast.Image)
	if !entering {
		_, _ = w.WriteString(`</a>`)
		return ast.WalkContinue, nil
	}
	_, _ = w.WriteString(`<a class="mem-img" href="`)
	if dest := util.URLEscape(n.Destination, true); !html.IsDangerousURL(dest) {
		_, _ = w.Write(util.EscapeHTML(dest))
	}
	// The alt text is the anchor's text, which is what the children already
	// render as — an image with no alt text is left as a bare link.
	_, _ = w.WriteString(`">`)
	return ast.WalkContinue, nil
}

// renderRawHTMLAsText writes inline raw HTML as the escaped text it reads as.
//
// goldmark's safe default omits it, which is right for a comment feed and wrong
// here: `SRHT_<NAME>_VER` is a placeholder somebody typed, CommonMark sees
// `<NAME>` as a tag, and omitting it turns the memory into a different sentence.
// Escaping shows what was written and is exactly as safe as omitting it.
func renderRawHTMLAsText(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
	if !entering {
		return ast.WalkSkipChildren, nil
	}
	n := node.(*ast.RawHTML)
	for i := 0; i < n.Segments.Len(); i++ {
		seg := n.Segments.At(i)
		_, _ = w.Write(util.EscapeHTML(seg.Value(source)))
	}
	return ast.WalkSkipChildren, nil
}

// renderHTMLBlockAsText is the same treatment for a block that opened with
// something tag-shaped: shown, escaped, in a paragraph of its own rather than
// dropped.
func renderHTMLBlockAsText(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
	n := node.(*ast.HTMLBlock)
	if !entering {
		if n.HasClosure() {
			_, _ = w.Write(util.EscapeHTML(n.ClosureLine.Value(source)))
		}
		_, _ = w.WriteString("</p>\n")
		return ast.WalkContinue, nil
	}
	_, _ = w.WriteString(`<p class="mem-raw">`)
	for i := 0; i < n.Lines().Len(); i++ {
		line := n.Lines().At(i)
		_, _ = w.Write(util.EscapeHTML(line.Value(source)))
	}
	return ast.WalkContinue, nil
}