~bigbes/sr-ht-spec

ref: f8a7a1743ad9704ff46f9e9319f8cc22b75bbdef sr-ht-spec/search/extract.go -rw-r--r-- 4.1 KiB
f8a7a174 — Eugene Blikh feat(graph): wire the proposals read to service.ListProposals (Phase 3) 26 days ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
package search

import (
	"fmt"
	"strings"
	"sync"

	"sourcecraft.dev/bigbes/sr-ht-spec/doc"
	"sourcecraft.dev/bigbes/sr-ht-spec/gitx"
)

// renderer is shared: doc.Renderer is documented as reusable and
// concurrency-safe, and building one per extraction would rebuild the whole
// goldmark pipeline for every space.
var renderer = sync.OnceValue(doc.NewRenderer)

// Bodies keys a revision's documents by tree path, which is the shape Extract
// wants them in. It pairs with doc.FromDocuments: the same []gitx.Document
// builds the Archive and supplies the text.
func Bodies(docs []gitx.Document) map[string][]byte {
	m := make(map[string][]byte, len(docs))
	for _, d := range docs {
		m[d.Path] = d.Data
	}
	return m
}

// Extract projects one space at one revision into the documents the index
// stores. bodies holds each page's raw markdown — frontmatter included, exactly
// gitx.Document.Data — keyed by doc.Page.Path.
//
// A page in the archive with no body in bodies is an error, not a page indexed
// with an empty body. The two are indistinguishable once indexed, and the
// second is how a document silently stops being findable.
//
// Three document shapes come out, matching what doc/ models:
//
//   - an ordinary document, indexed as its frontmatter projected to "key:
//     value" lines followed by its rendered plain text. The frontmatter is in
//     there because tags, owners and summaries render as chips rather than
//     prose, and a search for one of them should still find the document.
//   - a catalog (`type: catalog`, or index.md), indexed by title and section
//     only. A catalog is a page of one-line descriptions of other documents;
//     indexed whole, a query lands on the description instead of on the
//     document that owns it.
//   - an activity log (`type: log`, or log.md), which contributes its own
//     title-only document plus one document per dated entry. A hit anywhere in
//     a log otherwise resolves to the whole log; split, each entry is the size
//     of the thing it describes and carries an anchor into it.
func Extract(arc *doc.Archive, bodies map[string][]byte) ([]Document, error) {
	if arc == nil {
		return nil, fmt.Errorf("search: Extract needs an archive")
	}
	pages := arc.All()
	out := make([]Document, 0, len(pages))
	r := renderer()

	for _, p := range pages {
		src, ok := bodies[p.Path]
		if !ok {
			return nil, fmt.Errorf("search: no body supplied for %s in %s", p.Path, arc.Space)
		}
		front, body := doc.ParseFront(src)
		res := r.Render(body, doc.DirOf(p.Path), arc)

		d := Document{
			Space:   arc.Space,
			ID:      p.ID,
			Rev:     arc.Rev,
			Path:    p.Path,
			Section: p.Section,
			Title:   p.Title,
			Text:    front.SearchText() + res.PlainText,
		}
		switch p.Kind {
		case doc.KindCatalog:
			d.Text = ""
		case doc.KindLog:
			d.Section = doc.LogSection
			d.Text = ""
			out = append(out, d)
			out = append(out, logEntries(arc, p, body)...)
			continue
		}
		out = append(out, d)
	}
	return out, nil
}

// logEntries splits an activity log into one indexable document per dated
// entry.
//
// The entry ids doc.SplitLog produces are "log#<date>-<n>", named after
// warren's single vault-wide log. In a space that is not unique: a second
// document marked `type: log` — or simply a second file named log.md in another
// directory — produces the same ids, and in one index the same ids are the same
// documents, so one log would silently overwrite the other. The owning page's
// id is therefore substituted for the "log" prefix, which is a no-op for a log
// whose page id is in fact "log" and disambiguates every other case. The result
// also resolves better: "notes/dev-log#2026-05-31-1" names the document the
// entry is in.
func logEntries(arc *doc.Archive, p *doc.Page, body []byte) []Document {
	entries := doc.SplitLog(body)
	out := make([]Document, 0, len(entries))
	for _, e := range entries {
		suffix := strings.TrimPrefix(e.ID, "log#")
		out = append(out, Document{
			Space:   arc.Space,
			ID:      p.ID + "#" + suffix,
			Rev:     arc.Rev,
			Path:    p.Path,
			Anchor:  e.Anchor,
			Section: doc.LogSection,
			Title:   e.Date + " " + e.Title,
			Text:    e.SearchText(),
		})
	}
	return out
}