From 305bb09f7218d363c63a7d21d75e49c8df33a0cf Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Wed, 22 Jul 2026 14:03:06 +0300 Subject: [PATCH] =?UTF-8?q?feat:=20doc=20=E2=80=94=20warren's=20vault+rend?= =?UTF-8?q?er=20absorbed=20onto=20the=20git-object=20read=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit warren scanned a directory (filepath.WalkDir + os.ReadFile). There is no checkout here, so Scan walks a git tree through gitx instead and feeds the FromPages seam: the approved head, a pinned sha and a proposal branch are one code path with a different revision, and nothing downstream of Archive knows where its pages came from. Frontmatter is core's, not warren's. Front embeds core.Frontmatter and adds only what core deliberately does not model — parent, aliases, planned, and the ordered key list used for display and search text. Two parsers that disagree about a document header is a bug that surfaces in the ID registry months later. Documents key on their frontmatter id, falling back to their path: paths move and ids do not, and a duplicated id resolves to neither document rather than letting one win silently, matching what the merge already does with the approved branch. A header core rejects degrades to "no frontmatter" instead of failing, because --push-option=skip-validation means such a document can exist and refusing to render it would turn a typo into an outage. --- doc/archive.go | 426 ++++++++++++++++++++++++++++++++++++++++++++ doc/archive_test.go | 195 ++++++++++++++++++++ doc/doc.go | 112 ++++++++++++ doc/fixture_test.go | 102 +++++++++++ doc/front.go | 177 ++++++++++++++++++ doc/front_test.go | 164 +++++++++++++++++ doc/log.go | 95 ++++++++++ doc/log_test.go | 99 ++++++++++ doc/render.go | 249 ++++++++++++++++++++++++++ doc/render_test.go | 185 +++++++++++++++++++ doc/scan.go | 230 ++++++++++++++++++++++++ doc/scan_test.go | 224 +++++++++++++++++++++++ doc/table.go | 50 ++++++ doc/table_test.go | 53 ++++++ doc/wikilink.go | 220 +++++++++++++++++++++++ 15 files changed, 2581 insertions(+) create mode 100644 doc/archive.go create mode 100644 doc/archive_test.go create mode 100644 doc/doc.go create mode 100644 doc/fixture_test.go create mode 100644 doc/front.go create mode 100644 doc/front_test.go create mode 100644 doc/log.go create mode 100644 doc/log_test.go create mode 100644 doc/render.go create mode 100644 doc/render_test.go create mode 100644 doc/scan.go create mode 100644 doc/scan_test.go create mode 100644 doc/table.go create mode 100644 doc/table_test.go create mode 100644 doc/wikilink.go diff --git a/doc/archive.go b/doc/archive.go new file mode 100644 index 0000000000000000000000000000000000000000..85571edb4913897e16ca8ec4e43bb59d5bf2797f --- /dev/null +++ b/doc/archive.go @@ -0,0 +1,426 @@ +package doc + +import ( + "net/url" + "path" + "regexp" + "strings" + + "sourcecraft.dev/bigbes/sr-ht-spec/core" +) + +var schemeRe = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9+.-]*:`) + +// Archive is one space at one revision: the ordered document set plus the +// lookup structures the resolver, the read plane and the indexer need. +// +// It holds no bodies and touches nothing outside itself. That is the property +// the design leans on — the archive is built from a git tree by Scan, but +// everything below this line would work just as well against a page set that +// arrived some other way. +type Archive struct { + // Space is the space these documents belong to. It is what site hrefs are + // built from and what an indexed document is filtered by at query time. A + // zero SpaceRef yields root-relative hrefs, which is what a caller building + // an archive outside a space context gets. + Space core.SpaceRef + // Rev is the revision the documents were read at, as the caller named it. + // Pass a resolved commit sha when the archive must stay pinned; a branch + // name here means "whatever that branch pointed at when Scan ran". + Rev string + + Pages []*Page + + byID map[string]*Page + byPath map[string]*Page // repo-relative path, with extension + byStem map[string]*Page // filename stem -> the document that won the stem + // stemsIn maps "
/" to a document, so a bare wikilink written + // in one section prefers a document in the same section — Obsidian's + // proximity rule, which is what colliding stems across sections mean. + stemsIn map[string]*Page + // assets maps an attachment's base name and its full path to that path, so + // `![[image.png]]` resolves the way it is written. + assets map[string]string + // aliases maps a normalised `aliases:` entry to the canonical document ID. + aliases map[string]string +} + +// newArchive returns an Archive with empty lookup maps. +func newArchive(sp core.SpaceRef, rev string) *Archive { + return &Archive{ + Space: sp, + Rev: rev, + byID: make(map[string]*Page), + byPath: make(map[string]*Page), + byStem: make(map[string]*Page), + stemsIn: make(map[string]*Page), + assets: make(map[string]string), + aliases: make(map[string]string), + } +} + +// FromPages rebuilds an Archive's lookup structures from a page set that was +// produced earlier, without reading anything. +// +// This is the seam the git-tree walk feeds: Scan produces pages and calls into +// here, and nothing downstream of Archive knows the difference. aliases and the +// attachment index are supplied separately because they are not derivable from +// a Page; pass nil for either when they are not needed. +func FromPages(sp core.SpaceRef, rev string, pages []*Page, aliases, assets map[string]string) *Archive { + a := newArchive(sp, rev) + a.Pages = pages + for _, p := range pages { + a.register(p) + } + for k, v := range aliases { + a.aliases[k] = v + } + for k, v := range assets { + a.assets[k] = v + } + return a +} + +// register wires one document into the lookup maps. +// +// An ID already claimed is never overwritten: two documents that both resolve +// to one name is exactly the case the design refuses to guess about, and the +// loser stays reachable by path rather than being silently merged into the +// winner. +func (a *Archive) register(p *Page) { + if _, taken := a.byID[p.ID]; !taken { + a.byID[p.ID] = p + } + if p.Path != "" { + a.byPath[p.Path] = p + } + stem := Stem(p.Path) + if stem == "" { + return + } + if cur, ok := a.byStem[stem]; !ok || lessByRank(p.Path, cur.Path) { + a.byStem[stem] = p + } + if key := p.Section + "/" + stem; a.stemsIn[key] == nil { + a.stemsIn[key] = p + } +} + +// lessByRank orders two paths competing for the same bare stem: shorter path +// first, then lexicographic — total and deterministic. +// +// warren ranked by a fixed list of the vault's top-level directories +// ("wiki" beat "sources"). A space here has no such vocabulary — its +// directories are whatever the policy's auto_merge globs name — so ranking by +// them would be ranking by names that do not exist. +func lessByRank(a, b string) bool { + if len(a) != len(b) { + return len(a) < len(b) + } + return a < b +} + +// Stem returns the filename stem of a path ("specs/storage.md" -> "storage"). +// It returns "" for an empty path. +func Stem(p string) string { + if p == "" { + return "" + } + base := path.Base(p) + return strings.TrimSuffix(base, path.Ext(base)) +} + +// Page returns a document by ID. +func (a *Archive) Page(id string) (*Page, bool) { p, ok := a.byID[id]; return p, ok } + +// ByPath returns a document by its path in the tree, extension included. This +// is what the read plane's `GET /~user/space/` resolves through. +func (a *Archive) ByPath(p string) (*Page, bool) { pg, ok := a.byPath[p]; return pg, ok } + +// All returns all documents in path order. +func (a *Archive) All() []*Page { return a.Pages } + +// Aliases returns the alias -> canonical document ID map. +func (a *Archive) Aliases() map[string]string { return a.aliases } + +// Assets returns the attachment lookup (base name and path -> path). +func (a *Archive) Assets() map[string]string { return a.assets } + +// Canonical resolves an alias to its document. It reports ok=false when the +// name is not a known alias or the alias points at a document that is gone. +func (a *Archive) Canonical(alias string) (*Page, bool) { + id, ok := a.aliases[normalizeName(alias)] + if !ok { + return nil, false + } + p, ok := a.byID[id] + return p, ok +} + +// Children returns the documents whose immediate parent is id, in path order. +func (a *Archive) Children(id string) []*Page { + var out []*Page + for _, p := range a.Pages { + if p.ParentID == id { + out = append(out, p) + } + } + return out +} + +// Roots returns top-level documents (those without a parent). +func (a *Archive) Roots() []*Page { + var out []*Page + for _, p := range a.Pages { + if p.ParentID == "" { + out = append(out, p) + } + } + return out +} + +// Backlinks returns documents that link to id. Catalog and log documents are +// skipped: they link to nearly everything, so counting them would make every +// document look referenced and orphan detection would never return a result. +func (a *Archive) Backlinks(id string) []*Page { + var out []*Page + for _, p := range a.Pages { + if SuppressesEdges(p) { + continue + } + for _, l := range p.Links { + if l == id { + out = append(out, p) + break + } + } + } + return out +} + +// SuppressesEdges reports whether a document's outbound links are excluded from +// backlink counts. A catalog links to everything in its section and a log +// summarises everything that happened; left in, no document in the space can +// ever have zero inbound links. +func SuppressesEdges(p *Page) bool { + return p.Kind == KindCatalog || p.Kind == KindLog +} + +// base is the site path prefix every href in this archive hangs off: +// "/~owner/space", or "" when the archive has no space. +func (a *Archive) base() string { + if a.Space.Owner == "" || a.Space.Name == "" { + return "" + } + return "/" + a.Space.String() +} + +// DocHref is the site path a document renders at: its tree path without the +// ".md" extension, under the space prefix. +// +// The extension is dropped because the read plane negotiates content by it — +// "path.md" is the raw source — and a wikilink means "show me this document", +// not "show me its bytes". +func (a *Archive) DocHref(p *Page) string { + return a.base() + "/" + escapePath(strings.TrimSuffix(p.Path, core.DocExt)) +} + +// AssetHref is the site path an attachment is served at. Every segment is +// escaped so spaces, Cyrillic and literal percent signs survive. +func (a *Archive) AssetHref(p string) string { + return a.base() + "/" + escapePath(p) +} + +func escapePath(p string) string { + parts := strings.Split(p, "/") + for i, seg := range parts { + parts[i] = url.PathEscape(seg) + } + return strings.Join(parts, "/") +} + +// Resolve implements Resolver. dest is a link destination as written in a +// document living in fromDir (space-relative, "" for the space root): either a +// wikilink target ("SPEC-0007", "specs/storage", "note#heading") or an ordinary +// markdown destination (a URL, or a path relative to fromDir). +func (a *Archive) Resolve(fromDir, dest string) Target { + dest = strings.TrimSpace(dest) + if dest == "" || strings.HasPrefix(dest, "#") { + return Target{Href: dest} + } + if schemeRe.MatchString(dest) || strings.HasPrefix(dest, "//") { + return Target{Href: dest, IsExternal: true} + } + + base, frag := splitFragment(dest) + if base == "" { + return Target{Href: dest} + } + + if p := a.lookupPage(fromDir, base); p != nil { + return Target{ + Href: a.DocHref(p) + fragmentSuffix(frag), + PageID: p.ID, + Path: p.Path, + Kind: string(p.Kind), + } + } + if rel, ok := a.lookupAsset(fromDir, base); ok { + return Target{Href: a.AssetHref(rel), Path: rel} + } + // Nothing resolved. The destination is handed back exactly as it was + // written rather than pointed at an invented URL: the renderer marks it + // visibly broken, and a caller that wants to repair it needs to see what + // the author actually typed. + return Target{Href: dest, Missing: true} +} + +// lookupPage resolves a link target to a document, narrowest scope first: an +// explicit path wins outright, then a document id, then a name beside the +// linking document, then within its section, then space-wide, then aliases. +// +// The id step is what makes [[SPEC-0007]] work and is matched exactly — +// case-insensitive matching would let a lowercase or homograph id resolve to a +// document it is not, which is the confusion core.ParseDocID exists to prevent. +func (a *Archive) lookupPage(fromDir, base string) *Page { + bare := strings.TrimSuffix(base, core.DocExt) + if bare == "" { + return nil + } + + if strings.Contains(bare, "/") { + // Space-relative ("specs/storage") or relative to the linking document + // ("../specs/storage", as an ordinary markdown link would write it). A + // path-qualified target that matches nothing is a miss, not an + // invitation to fall back to a bare stem in some other directory. + for _, cand := range []string{bare, cleanJoin(fromDir, bare)} { + if cand == "" { + continue + } + if p, ok := a.byPath[cand+core.DocExt]; ok { + return p + } + if p, ok := a.byID[cand]; ok { + return p + } + } + return nil + } + + if p, ok := a.byID[bare]; ok && p.DocID == bare { + return p + } + + // A bare name carrying a non-markdown extension ("diagram.png") names an + // attachment. Stem matching would strip the extension and could hand back an + // unrelated document that happens to be called "diagram". + if ext := path.Ext(bare); ext != "" && !strings.EqualFold(ext, core.DocExt) { + return nil + } + + if cand := cleanJoin(fromDir, bare); cand != "" { + if p, ok := a.byPath[cand+core.DocExt]; ok { + return p + } + } + if p, ok := a.stemsIn[topSection(fromDir)+"/"+bare]; ok { + return p + } + if p, ok := a.byStem[bare]; ok { + return p + } + if p, ok := a.byID[bare]; ok { + return p + } + if p, ok := a.Canonical(bare); ok { + return p + } + return nil +} + +// lookupAsset resolves a link target to a blob in the space that is not a +// document — an image, a PDF. Embeds name attachments by base name alone +// (`![[diagram.png]]`), so the base name is tried after the paths. +func (a *Archive) lookupAsset(fromDir, base string) (string, bool) { + for _, cand := range []string{cleanJoin(fromDir, base), base} { + if cand == "" { + continue + } + if rel, ok := a.assets[cand]; ok { + return rel, true + } + } + if rel, ok := a.assets[path.Base(base)]; ok { + return rel, true + } + return "", false +} + +// splitFragment separates a heading or block reference from a link target. +// Document paths never contain '#', so the first one is always the separator. +func splitFragment(dest string) (base, frag string) { + if i := strings.IndexByte(dest, '#'); i >= 0 { + return dest[:i], dest[i+1:] + } + return dest, "" +} + +// fragmentSuffix renders a heading reference as a URL fragment matching +// goldmark's auto-generated heading anchors. Block references ("^block-id") +// have no anchor in the rendered HTML, so they are dropped. +func fragmentSuffix(frag string) string { + if frag == "" || strings.HasPrefix(frag, "^") { + return "" + } + return "#" + slugify(frag) +} + +// slugify lowercases a heading and replaces every run of non-alphanumerics with +// a single hyphen, matching goldmark's WithAutoHeadingID output for ASCII +// headings. +func slugify(s string) string { + var b strings.Builder + lastDash := true + for _, r := range strings.ToLower(s) { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + lastDash = false + case r > 127: // keep non-ASCII letters; goldmark passes them through + b.WriteRune(r) + lastDash = false + default: + if !lastDash { + b.WriteByte('-') + lastDash = true + } + } + } + return strings.Trim(b.String(), "-") +} + +// normalizeName folds an alias or link target for case-insensitive matching. +func normalizeName(s string) string { return strings.ToLower(strings.TrimSpace(s)) } + +// cleanJoin joins a relative destination onto the linking document's directory, +// returning "" when the result escapes the space root. +func cleanJoin(fromDir, dest string) string { + joined := path.Join(fromDir, dest) + joined = strings.TrimPrefix(joined, "./") + if joined == "." || joined == ".." || strings.HasPrefix(joined, "../") { + return "" + } + return joined +} + +// topSection returns the top-level directory of a space-relative path; a +// document at the space root has the empty section. +func topSection(rel string) string { + if i := strings.IndexByte(rel, '/'); i >= 0 { + return rel[:i] + } + if strings.HasSuffix(rel, core.DocExt) { + return "" + } + return rel +} diff --git a/doc/archive_test.go b/doc/archive_test.go new file mode 100644 index 0000000000000000000000000000000000000000..c81463a459b20940788f1b18c99382d5af156374 --- /dev/null +++ b/doc/archive_test.go @@ -0,0 +1,195 @@ +package doc + +import ( + "strings" + "testing" + + "sourcecraft.dev/bigbes/sr-ht-spec/core" +) + +func TestTitleFallbackChain(t *testing.T) { + arc := archiveOf(t, map[string]string{ + "notes/from-frontmatter.md": "---\ntitle: From Frontmatter\n---\n\n# Ignored H1\n", + "notes/from-h1.md": "# From H1\n\nbody\n", + "notes/from-filename.md": "just prose, no heading\n", + "notes/fenced-h1.md": "```\n# Not A Heading\n```\n\ntext\n", + }) + want := map[string]string{ + "notes/from-frontmatter.md": "From Frontmatter", + "notes/from-h1.md": "From H1", + "notes/from-filename.md": "from-filename", + "notes/fenced-h1.md": "fenced-h1", + } + for path, title := range want { + if got := mustPage(t, arc, path).Title; got != title { + t.Errorf("%s title = %q, want %q", path, got, title) + } + } +} + +func TestPageKindClassification(t *testing.T) { + arc := archiveOf(t, map[string]string{ + // No marker at all — the file-name fallback. + "notes/index.md": "# Notes Index\n", + "notes/log.md": "# Notes Log\n", + // The explicit frontmatter markers. + "specs/catalogue.md": "---\ntitle: Catalogue\ntype: catalog\n---\n", + "specs/journal.md": "---\ntitle: Journal\ntype: log\n---\n", + "specs/ordinary.md": "---\ntitle: Ordinary\ntype: concept\n---\n", + }) + want := map[string]PageKind{ + "notes/index.md": KindCatalog, + "notes/log.md": KindLog, + "specs/catalogue.md": KindCatalog, + "specs/journal.md": KindLog, + "specs/ordinary.md": KindMarkdown, + } + for path, kind := range want { + p := mustPage(t, arc, path) + if p.Kind != kind { + t.Errorf("%s kind = %q, want %q", path, p.Kind, kind) + } + if SuppressesEdges(p) != (kind != KindMarkdown) { + t.Errorf("%s: SuppressesEdges = %v for kind %q", path, SuppressesEdges(p), kind) + } + } +} + +func TestParentChainAndCycleGuard(t *testing.T) { + arc := archiveOf(t, map[string]string{ + "notes/root.md": "---\ntitle: Root\n---\n", + "notes/mid.md": "---\ntitle: Mid\nparent: \"[[root]]\"\n---\n", + "notes/leaf.md": "---\ntitle: Leaf\nparent: \"[[mid]]\"\n---\n", + // A deliberate two-document cycle, plus one that parents itself. + "notes/cyc-a.md": "---\ntitle: A\nparent: \"[[cyc-b]]\"\n---\n", + "notes/cyc-b.md": "---\ntitle: B\nparent: \"[[cyc-a]]\"\n---\n", + "notes/self.md": "---\ntitle: Self\nparent: \"[[self]]\"\n---\n", + // A parent that does not exist. + "notes/orphan.md": "---\ntitle: Orphan\nparent: \"[[nowhere]]\"\n---\n", + }) + + leaf := mustPage(t, arc, "notes/leaf.md") + if got := strings.Join(leaf.Crumbs, " > "); got != "notes/root > notes/mid" { + t.Errorf("leaf crumbs = %q", got) + } + if leaf.ParentID != "notes/mid" { + t.Errorf("leaf parent = %q", leaf.ParentID) + } + if kids := arc.Children("notes/mid"); len(kids) != 1 || kids[0] != leaf { + t.Errorf("Children(notes/mid) = %v", kids) + } + + for _, path := range []string{"notes/cyc-a.md", "notes/cyc-b.md", "notes/self.md", "notes/orphan.md"} { + p := mustPage(t, arc, path) + if p.ParentID != "" || len(p.Crumbs) != 0 { + t.Errorf("%s: cyclic/broken parent must detach, got parent=%q crumbs=%v", + path, p.ParentID, p.Crumbs) + } + } + if len(arc.Roots()) != len(arc.Pages)-2 { // mid and leaf have parents + t.Errorf("Roots() = %d of %d documents", len(arc.Roots()), len(arc.Pages)) + } +} + +func TestResolveWikilinks(t *testing.T) { + arc := archiveOf(t, map[string]string{ + "index.md": "---\ntitle: Home\n---\n", + "specs/index.md": "# Specs Index\n", + "specs/lsm-tree.md": "---\ntitle: LSM Tree\naliases:\n - log-structured-merge\n---\n", + "reports/lsm-tree.md": "Clipped report.\n", + "specs/0007-store.md": spec("SPEC-0007", "Storage model", "body"), + }) + + cases := []struct { + name string + fromDir string + dest string + want string + missing bool + }{ + {"bare stem prefers the shorter path", "specs", "lsm-tree", "/~bigbes/rfcs/specs/lsm-tree", false}, + {"path-qualified hits the other section", "specs", "reports/lsm-tree", "/~bigbes/rfcs/reports/lsm-tree", false}, + {"proximity: a report links its own section first", "reports", "lsm-tree", "/~bigbes/rfcs/reports/lsm-tree", false}, + {"a bare name prefers the linking document's own folder", "specs", "index", "/~bigbes/rfcs/specs/index", false}, + {"path-qualified index", "reports", "specs/index", "/~bigbes/rfcs/specs/index", false}, + {"a bare stem from the root falls to the stem contest", "", "lsm-tree", "/~bigbes/rfcs/specs/lsm-tree", false}, + {"document id resolves from anywhere", "reports", "SPEC-0007", "/~bigbes/rfcs/specs/0007-store", false}, + {"alias resolves to the canonical document", "specs", "log-structured-merge", "/~bigbes/rfcs/specs/lsm-tree", false}, + {"heading reference becomes an anchor", "specs", "lsm-tree#Core Components", "/~bigbes/rfcs/specs/lsm-tree#core-components", false}, + {"block reference has no anchor", "specs", "lsm-tree#^abc123", "/~bigbes/rfcs/specs/lsm-tree", false}, + {"trailing .md is stripped", "specs", "lsm-tree.md", "/~bigbes/rfcs/specs/lsm-tree", false}, + {"a lowercase id is not the id", "specs", "spec-0007", "", true}, + {"unknown target is missing", "specs", "no-such-document", "", true}, + {"path-qualified miss does not fall back to the stem", "specs", "notes/lsm-tree", "", true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := arc.Resolve(c.fromDir, c.dest) + if c.missing { + if !got.Missing { + t.Fatalf("Resolve(%q) = %+v, want Missing", c.dest, got) + } + if got.Href != c.dest { + t.Errorf("a missing target must be handed back as written, got %q", got.Href) + } + return + } + if got.Missing { + t.Fatalf("Resolve(%q) unexpectedly missing", c.dest) + } + if got.Href != c.want { + t.Errorf("Resolve(%q).Href = %q, want %q", c.dest, got.Href, c.want) + } + }) + } +} + +func TestResolveExternalAndAnchors(t *testing.T) { + arc := archiveOf(t, map[string]string{"notes/a.md": "# A\n"}) + cases := []struct { + dest string + external bool + href string + }{ + {"https://example.com/x", true, "https://example.com/x"}, + {"mailto:a@b.c", true, "mailto:a@b.c"}, + {"//cdn.example.com/x", true, "//cdn.example.com/x"}, + {"#same-page", false, "#same-page"}, + } + for _, c := range cases { + got := arc.Resolve("notes", c.dest) + if got.IsExternal != c.external || got.Href != c.href { + t.Errorf("Resolve(%q) = %+v, want href=%q external=%v", c.dest, got, c.href, c.external) + } + } +} + +// Every href is scoped to the space, because a document only means anything +// inside one, and every segment is escaped because paths carry spaces and +// Cyrillic. +func TestHrefsAreSpaceScopedAndEscaped(t *testing.T) { + arc := archiveOf(t, map[string]string{"notes/тех долг.md": "# Debt\n"}) + p := mustPage(t, arc, "notes/тех долг.md") + if got := arc.DocHref(p); got != "/~bigbes/rfcs/notes/%D1%82%D0%B5%D1%85%20%D0%B4%D0%BE%D0%BB%D0%B3" { + t.Errorf("DocHref = %q", got) + } + + if got := FromPages(core.SpaceRef{}, "main", []*Page{p}, nil, nil).DocHref(p); !strings.HasPrefix(got, "/notes/") { + t.Errorf("an archive with no space must yield root-relative hrefs, got %q", got) + } +} + +func TestBacklinksIgnoreCatalogsAndLogs(t *testing.T) { + arc := archiveOf(t, map[string]string{ + "notes/a.md": "# A\n", + "notes/b.md": "# B\n", + "notes/index.md": "# Index\n", + }) + mustPage(t, arc, "notes/b.md").Links = []string{"notes/a"} + mustPage(t, arc, "notes/index.md").Links = []string{"notes/a"} + + back := arc.Backlinks("notes/a") + if len(back) != 1 || back[0].Path != "notes/b.md" { + t.Fatalf("Backlinks = %v, want only notes/b.md", back) + } +} diff --git a/doc/doc.go b/doc/doc.go new file mode 100644 index 0000000000000000000000000000000000000000..fd0b8f5a5de7c08cad1ee329917f1982f8938e5c --- /dev/null +++ b/doc/doc.go @@ -0,0 +1,112 @@ +// Package doc is spec.sr.ht's read model for a space's documents: it turns the +// markdown blobs of one git revision into an addressable Archive, resolves the +// [[wikilinks]] between them, and renders them to HTML. +// +// It is warren's vault/ + render/ packages absorbed, with exactly one +// structural change and one consolidation. +// +// # No filesystem +// +// warren scanned a directory: filepath.WalkDir plus os.ReadFile. There is no +// checkout here, so Scan walks a git tree instead — every read resolves a +// revision and reads blobs, which is what makes the approved head, a pinned +// ?rev= and a proposal branch the same code path with a different rev. +// The seam warren already had, FromPages, is untouched in spirit: it builds an +// Archive out of a page set with no I/O at all, and everything downstream of +// Archive — resolution, backlinks, hierarchy, rendering — is unaware of where +// the pages came from. +// +// # One frontmatter parser +// +// warren's vault/frontmatter.go had its own YAML header parser. core/ already +// owns that contract (core.SplitFrontmatter, core.ParseFrontmatter, +// core.Frontmatter, core.Schema, core.DocID) and it is the one the write plane +// and the push hook validate against. Front therefore embeds core.Frontmatter +// and adds only what core deliberately does not model — `parent:`, `aliases:`, +// `planned:`, and the ordered key list used for display and search text. Two +// parsers that disagree about a document header is a bug that surfaces months +// later, in the registry, not at the door. +// +// # Tolerant on the read path, strict at the door +// +// core's parser is strict, because a malformed header must be rejected when it +// is proposed or pushed. This package is not the door: `--push-option= +// skip-validation` exists, so a document with a broken header can be on the +// approved branch, and refusing to render it would turn a cosmetic typo into an +// outage. A header this package cannot parse degrades to "no frontmatter" — the +// document still renders, still indexes, and still has a title from its first +// H1 or its file name. +package doc + +import "sourcecraft.dev/bigbes/sr-ht-spec/core" + +// PageKind distinguishes ordinary documents from the two structurally unusual +// kinds — hand-maintained catalogs and append-only logs — whose links are +// suppressed in backlink counts. +type PageKind string + +const ( + KindMarkdown PageKind = "markdown" + // KindCatalog is a hand-maintained listing page that links to nearly every + // document in its section. Marked by frontmatter `type: catalog`, or by the + // file name `index.md` as a fallback. + KindCatalog PageKind = "catalog" + // KindLog is an append-only activity log carrying dated entries. Marked by + // frontmatter `type: log`, or by the file name `log.md` as a fallback. + KindLog PageKind = "log" +) + +// Page is one document of a space at one revision, as the read plane addresses +// it. It carries no body: bodies live in git and are read by blob sha, which is +// what keeps an Archive cheap to build and impossible to serve stale. +type Page struct { + // ID is the archive's addressing key: the document's frontmatter id when it + // has a well-formed one that no other document in the space claims, and + // otherwise its path without the ".md" extension. + // + // Falling back to the path rather than to warren's bare filename stem is + // deliberate. warren keyed on the stem because Obsidian resolves wikilinks + // that way; here `id` is the load-bearing field and paths are unique by + // construction, so the fallback is unique too and cannot be taken away from + // a document by an unrelated file appearing elsewhere. + ID string `json:"id"` + // DocID is the frontmatter `id:` when it parses as a core.DocID, otherwise + // empty. A document with a malformed or duplicated id keeps its path and + // stays readable — it is excluded from id resolution, not from the archive. + DocID string `json:"doc_id,omitempty"` + + Kind PageKind `json:"kind"` + Title string `json:"title"` + // Path is the document's path in the git tree, forward-slashed, with its + // ".md" extension. + Path string `json:"path"` + // Blob is the hex sha of the document's blob at this revision: the render + // cache key. Content-addressed, so an entry keyed by it can never go stale. + Blob string `json:"blob,omitempty"` + + Status core.Status `json:"status,omitempty"` + Summary string `json:"summary,omitempty"` + Tags []string `json:"tags,omitempty"` + + // ParentID is the resolved ID of the document named by `parent:`, or empty. + ParentID string `json:"parent_id,omitempty"` + // Section is the top-level directory the document lives under ("specs", + // "notes", "reports"), used to group results and to resolve a bare wikilink + // in favour of the linking document's own section. + Section string `json:"section"` + // Crumbs is the ordered list of ancestor IDs from root to this document's + // parent (not including the document itself). + Crumbs []string `json:"crumbs,omitempty"` + // Links holds the IDs of documents this one links to, deduplicated, in + // order of first appearance. Filled in by a render pass, not by Scan. + Links []string `json:"links,omitempty"` + // WordCount is an approximate word count of the rendered text. Filled in by + // a render pass, not by Scan. + WordCount int `json:"word_count,omitempty"` +} + +// DocProperty is one frontmatter key flattened to text, in document order. +type DocProperty struct { + Name string `json:"name"` + Value string `json:"value"` +} diff --git a/doc/fixture_test.go b/doc/fixture_test.go new file mode 100644 index 0000000000000000000000000000000000000000..2b3c2e6d1717ffeed2749fac2b9aafb03b030903 --- /dev/null +++ b/doc/fixture_test.go @@ -0,0 +1,102 @@ +package doc + +import ( + "context" + "fmt" + "sort" + "testing" + "time" + + "sourcecraft.dev/bigbes/sr-ht-spec/core" + "sourcecraft.dev/bigbes/sr-ht-spec/gitx" +) + +// Fixtures are real git objects, built in-process through gitx. Nothing here +// shells out to git and nothing reads a checkout — which is the point of the +// port: if these tests could be satisfied by a directory of files, they would +// not be testing the thing that changed. + +var fxSpace = core.SpaceRef{Owner: "bigbes", Name: "rfcs"} + +func fxSig(n int) gitx.Signature { + return gitx.Signature{ + Name: "bigbes", + Email: "bigbes@gmail.com", + When: time.Date(2026, 7, 22, 12, 0, 0, 0, time.UTC).Add(time.Duration(n) * time.Minute), + } +} + +// space creates an empty bare space under a temp repos root. +func space(t *testing.T) *gitx.Repo { + t.Helper() + repo, err := gitx.Create(context.Background(), t.TempDir(), fxSpace, gitx.CreateOptions{Owner: fxSig(0)}) + if err != nil { + t.Fatalf("gitx.Create: %v", err) + } + return repo +} + +// commit writes files onto a fresh proposal branch cut from base and returns +// the branch name. A proposal branch is used rather than the approved branch +// because the write path refuses the latter by design, and because reading a +// proposal branch is one of the three revisions the read plane must serve +// through this exact code path. +func commit(t *testing.T, repo *gitx.Repo, n int, base string, files map[string]string) string { + t.Helper() + ctx := context.Background() + + branch, err := gitx.ProposalBranch(int64(n)) + if err != nil { + t.Fatalf("ProposalBranch(%d): %v", n, err) + } + if _, err := repo.CreateProposalBranch(ctx, branch, base); err != nil { + t.Fatalf("CreateProposalBranch(%q, %q): %v", branch, base, err) + } + + paths := make([]string, 0, len(files)) + for p := range files { + paths = append(paths, p) + } + sort.Strings(paths) + + writes := make([]gitx.Write, 0, len(files)) + for _, p := range paths { + writes = append(writes, gitx.Write{Path: p, Content: []byte(files[p])}) + } + meta := gitx.CommitMeta{ + Message: fmt.Sprintf("fixture %d", n), + Author: fxSig(n), + Committer: fxSig(n), + } + if _, err := repo.CommitProposal(ctx, branch, writes, meta); err != nil { + t.Fatalf("CommitProposal(%q): %v", branch, err) + } + return branch +} + +// archiveOf commits files into a fresh space and scans the result — the whole +// path from git objects to Archive. +func archiveOf(t *testing.T, files map[string]string) *Archive { + t.Helper() + repo := space(t) + rev := commit(t, repo, 1, repo.ApprovedBranch(), files) + arc, err := Scan(context.Background(), repo, fxSpace, rev) + if err != nil { + t.Fatalf("Scan(%q): %v", rev, err) + } + return arc +} + +// mustPage looks a document up by path, failing the test when it is absent. +func mustPage(t *testing.T, a *Archive, path string) *Page { + t.Helper() + p, ok := a.ByPath(path) + if !ok { + var have []string + for _, p := range a.Pages { + have = append(have, p.Path) + } + t.Fatalf("document %q missing; archive holds %v", path, have) + } + return p +} diff --git a/doc/front.go b/doc/front.go new file mode 100644 index 0000000000000000000000000000000000000000..3ee7bbe4b7de0abef371fade2c99324d20472eda --- /dev/null +++ b/doc/front.go @@ -0,0 +1,177 @@ +package doc + +import ( + "bytes" + "strings" + + "gopkg.in/yaml.v3" + + "sourcecraft.dev/bigbes/sr-ht-spec/core" +) + +// Front is a document's YAML frontmatter as the read plane sees it: everything +// core models, plus the three keys core deliberately does not and the ordered +// key list used for display and for search text. +// +// The modelled fields all come from core.Frontmatter. Nothing here re-derives +// `id`, `title`, `status`, `tags`, `type`, `summary`, `supersedes` or `owners` +// from the YAML a second time — the write plane and the push hook validate +// against core's reading of those keys, and a second interpretation of them +// here would be a disagreement waiting to be discovered in the ID registry. +type Front struct { + core.Frontmatter + + // Parent is the raw `parent:` value, still in wikilink form + // ("[[storage-model]]"). Resolved to an ID by the archive's hierarchy pass. + Parent string + // Aliases are alternative names that resolve to this document. They are what + // keeps a link working across a rename in a corpus whose links are written + // by hand. + Aliases []string + // Planned lists wikilink targets a document deliberately leaves unresolved + // because the target is meant to be written later. + Planned []string + // Props is every top-level frontmatter key in document order, flattened to + // text. It is a projection for display and search, not an interpretation: + // nothing reads meaning out of it. + Props []DocProperty +} + +// ParseFront splits a document into its frontmatter and its markdown body. +// +// It is deliberately more forgiving than core.ParseDocument, and the two +// failure modes are kept apart: +// +// - No frontmatter block, or one that is never closed, yields a zero Front and +// the source unchanged as the body. A document that opens with a thematic +// break is not a document with a broken header. +// - A block that is present and closed but that core rejects — malformed YAML, +// a duplicate key, a non-mapping — yields a zero Front and the body after +// the closing fence. The header is dropped, not rendered as prose. +// +// Neither case is an error here. This is the read path, and a document that a +// `--push-option=skip-validation` push put on the approved branch with a broken +// header is still a document worth serving and searching; rejecting it belongs +// at the door, where core is used directly. +func ParseFront(src []byte) (Front, []byte) { + front, body, err := core.SplitFrontmatter(src) + if err != nil { + return Front{}, src + } + body = bytes.TrimLeft(body, "\r\n") + + fm, err := core.ParseFrontmatter(front) + if err != nil { + return Front{}, body + } + return Front{Frontmatter: fm}.withExtras(front), body +} + +// withExtras fills the keys core does not model, walking the YAML block a +// second time for document order. Only blocks core already accepted reach this, +// so the walk can assume a well-formed mapping and simply skip what it is not +// looking at. +func (f Front) withExtras(block []byte) Front { + var node yaml.Node + if err := yaml.Unmarshal(block, &node); err != nil { + return f + } + if len(node.Content) == 0 || node.Content[0].Kind != yaml.MappingNode { + return f + } + m := node.Content[0] + + for i := 0; i+1 < len(m.Content); i += 2 { + key := m.Content[i].Value + val := m.Content[i+1] + list := nodeList(val) + + switch strings.ToLower(key) { + case "parent": + f.Parent = val.Value + case "aliases", "alias": + f.Aliases = list + case "planned": + f.Planned = list + } + if text := strings.Join(list, ", "); text != "" { + f.Props = append(f.Props, DocProperty{Name: key, Value: text}) + } + } + return f +} + +// nodeList flattens a YAML value to a list of strings: a scalar becomes one +// entry, a sequence one entry per element, and a nested mapping is skipped. +func nodeList(n *yaml.Node) []string { + switch n.Kind { + case yaml.ScalarNode: + if v := strings.TrimSpace(n.Value); v != "" { + return []string{v} + } + case yaml.SequenceNode: + out := make([]string, 0, len(n.Content)) + for _, c := range n.Content { + out = append(out, nodeList(c)...) + } + return out + } + return nil +} + +// Prop returns the first of the named keys that carries a value, or "" when +// none do. Key matching is case-insensitive, matching how documents are +// actually written. +func (f Front) Prop(keys ...string) string { + for _, want := range keys { + for _, p := range f.Props { + if strings.EqualFold(p.Name, want) { + return p.Value + } + } + } + return "" +} + +// SearchText renders the frontmatter as "key: value" lines for the keyword +// index, so a search for a tag, an owner or a summary phrase finds the document +// even though those render as chips rather than prose. +func (f Front) SearchText() string { + if len(f.Props) == 0 { + return "" + } + var b strings.Builder + for _, p := range f.Props { + b.WriteString(p.Name) + b.WriteString(": ") + b.WriteString(stripWikiBrackets(p.Value)) + b.WriteByte('\n') + } + b.WriteByte('\n') + return b.String() +} + +// stripWikiBrackets removes [[ ]] wrappers from a frontmatter value so the +// keyword index sees "storage-model" rather than "[[storage-model]]". +func stripWikiBrackets(s string) string { + s = strings.ReplaceAll(s, "[[", "") + return strings.ReplaceAll(s, "]]", "") +} + +// LinkTarget reduces a frontmatter wikilink value ("\"[[storage|Storage]]\"") to +// its bare target ("storage"). A value that is not a wikilink is returned +// trimmed, since `parent: storage` is written that way too. +func LinkTarget(v string) string { + v = strings.TrimSpace(v) + v = strings.Trim(v, `"'`) + v = strings.TrimSpace(v) + if inner, ok := strings.CutPrefix(v, "[["); ok { + if inner, ok = strings.CutSuffix(inner, "]]"); ok { + v = inner + } + } + if i := strings.IndexByte(v, '|'); i >= 0 { + v = v[:i] + } + return strings.TrimSpace(v) +} diff --git a/doc/front_test.go b/doc/front_test.go new file mode 100644 index 0000000000000000000000000000000000000000..f89107a60dea6d530dd4b544972864bb62f1eb45 --- /dev/null +++ b/doc/front_test.go @@ -0,0 +1,164 @@ +package doc + +import ( + "strings" + "testing" + + "sourcecraft.dev/bigbes/sr-ht-spec/core" +) + +func TestParseFront(t *testing.T) { + cases := []struct { + name string + src string + wantTitle string + wantType string + wantStatus core.Status + wantID string + wantTags []string + wantProps int + wantBody string + }{ + { + name: "full document", + src: "---\nid: SPEC-0007\ntitle: Proposal storage model\nstatus: draft\n" + + "type: concept\nsummary: \"Write-optimized storage\"\n" + + "tags:\n - databases\n - storage\n---\n\nA Log-Structured Merge tree.\n", + wantTitle: "Proposal storage model", wantType: "concept", + wantStatus: core.StatusDraft, wantID: "SPEC-0007", + wantTags: []string{"databases", "storage"}, wantProps: 6, + wantBody: "A Log-Structured Merge tree.\n", + }, + { + // Pushed with --push-option=skip-validation, or written by hand. + name: "no frontmatter at all", + src: "Loading... [13 kB]\n\nProse begins here.\n", + wantBody: "Loading... [13 kB]\n\nProse begins here.\n", + }, + { + // A `**Source:**` bold-label header is body text, not frontmatter, + // and must be left exactly alone. + name: "bold-label header is not frontmatter", + src: "**Source:** https://example.com/a\n\nArticle text.\n", + wantBody: "**Source:** https://example.com/a\n\nArticle text.\n", + }, + { + name: "unterminated block stays body", + src: "---\ntitle: broken\n\nno closing fence\n", + wantBody: "---\ntitle: broken\n\nno closing fence\n", + }, + { + name: "malformed yaml degrades to no properties", + src: "---\ntitle: [unclosed\n---\n\nBody.\n", + wantBody: "Body.\n", + }, + { + // core rejects a duplicated key rather than taking the last one: + // two `id:` lines is exactly the typo that corrupts the registry. + // On the read path that degrades to an untitled document, not to a + // document with one of the two ids silently chosen. + name: "duplicate key degrades to no properties", + src: "---\nid: SPEC-0007\nid: SPEC-0008\ntitle: Two ids\n---\n\nBody.\n", + wantBody: "Body.\n", + }, + { + // A thematic break must not be mistaken for an opening fence. + name: "leading thematic break is not frontmatter", + src: "---\n\nJust a rule.\n", + wantBody: "---\n\nJust a rule.\n", + }, + { + name: "explicit yaml end marker closes the block", + src: "---\ntitle: Ends early\n...\nBody.\n", + wantTitle: "Ends early", wantProps: 1, + wantBody: "Body.\n", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + front, body := ParseFront([]byte(c.src)) + if front.Title != c.wantTitle { + t.Errorf("Title = %q, want %q", front.Title, c.wantTitle) + } + if front.Type != c.wantType { + t.Errorf("Type = %q, want %q", front.Type, c.wantType) + } + if front.Status != c.wantStatus { + t.Errorf("Status = %q, want %q", front.Status, c.wantStatus) + } + if front.ID != c.wantID { + t.Errorf("ID = %q, want %q", front.ID, c.wantID) + } + if strings.Join(front.Tags, ",") != strings.Join(c.wantTags, ",") { + t.Errorf("Tags = %v, want %v", front.Tags, c.wantTags) + } + if len(front.Props) != c.wantProps { + t.Errorf("Props = %+v, want %d entries", front.Props, c.wantProps) + } + if string(body) != c.wantBody { + t.Errorf("body = %q, want %q", body, c.wantBody) + } + }) + } +} + +// The modelled fields must come from core and nowhere else, so that what the +// read plane believes a header says is what the push hook validated. +func TestParseFrontUsesCoreForModelledFields(t *testing.T) { + src := "---\nid: SPEC-0007\ntitle: T\nstatus: review\nsupersedes: SPEC-0003\n" + + "owners: [~bigbes]\nparent: \"[[storage]]\"\naliases: [storage-model]\n" + + "planned: [SPEC-0009]\n---\n\nbody\n" + front, _ := ParseFront([]byte(src)) + + if err := core.DefaultSchema().ValidateFrontmatter(front.Frontmatter); err != nil { + t.Fatalf("core rejects the frontmatter this package parsed: %v", err) + } + if front.Supersedes != "SPEC-0003" || len(front.Owners) != 1 { + t.Errorf("core fields lost: %+v", front.Frontmatter) + } + if !front.Has("parent") { + t.Errorf("Present must cover every key, including ones core does not model") + } + if front.Parent != "[[storage]]" { + t.Errorf("Parent = %q", front.Parent) + } + if strings.Join(front.Aliases, ",") != "storage-model" { + t.Errorf("Aliases = %v", front.Aliases) + } + if strings.Join(front.Planned, ",") != "SPEC-0009" { + t.Errorf("Planned = %v", front.Planned) + } +} + +func TestFrontSearchTextAndProp(t *testing.T) { + front, _ := ParseFront([]byte("---\nid: SPEC-0007\ntitle: T\nparent: \"[[storage]]\"\n---\n\nbody\n")) + + if got := front.Prop("Parent"); got != "[[storage]]" { + t.Errorf("Prop(Parent) = %q", got) + } + st := front.SearchText() + if strings.Contains(st, "[[") { + t.Errorf("search text kept wikilink brackets: %q", st) + } + if !strings.Contains(st, "id: SPEC-0007") { + t.Errorf("search text missing the id line: %q", st) + } +} + +func TestLinkTarget(t *testing.T) { + cases := map[string]string{ + `"[[storage]]"`: "storage", + `[[storage]]`: "storage", + `[[storage|Storage]]`: "storage", + ` [[specs/index]] `: "specs/index", + `storage`: "storage", + `"[[a-b]]"`: "a-b", + ``: "", + } + for in, want := range cases { + if got := LinkTarget(in); got != want { + t.Errorf("LinkTarget(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/doc/log.go b/doc/log.go new file mode 100644 index 0000000000000000000000000000000000000000..f428e8c9ec14ec6ee79e6580083da0aa6ca7bc5d --- /dev/null +++ b/doc/log.go @@ -0,0 +1,95 @@ +package doc + +import ( + "fmt" + "regexp" + "strings" +) + +// LogSection is the Section every log entry carries in the index. It is not a +// real directory: the entries all come out of one document. Default search +// excludes it, so a query returns the document describing a thing rather than +// the log entry paraphrasing it, and an explicit filter reaches it. +const LogSection = "log" + +// entryHeadRe matches an activity log's entry header: +// "## [YYYY-MM-DD] action | Title". +var entryHeadRe = regexp.MustCompile(`^##\s+\[(\d{4}-\d{2}-\d{2})\]\s+([a-z]+)\s*\|\s*(.*)$`) + +// LogEntry is one dated entry of an activity log (a document marked +// `type: log`), indexed as its own document. +// +// Indexing such a log as a single document is the wrong shape twice over: it is +// a large body of text summarising other documents, so it produces chunks that +// are near duplicates of what they describe and compete with them for the same +// queries, and a hit anywhere in the file resolves to the whole file. Split by +// entry, each piece is the size of the thing it describes and answers the +// question a log is uniquely good for — when something landed, what changed in +// a month, what came out of one session. +type LogEntry struct { + ID string // "log#-", unique even when a date repeats + Date string // YYYY-MM-DD + Action string // ingest | query | lint | update + Title string + Body string // entry text, header line excluded + Anchor string // heading anchor within /log +} + +// SplitLog parses an activity log's body into entries. Text before the first +// entry header (the file's own title) is dropped. A file that matches no header +// at all yields no entries, and the caller keeps treating it as one ordinary +// document. +func SplitLog(body []byte) []LogEntry { + lines := strings.Split(string(body), "\n") + + var ( + entries []LogEntry + cur *LogEntry + buf []string + perDate = map[string]int{} + ) + flush := func() { + if cur == nil { + return + } + cur.Body = strings.TrimSpace(strings.Join(buf, "\n")) + entries = append(entries, *cur) + cur, buf = nil, nil + } + + inFence := false + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "```") || strings.HasPrefix(trimmed, "~~~") { + inFence = !inFence + } + if !inFence { + if m := entryHeadRe.FindStringSubmatch(line); m != nil { + flush() + date, action, title := m[1], m[2], strings.TrimSpace(m[3]) + perDate[date]++ + cur = &LogEntry{ + ID: fmt.Sprintf("log#%s-%d", date, perDate[date]), + Date: date, + Action: action, + Title: title, + Anchor: fmt.Sprintf("e-%s-%d", date, perDate[date]), + } + continue + } + } + if cur != nil { + buf = append(buf, line) + } + } + flush() + return entries +} + +// SearchText is what the keyword index stores for an entry: the header fields +// followed by the body, with wikilink brackets left in place — the renderer is +// not involved here, and the bracketed names are still the words a reader would +// search for. +func (e LogEntry) SearchText() string { + return e.Date + " " + e.Action + " " + e.Title + "\n" + e.Body +} diff --git a/doc/log_test.go b/doc/log_test.go new file mode 100644 index 0000000000000000000000000000000000000000..47c486f84607b667a3dd31ab8c0c289ce09aafde --- /dev/null +++ b/doc/log_test.go @@ -0,0 +1,99 @@ +package doc + +import ( + "strings" + "testing" +) + +func TestSplitLog(t *testing.T) { + body := strings.Join([]string{ + "# Wiki Log", + "", + "## [2026-05-31] ingest | specification.website — modern web spec checklist", + "Single-page catalog. Pages created: [[specification-website-checklist]].", + "", + "## [2026-05-22] ingest | AgentMail — hosted inboxes", + "Service-side ingest, no source page.", + "", + "## [2026-05-22] lint | consistency pass", + "Second entry on the same date.", + "", + "## [2026-04-01] update | rewrote the storage cluster", + "```", + "## [2020-01-01] not-an-entry | inside a fence", + "```", + "Trailing prose after the fence.", + "", + }, "\n") + + entries := SplitLog([]byte(body)) + if len(entries) != 4 { + for _, e := range entries { + t.Logf(" %s %s | %s", e.ID, e.Action, e.Title) + } + t.Fatalf("got %d entries, want 4", len(entries)) + } + + t.Run("header fields are parsed", func(t *testing.T) { + e := entries[0] + if e.Date != "2026-05-31" || e.Action != "ingest" { + t.Errorf("date/action = %q/%q", e.Date, e.Action) + } + if e.Title != "specification.website — modern web spec checklist" { + t.Errorf("title = %q", e.Title) + } + if !strings.Contains(e.Body, "Pages created") { + t.Errorf("body lost: %q", e.Body) + } + if strings.Contains(e.Body, "## [") { + t.Errorf("body kept its own header line: %q", e.Body) + } + }) + + t.Run("the file title before the first entry is dropped", func(t *testing.T) { + for _, e := range entries { + if strings.Contains(e.Body, "# Wiki Log") { + t.Fatalf("preamble leaked into %s", e.ID) + } + } + }) + + t.Run("same-date entries get distinct ids", func(t *testing.T) { + if entries[1].ID != "log#2026-05-22-1" || entries[2].ID != "log#2026-05-22-2" { + t.Errorf("ids = %q, %q", entries[1].ID, entries[2].ID) + } + if entries[1].Anchor == entries[2].Anchor { + t.Errorf("same-date entries share an anchor: %q", entries[1].Anchor) + } + }) + + t.Run("a header inside a fence is not an entry", func(t *testing.T) { + last := entries[3] + if last.Date != "2026-04-01" { + t.Fatalf("last entry date = %q", last.Date) + } + if !strings.Contains(last.Body, "not-an-entry") { + t.Errorf("fenced text should stay in the body: %q", last.Body) + } + if !strings.Contains(last.Body, "Trailing prose") { + t.Errorf("content after the fence was dropped: %q", last.Body) + } + }) + + t.Run("search text carries the header fields", func(t *testing.T) { + st := entries[0].SearchText() + for _, want := range []string{"2026-05-31", "ingest", "specification.website"} { + if !strings.Contains(st, want) { + t.Errorf("search text missing %q: %q", want, st) + } + } + }) +} + +func TestSplitLogWithNoEntries(t *testing.T) { + // A page classified as a log that turns out not to follow the format keeps + // working as an ordinary page rather than vanishing from the index. + if got := SplitLog([]byte("# Notes\n\nJust prose, no dated headers.\n")); len(got) != 0 { + t.Fatalf("got %d entries, want none", len(got)) + } +} diff --git a/doc/render.go b/doc/render.go new file mode 100644 index 0000000000000000000000000000000000000000..ccef2dbef1eeba18e59a774d510a5eb34e503fd0 --- /dev/null +++ b/doc/render.go @@ -0,0 +1,249 @@ +package doc + +import ( + "bytes" + "strings" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/extension" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +// Link rewriting happens on the goldmark AST rather than via regex so that +// destinations are split from titles correctly and — for the [[wikilink]] +// syntax — so that links inside code spans and fenced code blocks are never +// mistaken for real links. A spec describing this service's own link syntax is +// exactly the document a regex would corrupt. + +// Target is the outcome of resolving a raw link destination found in a +// document. +type Target struct { + Href string // final href: a site path, or the destination unchanged when missing + PageID string // non-empty when the link points at another document in the archive + Path string // the target's path in the tree, when it resolved to one + Kind string // the target document's PageKind when PageID is set + // IsExternal marks http(s)/mailto and friends. + IsExternal bool + // Missing marks an internal reference that resolved to no document and no + // attachment. The renderer emits it with a distinct CSS class rather than + // dropping it, so the read plane doubles as a link checker. + Missing bool +} + +// Resolver maps a raw link destination (as written in a document located in +// directory fromDir, relative to the space root) to a Target. +type Resolver interface { + Resolve(fromDir, dest string) Target +} + +// Heading is one entry in a document's table of contents. +type Heading struct { + Level int `json:"level"` + Text string `json:"text"` + ID string `json:"id"` +} + +// Result bundles everything produced from rendering one document. +type Result struct { + HTML string + Headings []Heading + LinkedIDs []string // outbound document IDs, deduped, first-appearance order + PlainText string + WordCount int + // Wikilinks counts every [[…]] and ![[…]] in the document, resolved or not. + Wikilinks int + // MissingWikilinks holds the targets of wikilinks that resolved to no + // document, alias or attachment — in first-appearance order, deduped. + // Counted separately from ordinary markdown links because the two mean + // different things: a broken wikilink is a defect in the space's own link + // graph, while a broken relative link is usually a pasted external path. + MissingWikilinks []string +} + +// Renderer is a reusable, concurrency-safe markdown renderer. +type Renderer struct { + md goldmark.Markdown +} + +// NewRenderer builds a Renderer configured for GitHub-flavoured markdown with +// automatic heading anchors and the [[wikilink]] syntax. +func NewRenderer() *Renderer { + md := goldmark.New( + goldmark.WithExtensions( + extension.GFM, // tables, strikethrough, autolinks, task lists + extension.DefinitionList, + extension.Footnote, + wikilinkExtension{}, + ), + goldmark.WithParserOptions( + parser.WithAutoHeadingID(), + ), + goldmark.WithRendererOptions( + html.WithUnsafe(), // documents here are first-party and reviewed + renderer.WithNodeRenderers( + util.Prioritized(tableRenderer{}, tableRendererPriority), + ), + ), + ) + return &Renderer{md: md} +} + +// Render parses source, rewrites links relative to fromDir via res, and returns +// the HTML plus the extracted structure. source must already have its YAML +// frontmatter removed (see ParseFront); a leading "---" block would otherwise +// render as a thematic break followed by stray text. +func (r *Renderer) Render(source []byte, fromDir string, res Resolver) Result { + reader := text.NewReader(source) + doc := r.md.Parser().Parse(reader) + + var headings []Heading + linked := make([]string, 0, 8) + seen := make(map[string]struct{}, 8) + wikilinks := 0 + var missing []string + missingSeen := make(map[string]struct{}) + + _ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + switch node := n.(type) { + case *wikilink: + wikilinks++ + node.target = res.Resolve(fromDir, node.Dest) + if id := node.target.PageID; id != "" { + if _, ok := seen[id]; !ok { + seen[id] = struct{}{} + linked = append(linked, id) + } + } + if node.target.Missing { + if _, ok := missingSeen[node.Dest]; !ok { + missingSeen[node.Dest] = struct{}{} + missing = append(missing, node.Dest) + } + } + case *ast.Link: + t := res.Resolve(fromDir, string(node.Destination)) + node.Destination = []byte(t.Href) + if t.PageID != "" { + if _, ok := seen[t.PageID]; !ok { + seen[t.PageID] = struct{}{} + linked = append(linked, t.PageID) + } + } + case *ast.Image: + t := res.Resolve(fromDir, string(node.Destination)) + node.Destination = []byte(t.Href) + case *ast.Heading: + id, _ := node.AttributeString("id") + hid, _ := id.([]byte) + headings = append(headings, Heading{ + Level: node.Level, + Text: string(nodeText(node, source)), + ID: string(hid), + }) + } + return ast.WalkContinue, nil + }) + + var buf bytes.Buffer + _ = r.md.Renderer().Render(&buf, source, doc) + + plain := plainText(doc, source) + return Result{ + HTML: buf.String(), + Headings: headings, + LinkedIDs: linked, + PlainText: plain, + WordCount: len(strings.Fields(plain)), + Wikilinks: wikilinks, + MissingWikilinks: missing, + } +} + +// RenderInline renders a one-line fragment — a frontmatter property value — and +// returns just its inline HTML, without the wrapping paragraph. +// +// Frontmatter carries real links: `parent: "[[storage-model]]"`, and often the +// only pointer a document has to an attachment. Emitted as escaped text those +// read as literal double brackets and the attachment is unreachable. Running +// them through the same renderer as the body resolves wikilinks, marks the +// unresolved ones, and autolinks bare URLs. +// +// A value spanning more than one block is returned as rendered, paragraphs and +// all; mangling one would be worse than an extra

. +func (r *Renderer) RenderInline(source []byte, fromDir string, res Resolver) string { + h := strings.TrimSuffix(r.Render(source, fromDir, res).HTML, "\n") + inner, ok := strings.CutPrefix(h, "

") + if !ok { + return h + } + inner, ok = strings.CutSuffix(inner, "

") + if !ok || strings.Contains(inner, "

") { + return h + } + return inner +} + +// nodeText returns the concatenated text of a node's descendants. +func nodeText(n ast.Node, source []byte) []byte { + var b bytes.Buffer + _ = ast.Walk(n, func(c ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + if t, ok := c.(*ast.Text); ok { + b.Write(t.Segment.Value(source)) + } + return ast.WalkContinue, nil + }) + return b.Bytes() +} + +// plainText projects the document to searchable text: inline text with a +// newline after each block-level node, and code-block contents included +// verbatim. +func plainText(doc ast.Node, source []byte) string { + var b strings.Builder + _ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + switch node := n.(type) { + case *wikilink: + // A wikilink holds no Text child, so without this its label — often + // the only mention of a related concept in the document — would be + // absent from the keyword index. + if entering { + b.WriteString(node.DisplayText()) + b.WriteByte(' ') + } + case *ast.Text: + if entering { + b.Write(node.Segment.Value(source)) + if node.SoftLineBreak() || node.HardLineBreak() { + b.WriteByte('\n') + } + } + case *ast.FencedCodeBlock, *ast.CodeBlock: + if entering { + lines := n.Lines() + for i := 0; i < lines.Len(); i++ { + seg := lines.At(i) + b.Write(seg.Value(source)) + } + } + default: + // After leaving a block-level node, emit a separator so words from + // adjacent blocks don't run together in the search index. + if !entering && node != nil && n.Type() == ast.TypeBlock { + b.WriteByte('\n') + } + } + return ast.WalkContinue, nil + }) + return b.String() +} diff --git a/doc/render_test.go b/doc/render_test.go new file mode 100644 index 0000000000000000000000000000000000000000..723fb57bf5cba41585831200ed4a536bd1155e5c --- /dev/null +++ b/doc/render_test.go @@ -0,0 +1,185 @@ +package doc + +import ( + "strings" + "testing" +) + +// fakeResolver resolves every target except those starting with "missing" and +// those ending in an attachment extension, so the tests can exercise all four +// render paths without a real archive. The hrefs it hands back are deliberately +// arbitrary: what is under test is the renderer, not the resolution rules. +type fakeResolver struct{} + +func (fakeResolver) Resolve(fromDir, dest string) Target { + switch { + case strings.HasPrefix(dest, "http"): + return Target{Href: dest, IsExternal: true} + case strings.HasPrefix(dest, "missing"): + return Target{Href: "/p/" + dest, Missing: true} + case strings.HasSuffix(dest, ".png"), strings.HasSuffix(dest, ".pdf"), strings.HasSuffix(dest, ".base"): + return Target{Href: "/assets/" + dest} + } + id, frag, _ := strings.Cut(dest, "#") + href := "/p/" + id + if frag != "" { + href += "#" + strings.ToLower(frag) + } + return Target{Href: href, PageID: id, Kind: "markdown"} +} + +func renderMD(t *testing.T, src string) Result { + t.Helper() + return NewRenderer().Render([]byte(src), "specs", fakeResolver{}) +} + +func TestWikilink(t *testing.T) { + cases := []struct { + name string + src string + wantHTML []string + wantAbsent []string + wantLinked []string + }{ + { + name: "plain", + src: "see [[lsm-tree]] for details", + wantHTML: []string{`lsm-tree`}, + wantLinked: []string{"lsm-tree"}, + }, + { + name: "aliased", + src: "see [[lsm-tree|LSM trees]]", + wantHTML: []string{`href="/p/lsm-tree">LSM trees`}, + wantLinked: []string{"lsm-tree"}, + }, + { + name: "path qualified shows the last segment", + src: "[[reports/watchlist]]", + wantHTML: []string{`href="/p/reports/watchlist">watchlist`}, + wantLinked: []string{"reports/watchlist"}, + }, + { + name: "heading reference", + src: "[[ai-too-expensive#Zillow|AI Chernobyl]]", + wantHTML: []string{`href="/p/ai-too-expensive#zillow">AI Chernobyl`}, + wantLinked: []string{"ai-too-expensive"}, + }, + { + name: "image embed renders inline", + src: "![[diagram.png]]", + wantHTML: []string{`wiki.base`}, + wantAbsent: []string{"lsm-tree`}, + wantLinked: []string{"lsm-tree"}, + }, + { + name: "unresolved link is visibly broken, never dropped", + src: "[[missing-page]]", + wantHTML: []string{`missing-page`}, + wantAbsent: []string{"[[page]]", + }, + wantAbsent: []string{`class="wikilink"`}, + }, + { + name: "inside a fenced block it is literal text", + src: "```\n[[page]] and ![[transclusions]]\n```\n", + wantHTML: []string{ + "[[page]] and ![[transclusions]]", + }, + wantAbsent: []string{`class="wikilink"`, "LSM`}, + wantLinked: []string{"lsm-tree"}, + }, + { + name: "label is HTML-escaped", + src: "[[lsm-tree|]]", + wantAbsent: []string{"