package doc import ( "errors" "fmt" "net/url" "path" "regexp" "strings" "unicode" "sourcecraft.dev/bigbes/sr-ht-spec/core" ) var schemeRe = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9+.-]*:`) // Archive is one space at one revision: the ordered document set plus the // lookup structures the resolver, the read plane and the indexer need. // // It holds no bodies and touches nothing outside itself. That is the property // the design leans on — the archive is built from a git tree by Scan, but // everything below this line would work just as well against a page set that // arrived some other way. type Archive struct { // Space is the space these documents belong to. It is what site hrefs are // built from and what an indexed document is filtered by at query time. A // zero SpaceRef yields root-relative hrefs, which is what a caller building // an archive outside a space context gets. Space core.SpaceRef // Rev is the revision the documents were read at, as the caller named it. // Pass a resolved commit sha when the archive must stay pinned; a branch // name here means "whatever that branch pointed at when Scan ran". Rev string Pages []*Page byID map[string]*Page byPath map[string]*Page // repo-relative path, with extension byStem map[string]*Page // filename stem -> the document that won the stem // stemsIn maps "
/" to a document, so a bare wikilink written // in one section prefers a document in the same section — Obsidian's // proximity rule, which is what colliding stems across sections mean. stemsIn map[string]*Page // assets maps an attachment's base name and its full path to that path, so // `![[image.png]]` resolves the way it is written. assets map[string]string // aliases maps a normalised `aliases:` entry to the canonical document ID. aliases map[string]string } // newArchive returns an Archive with empty lookup maps. func newArchive(sp core.SpaceRef, rev string) *Archive { return &Archive{ Space: sp, Rev: rev, byID: make(map[string]*Page), byPath: make(map[string]*Page), byStem: make(map[string]*Page), stemsIn: make(map[string]*Page), assets: make(map[string]string), aliases: make(map[string]string), } } // FromPages rebuilds an Archive's lookup structures from a page set that was // produced earlier, without reading anything. // // This is the seam the git-tree walk feeds: a caller reads a revision's // documents, [FromDocuments] turns them into pages, and nothing downstream of // Archive knows the difference. aliases and the attachment index are supplied // separately because they are not derivable from a Page; pass nil for either // when they are not needed. func FromPages(sp core.SpaceRef, rev string, pages []*Page, aliases, assets map[string]string) *Archive { a := newArchive(sp, rev) a.Pages = pages for _, p := range pages { a.register(p) } for k, v := range aliases { a.aliases[k] = v } for k, v := range assets { a.assets[k] = v } return a } // register wires one document into the lookup maps. // // An ID already claimed is never overwritten: two documents that both resolve // to one name is exactly the case the design refuses to guess about, and the // loser stays reachable by path rather than being silently merged into the // winner. func (a *Archive) register(p *Page) { if _, taken := a.byID[p.ID]; !taken { a.byID[p.ID] = p } if p.Path != "" { a.byPath[p.Path] = p } stem := Stem(p.Path) if stem == "" { return } if cur, ok := a.byStem[stem]; !ok || lessByRank(p.Path, cur.Path) { a.byStem[stem] = p } if key := p.Section + "/" + stem; a.stemsIn[key] == nil { a.stemsIn[key] = p } } // lessByRank orders two paths competing for the same bare stem: shorter path // first, then lexicographic — total and deterministic. // // warren ranked by a fixed list of the vault's top-level directories // ("wiki" beat "sources"). A space here has no such vocabulary — its // directories are whatever the policy's auto_merge globs name — so ranking by // them would be ranking by names that do not exist. func lessByRank(a, b string) bool { if len(a) != len(b) { return len(a) < len(b) } return a < b } // Stem returns the filename stem of a path ("specs/storage.md" -> "storage"). // It returns "" for an empty path. func Stem(p string) string { if p == "" { return "" } base := path.Base(p) return strings.TrimSuffix(base, path.Ext(base)) } // Page returns a document by ID. func (a *Archive) Page(id string) (*Page, bool) { p, ok := a.byID[id]; return p, ok } // ByPath returns a document by its path in the tree, extension included. This // is what the read plane's `GET /~user/space/` resolves through. func (a *Archive) ByPath(p string) (*Page, bool) { pg, ok := a.byPath[p]; return pg, ok } // All returns all documents in path order. func (a *Archive) All() []*Page { return a.Pages } // Aliases returns the alias -> canonical document ID map. func (a *Archive) Aliases() map[string]string { return a.aliases } // Assets returns the attachment lookup (base name and path -> path). func (a *Archive) Assets() map[string]string { return a.assets } // Canonical resolves an alias to its document. It reports ok=false when the // name is not a known alias or the alias points at a document that is gone. func (a *Archive) Canonical(alias string) (*Page, bool) { id, ok := a.aliases[normalizeName(alias)] if !ok { return nil, false } p, ok := a.byID[id] return p, ok } // Children returns the documents whose immediate parent is id, in path order. func (a *Archive) Children(id string) []*Page { var out []*Page for _, p := range a.Pages { if p.ParentID == id { out = append(out, p) } } return out } // Roots returns top-level documents (those without a parent). func (a *Archive) Roots() []*Page { var out []*Page for _, p := range a.Pages { if p.ParentID == "" { out = append(out, p) } } return out } // LinkPass fills in Page.Anchors, Page.Links and Page.WordCount for every // document of the archive, by rendering each body against the archive itself. // // It is a second pass rather than part of Scan because links come out of a // render, not out of a frontmatter parse: a wikilink inside a fenced code block // is not a link, and deciding that needs the markdown AST. It is here rather // than in a caller because Archive.Backlinks reads exactly what this writes — // left to each surface, one of them renders the revision twice a page view and // the next one silently reports no backlinks at all. // // bodies holds each page's raw markdown, frontmatter included, keyed by // Page.Path — the map a caller already has from the same tree walk that built // the archive. A page with no body is an inconsistency between the two and is // reported rather than skipped: skipping it would drop that document's outbound // links and under-report backlinks everywhere else, invisibly. // // The archive resolves the links, so every href produced here is the plain, // unpinned site path. A caller rendering for display wraps the resolver to // carry its own ?rev=; that wrapper must not be used here, or the link graph // would depend on how the reader arrived. // Heading slugs are collected for every document in a loop of their own, before // any document is resolved. Resolution now answers whether an anchor reference // names a heading that exists, and that answer must not depend on where in the // archive the two documents sit — pages are in path order, so ANALYSIS.md is // rendered before the SPEC.md it cites. // // Two things are needed and they do different jobs. What makes a wrong answer // impossible is [Page.missingAnchor] refusing to answer for a document whose // anchors are nil: a false "broken anchor" is worse than no check at all, // because it teaches the reader to ignore the marks. Measured over the real // corpus, reading nil as "this document has no headings" reports 26 correct // citations as broken. What this loop adds is coverage: interleaved, 98 of that // corpus's 218 anchor references resolve against a target whose headings are // still unread, so they are silently not checked at all. // // It is a parse per document rather than a second render, but a parse is most of // what a render costs: measured on the 208-document benchmark corpus, // BenchmarkLinkPass goes from ~11ms to ~17ms and from 106k to 146k allocations. // At the volume this service holds — tens of documents — that is a fraction of a // millisecond per archive build. func (a *Archive) LinkPass(r *Renderer, bodies map[string][]byte) error { if r == nil { return errors.New("doc: link pass needs a renderer") } stripped := make([][]byte, len(a.Pages)) for i, p := range a.Pages { raw, ok := bodies[p.Path] if !ok { return fmt.Errorf("doc: %s is in the archive of %s at %s but has no body", p.Path, a.Space, a.Rev) } _, body := ParseFront(raw) stripped[i] = body p.Anchors = r.Anchors(body) } for i, p := range a.Pages { res := r.Render(stripped[i], DirOf(p.Path), a) p.Links = res.LinkedIDs p.WordCount = res.WordCount } return nil } // DirOf is the directory a document lives in, space-relative, with "" for the // space root — the shape Resolve expects as fromDir. func DirOf(p string) string { d := path.Dir(p) if d == "." || d == "/" { return "" } return d } // Backlinks returns documents that link to id. Catalog and log documents are // skipped: they link to nearly everything, so counting them would make every // document look referenced and orphan detection would never return a result. // // It reads Page.Links, which LinkPass fills: an archive that has not been // through one has no link graph, and every document looks unreferenced. func (a *Archive) Backlinks(id string) []*Page { var out []*Page for _, p := range a.Pages { if SuppressesEdges(p) { continue } for _, l := range p.Links { if l == id { out = append(out, p) break } } } return out } // SuppressesEdges reports whether a document's outbound links are excluded from // backlink counts. A catalog links to everything in its section and a log // summarises everything that happened; left in, no document in the space can // ever have zero inbound links. func SuppressesEdges(p *Page) bool { return p.Kind == KindCatalog || p.Kind == KindLog } // base is the site path prefix every href in this archive hangs off: // "/~owner/space", or "" when the archive has no space. func (a *Archive) base() string { if a.Space.Owner == "" || a.Space.Name == "" { return "" } return "/" + a.Space.String() } // DocHref is the site path a document renders at: its tree path without the // ".md" extension, under the space prefix. // // The extension is dropped because the read plane negotiates content by it — // "path.md" is the raw source — and a wikilink means "show me this document", // not "show me its bytes". func (a *Archive) DocHref(p *Page) string { return a.base() + "/" + escapePath(strings.TrimSuffix(p.Path, core.DocExt)) } // AssetHref is the site path an attachment is served at. Every segment is // escaped so spaces, Cyrillic and literal percent signs survive. func (a *Archive) AssetHref(p string) string { return a.base() + "/" + escapePath(p) } func escapePath(p string) string { parts := strings.Split(p, "/") for i, seg := range parts { parts[i] = url.PathEscape(seg) } return strings.Join(parts, "/") } // Resolve implements Resolver. dest is a link destination as written in a // document living in fromDir (space-relative, "" for the space root): either a // wikilink target ("SPEC-0007", "specs/storage", "note#heading") or an ordinary // markdown destination (a URL, or a path relative to fromDir). func (a *Archive) Resolve(fromDir, dest string) Target { dest = strings.TrimSpace(dest) if dest == "" { return Target{Href: dest} } if strings.HasPrefix(dest, "#") { // A same-page reference — [[#Состояние]], the form a long RFC cites its // own chapters with. There is no document to look up, so this keeps the // early return: reporting it as a missing document would mark every // internal cross-reference broken. The fragment still goes through the // slugifier, because the heading it names renders the slug and not the // text. A block reference ("#^abc123") slugifies to nothing — fragmentSuffix // drops it — and is handed back as written rather than turned into a bare // "#" pointing at the top of the page. // // The heading is not checked for existence here the way a cross-document // one is: Resolve is told the linking document's directory, not which // document it is, so a same-page anchor's target set is not knowable // through this interface. if frag := fragmentSuffix(dest[1:]); frag != "" { return Target{Href: frag} } return Target{Href: dest} } if schemeRe.MatchString(dest) || strings.HasPrefix(dest, "//") { return Target{Href: dest, IsExternal: true} } base, frag := splitFragment(dest) if base == "" { return Target{Href: dest} } if p := a.lookupPage(fromDir, base); p != nil { return Target{ Href: a.DocHref(p) + fragmentSuffix(frag), PageID: p.ID, Path: p.Path, Kind: string(p.Kind), MissingAnchor: p.missingAnchor(frag), } } if rel, ok := a.lookupAsset(fromDir, base); ok { return Target{Href: a.AssetHref(rel), Path: rel} } // Nothing resolved. The destination is handed back exactly as it was // written rather than pointed at an invented URL: the renderer marks it // visibly broken, and a caller that wants to repair it needs to see what // the author actually typed. return Target{Href: dest, Missing: true} } // lookupPage resolves a link target to a document, narrowest scope first: an // explicit path wins outright, then a document id, then a name beside the // linking document, then within its section, then space-wide, then aliases. // // The id step is what makes [[SPEC-0007]] work and is matched exactly — // case-insensitive matching would let a lowercase or homograph id resolve to a // document it is not, which is the confusion core.ParseDocID exists to prevent. func (a *Archive) lookupPage(fromDir, base string) *Page { bare := strings.TrimSuffix(base, core.DocExt) if bare == "" { return nil } if strings.Contains(bare, "/") { // Space-relative ("specs/storage") or relative to the linking document // ("../specs/storage", as an ordinary markdown link would write it). A // path-qualified target that matches nothing is a miss, not an // invitation to fall back to a bare stem in some other directory. for _, cand := range []string{bare, cleanJoin(fromDir, bare)} { if cand == "" { continue } if p, ok := a.byPath[cand+core.DocExt]; ok { return p } if p, ok := a.byID[cand]; ok { return p } } return nil } if p, ok := a.byID[bare]; ok && p.DocID == bare { return p } // A bare name carrying a non-markdown extension ("diagram.png") names an // attachment. Stem matching would strip the extension and could hand back an // unrelated document that happens to be called "diagram". if ext := path.Ext(bare); ext != "" && !strings.EqualFold(ext, core.DocExt) { return nil } if cand := cleanJoin(fromDir, bare); cand != "" { if p, ok := a.byPath[cand+core.DocExt]; ok { return p } } if p, ok := a.stemsIn[topSection(fromDir)+"/"+bare]; ok { return p } if p, ok := a.byStem[bare]; ok { return p } if p, ok := a.byID[bare]; ok { return p } if p, ok := a.Canonical(bare); ok { return p } return nil } // lookupAsset resolves a link target to a blob in the space that is not a // document — an image, a PDF. Embeds name attachments by base name alone // (`![[diagram.png]]`), so the base name is tried after the paths. func (a *Archive) lookupAsset(fromDir, base string) (string, bool) { for _, cand := range []string{cleanJoin(fromDir, base), base} { if cand == "" { continue } if rel, ok := a.assets[cand]; ok { return rel, true } } if rel, ok := a.assets[path.Base(base)]; ok { return rel, true } return "", false } // splitFragment separates a heading or block reference from a link target. // Document paths never contain '#', so the first one is always the separator. func splitFragment(dest string) (base, frag string) { if i := strings.IndexByte(dest, '#'); i >= 0 { return dest[:i], dest[i+1:] } return dest, "" } // missingAnchor reports whether frag names a heading this document does not // render. It is the anchor half of link checking: the document resolved, so the // only thing left to be wrong is where in it the reference points. // // It answers false whenever it cannot know, and there are three such cases. An // empty fragment names no heading. A block reference ("^abc123") names a // position rather than a heading and has no anchor in the rendered HTML at all, // so it is neither rewritten nor reported. And a nil Anchors means the heading // slugs of this document were never collected — see [Page.Anchors] — which is // the state of every archive that has not been through [Archive.LinkPass]. // // The comparison goes through headingID, the one function both ends of a heading // link use, so this asks exactly the question the browser will: does the // fragment the href carries exist as an id on the target page. func (p *Page) missingAnchor(frag string) bool { if frag == "" || strings.HasPrefix(frag, "^") || p.Anchors == nil { return false } want := headingID(frag) for _, id := range p.Anchors { if id == want { return false } } return true } // fragmentSuffix renders a heading reference as a URL fragment. It goes through // headingID, the same function the parser's id generator uses, so the fragment // written here and the anchor the target document renders are one string. Block // references ("^block-id") have no anchor in the rendered HTML, so they are // dropped. func fragmentSuffix(frag string) string { if frag == "" || strings.HasPrefix(frag, "^") { return "" } return "#" + headingID(frag) } // slugify turns heading text into an anchor id: ASCII letters and digits are // lowercased, a space, a hyphen or an underscore becomes a hyphen, every other // ASCII byte is dropped; Unicode letters and digits are kept lowercased, Unicode // spaces become hyphens and the rest of Unicode is dropped. // // For pure-ASCII text the result is byte-identical to goldmark's own // WithAutoHeadingID output — including the runs of hyphens it leaves in place, // which is why nothing is trimmed here. TestASCIIHeadingIDsMatchGoldmark pins // that against goldmark itself rather than against this description. // // Non-ASCII is where the two part company, deliberately: goldmark skips every // multibyte rune, so a Russian heading has no id worth linking to. Keeping the // Cyrillic produces a fragment that is percent-encoded in a URL and readable // everywhere else. func slugify(s string) string { var b strings.Builder // goldmark trims the ASCII spaces of the heading line before slugifying it; // the trailing newline of the source line is the one that always matters. for _, r := range strings.Trim(s, " \t\r\n") { switch { case r >= 'a' && r <= 'z', r >= '0' && r <= '9': b.WriteRune(r) case r >= 'A' && r <= 'Z': b.WriteRune(r + ('a' - 'A')) case r == ' ', r == '\t', r == '\r', r == '\n', r == '-', r == '_': b.WriteByte('-') case r < 0x80: // Other ASCII — punctuation, and the vertical tab and form feed // goldmark does not count as space — is dropped, not hyphenated. case unicode.IsLetter(r), unicode.IsDigit(r): b.WriteRune(unicode.ToLower(r)) case unicode.IsSpace(r): b.WriteByte('-') default: // Non-ASCII punctuation and symbols: an em dash, a guillemet. Dropped, // so they behave like their ASCII counterparts. } } return b.String() } // normalizeName folds an alias or link target for case-insensitive matching. func normalizeName(s string) string { return strings.ToLower(strings.TrimSpace(s)) } // cleanJoin joins a relative destination onto the linking document's directory, // returning "" when the result escapes the space root. func cleanJoin(fromDir, dest string) string { joined := path.Join(fromDir, dest) joined = strings.TrimPrefix(joined, "./") if joined == "." || joined == ".." || strings.HasPrefix(joined, "../") { return "" } return joined } // topSection returns the top-level directory of a space-relative path; a // document at the space root has the empty section. func topSection(rel string) string { if i := strings.IndexByte(rel, '/'); i >= 0 { return rel[:i] } if strings.HasSuffix(rel, core.DocExt) { return "" } return rel }