~bigbes/sr-ht-dolt

377c616a8a3b385f18a8d6509c7a6be3a31af2bd — Eugene Blikh 3 days ago bd2d8a3 
web: render memory bodies as markdown, and resolve their references

A memory's value is markdown and always was: `bd remember` stores what was
typed, and what is typed is the same prose the memory files carry — bold
leaders, code spans, fenced recipes, and [[slug]] references to related
memories. The view printed it as pre-wrapped paragraphs, which shows the
source rather than the document and leaves every reference a dead end.

The renderer is web/markdown.go, one shared goldmark with three departures
from stock:

  - [[slug]] is an inline parser rather than a text rewrite, so a reference
    written inside a code span stays text by construction.
  - Raw HTML is escaped and shown, not dropped. The safe default omits it,
    and these memories are full of <placeholder> spellings CommonMark reads
    as tags; omitting one rewrites SRHT_<NAME>_VER to SRHT__VER silently,
    which is worse than showing markup.
  - An image renders as a link. An <img> at another host is a request that
    host makes on behalf of whoever opened the page.

References resolve across databases, because that is where they point: the
mirroring workflow files a memory by its type, so a related memory is as
likely to be in another tracker as in this one. The index is the one the
issue links already use — issue_prefix and the kv.memory.* keys are rows of
the same config table, so one read per database now answers both questions
and one cached projection carries both. A slug two trackers hold resolves to
the first listed, the page's own database, unlike a prefix two trackers claim
which resolves to neither: the same slug in two trackers is normally one
memory that was re-filed and left a copy behind.

A slug nobody wrote and a slug held by a database the caller may not browse
are one rendering — a muted, inert marker. A link, a distinct class or a
different tooltip would each publish the existence of a private database.

Memory.Paragraphs goes with the paragraphs it existed for.
patch browse .tar.gz .zip 
13 files changed, 1121 insertions(+), 108 deletions(-)

M beads/memory.go
M beads/memory_test.go
M beads/prefixes.go
M beads/prefixes_test.go
M docs/DESIGN.views.md
M go.mod
M go.sum
M web/beads.go
M web/handlers_view.go
A web/markdown.go
A web/markdown_test.go
M web/memory.go
M web/templates/memory.html

M beads/memory.go => beads/memory.go +8 -29
@@ 61,11 61,12 @@ type MemoryRevision struct {
// Memory is one `bd remember` entry.
type Memory struct {
	Slug string // the config key with the "kv.memory." prefix stripped
	Text string // the value, with both newline spellings normalised
	// Paragraphs is Text split on blank lines. Values are typed into shell
	// strings, so they arrive with a mix of real newlines and literal "\n"
	// escapes; a page that dumped the raw value would render either as one blob.
	Paragraphs []string
	// Text is the value with both newline spellings normalised. It is markdown —
	// `bd remember` stores whatever was written, and what is written is the same
	// markdown the memory files carry — and it is left as text here: this package
	// renders nothing, and the split into paragraphs, lists and code blocks is
	// the renderer's, not a projection's.
	Text string
	// Revision is the commit that last changed this key's value, or nil when the
	// walk could not reach it (see MemoryView.WalkTruncated).
	Revision *MemoryRevision


@@ 222,9 223,8 @@ func BuildMemories(ctx context.Context, sess MemorySession, ref string, query ur
	view.Memories = make([]Memory, 0, len(tracked))
	for key := range tracked {
		m := Memory{
			Slug:       strings.TrimPrefix(key, memoryPrefix),
			Text:       texts[key],
			Paragraphs: memoryParagraphs(texts[key]),
			Slug: strings.TrimPrefix(key, memoryPrefix),
			Text: texts[key],
		}
		if rev, ok := walk.revisions[key]; ok {
			m.Revision = &rev


@@ 395,9 395,6 @@ func memoryValuesAt(ctx context.Context, sess MemorySession, at string) (map[str
// as backslash sequences rather than as newlines.
var memoryEscapedNewline = regexp.MustCompile(`\\r\\n|\\n`)

// memoryBlankLine splits a memory into paragraphs on runs of blank lines.
var memoryBlankLine = regexp.MustCompile(`\n[ \t]*\n[ \t\n]*`)

// normalizeMemoryText brings a stored value's two newline spellings together.
// The same memory arrives with real newlines when it was written from a file or
// a heredoc and with literal backslash-n when it was typed into a shell string,


@@ 409,24 406,6 @@ func normalizeMemoryText(v string) string {
	return strings.TrimSpace(v)
}

// memoryParagraphs splits normalised text on blank lines. Line breaks *inside* a
// paragraph are kept — memories are full of bullet lists, and joining those into
// a sentence is not rendering, it is losing the structure the author typed.
func memoryParagraphs(text string) []string {
	if text == "" {
		return nil
	}
	parts := memoryBlankLine.Split(text, -1)
	out := make([]string, 0, len(parts))
	for _, p := range parts {
		if strings.TrimSpace(p) == "" {
			continue
		}
		out = append(out, strings.Trim(p, "\n"))
	}
	return out
}

// matchesMemorySearch is the ?q= rule: a case-insensitive substring of the slug
// or of the text. An empty query matches everything.
func matchesMemorySearch(q, slug, text string) bool {


M beads/memory_test.go => beads/memory_test.go +3 -3
@@ 157,7 157,9 @@ func TestMemoriesPrefixFiltering(t *testing.T) {

// A value carries both newline spellings — real ones when it was written from a
// file, literal backslash-n when it was typed into a shell string — and both
// have to become the same paragraphs.
// have to become the same text. It is the one shaping this projection does: the
// markdown itself is the renderer's to read, and it cannot read a value whose
// line breaks are still two characters.
func TestMemoriesNormaliseBothNewlineSpellings(t *testing.T) {
	const escaped = `First paragraph.\n\nSecond paragraph, line one.\nLine two.`
	real := "First paragraph.\n\nSecond paragraph, line one.\nLine two."


@@ 174,9 176,7 @@ func TestMemoriesNormaliseBothNewlineSpellings(t *testing.T) {
	v := buildMemories(t, sess, "")
	require.Len(t, v.Memories, 3)

	want := []string{"First paragraph.", "Second paragraph, line one.\nLine two."}
	for _, m := range v.Memories {
		assert.Equal(t, want, m.Paragraphs, "memory %q", m.Slug)
		assert.Equal(t, "First paragraph.\n\nSecond paragraph, line one.\nLine two.", m.Text,
			"memory %q", m.Slug)
		assert.NotContains(t, m.Text, `\n`, "the literal escape must not survive into the text")


M beads/prefixes.go => beads/prefixes.go +113 -39
@@ 10,17 10,26 @@ import (
	"sourcecraft.dev/bigbes/sr-ht-dolt/browse"
)

// --- which database owns "<prefix>-<id>" --------------------------------------
// --- which database owns "<prefix>-<id>", and which one holds "[[slug]]" ------
//
// A global-tracker issue that says "blocked by artifacts-nex.2" is naming a row
// in another database, and without this the reader has to work out which one and
// go there by hand. Two levels of tracker only pay for themselves if they point
// at each other.
//
// Memories name each other the same way and land the same way: the mirroring
// workflow files a memory by its type, so `[[srht-service-release]]` written in
// one tracker routinely lives in another, and the reference is a dead end
// wherever it is read. Both questions are answered from one read of one table —
// issue_prefix and the kv.memory.* keys are rows of the same config — so they
// are one index and one cached projection rather than two passes over every
// database.
//
// This file answers the one question that needs several databases: given an
// id-shaped string in some text, which database — of the ones this caller may
// browse — owns it. Whether the answer becomes a link, and what that link looks
// like, is the renderer's business; this package renders nothing.
// id-shaped string or a memory slug in some text, which database — of the ones
// this caller may browse — holds it. Whether the answer becomes a link, and what
// that link looks like, is the renderer's business; this package renders
// nothing.
//
// Authorization is not here, exactly as it is not in ReadyAcross: handing this
// function a database is the statement that the caller may read it. A database


@@ 32,32 41,47 @@ import (
// memories do, beside compact_tier2_days and the rest of the tracker's settings.
const prefixKey = "issue_prefix"

// PrefixCache is the prefix index's projection cache: one prefix per database,
// gated on the head hash and expiring on ReadyCacheTTL — /ready's bounds and not
// a second set (see cache.go). The zero value is usable, so a holder that has no
// constructor can carry one as a field.
// PrefixCache is the index's projection cache: one config projection per
// database, gated on the head hash and expiring on ReadyCacheTTL — /ready's
// bounds and not a second set (see cache.go). The zero value is usable, so a
// holder that has no constructor can carry one as a field.
type PrefixCache struct {
	projectionCache[prefixEntry]
}

// prefixEntry is one database's cached prefix projection: the prefix its config
// names, and the total that read reported. The total is cached with the prefix
// rather than recomputed because a cache hit reads no row — it must say what
// the read that produced it said, including that the read was partial.
// prefixEntry is one database's cached config projection: the issue prefix its
// config names, the memory slugs it stores, and the total that read reported.
// The total is cached with them rather than recomputed because a cache hit reads
// no row — it must say what the read that produced it said, including that the
// read was partial.
type prefixEntry struct {
	prefix string
	// slugs is every kv.memory.* key in that config, prefix stripped, in the
	// order the read returned them. It is cached beside the prefix because it
	// comes out of the same rows: a second projection over the same table would
	// double the reads to answer half the question.
	slugs []string
	// configTotal is the config table's reported total, clipped or not. Over Max
	// it means the read stopped short, and an absent prefix is then a row that
	// was never reached rather than a row that does not exist.
	// it means the read stopped short, and an absent prefix — or an absent
	// memory — is then a row that was never reached rather than a row that does
	// not exist.
	configTotal int
}

// PrefixIndex is prefix → the database that owns it, over the databases one
// caller may browse. It is built per caller and must not be shared between them:
// what it holds is precisely the set of databases that caller is allowed to know
// exists.
// PrefixIndex is prefix → the database that owns it and memory slug → the
// database that holds it, over the databases one caller may browse. It is built
// per caller and must not be shared between them: what it holds is precisely the
// set of databases that caller is allowed to know exists.
type PrefixIndex struct {
	byPrefix map[string]ReadyDatabase
	// byMemory is slug → the first database listed that stores a memory under it.
	// First and not "the only one", unlike byPrefix: a prefix two trackers both
	// claim is an ambiguity, but the same memory slug in two trackers is normally
	// the same memory — the mirroring workflow re-files one by its type, and a
	// re-typed memory is left behind in the tracker it moved out of. The caller
	// lists the database whose page this is first, so a slug present locally
	// always resolves locally, and the fallback is the caller's listing order.
	byMemory map[string]ReadyDatabase
	// Failed lists the databases that could not be read. It is here for the
	// caller's log and for nothing else: a database that could not be read costs
	// the ids it owns their link, and a page may not say more than that.


@@ 103,15 127,15 @@ type Reference struct {
var idPattern = regexp.MustCompile(`\b[0-9a-z_]+(?:-[0-9a-z_]+)*-[0-9a-z]+(?:\.[0-9a-z]+)*\b`)

// PrefixesAcross builds the index over the databases it is handed: for each, the
// issue_prefix its config names.
// issue_prefix its config names and the memory slugs its config stores.
//
// It is bounded exactly as ReadyAcross is — the same head-hash gate, the same
// TTL, the same ceiling — because it is the same read pattern: N stores opened
// for one request. It is cheaper per database, though: the projection is one row
// of a dozen-row table, and the fingerprint is not asked for separately, since a
// config table carrying issue_prefix is itself the statement that this is a bd
// tracker. A database with no such row simply owns no prefix, and that answer is
// cached like any other.
// for one request. It is cheaper per database, though: the projection is a
// dozen-row table read once for both halves, and the fingerprint is not asked
// for separately, since a config table carrying issue_prefix is itself the
// statement that this is a bd tracker. A database with no such row simply owns
// no prefix, and that answer is cached like any other.
//
// now is the clock the TTL is measured against, passed in rather than read here
// for the reason ReadyAcross takes one: this package reads no hidden clock.


@@ 125,7 149,10 @@ func PrefixesAcross(
	cache *PrefixCache,
	now time.Time,
) *PrefixIndex {
	index := &PrefixIndex{byPrefix: map[string]ReadyDatabase{}}
	index := &PrefixIndex{
		byPrefix: map[string]ReadyDatabase{},
		byMemory: map[string]ReadyDatabase{},
	}

	if len(dbs) > ReadyMaxDatabases {
		// The first Max in the order the caller listed them. The caller puts the


@@ 153,6 180,15 @@ func PrefixesAcross(
				ShownOf:  entry.configTotal,
			})
		}
		// The memory half is recorded before the prefix skip below: a tracker that
		// names no issue prefix still stores memories, and its slugs are as
		// linkable as any other's.
		for _, slug := range entry.slugs {
			if _, taken := index.byMemory[slug]; !taken {
				index.byMemory[slug] = d
			}
		}

		prefix := entry.prefix
		if prefix == "" {
			continue


@@ 205,7 241,7 @@ func databasePrefix(
		return entry, nil
	}

	entry, err := readPrefix(ctx, sess, ref)
	entry, err := readConfigProjection(ctx, sess, ref)
	if err != nil {
		return prefixEntry{}, err
	}


@@ 213,14 249,19 @@ func databasePrefix(
	return entry, nil
}

// readPrefix reads the issue_prefix row out of a database's config table, and
// the total that read reported. A missing table — this is not a bd tracker — is
// no prefix, the treatment every optional table gets here.
// readConfigProjection reads one database's config table down to what the index
// needs of it: the issue_prefix row, the kv.memory.* keys, and the total that
// read reported. A missing table — this is not a bd tracker — is neither a
// prefix nor a memory, the treatment every optional table gets here.
//
// The total comes back with them because the answers this can produce are
// otherwise identical: a config with no issue_prefix row and a config whose
// issue_prefix row was left past Max both arrive here as no prefix at all, and
// the same holds of a memory key.
//
// The total comes back with the prefix because the two answers this can produce
// are otherwise identical: a config with no issue_prefix row and a config whose
// issue_prefix row was left past Max both arrive here as no prefix at all.
func readPrefix(ctx context.Context, sess BrowseSession, ref string) (prefixEntry, error) {
// The whole table is walked rather than stopped at the prefix row: the memory
// keys sit anywhere in it, and one pass is what makes this one read.
func readConfigProjection(ctx context.Context, sess BrowseSession, ref string) (prefixEntry, error) {
	rows, total, err := readRowsOptional(ctx, sess, ref, memoryTable)
	if err != nil {
		return prefixEntry{}, err


@@ 231,13 272,20 @@ func readPrefix(ctx context.Context, sess BrowseSession, ref string) (prefixEntr
	}
	cols := indexCols(rows.Columns)
	for _, r := range rowsOf(rows) {
		if cell(cols, r, "key") != prefixKey {
			continue
		key := cell(cols, r, "key")
		switch {
		case key == prefixKey:
			// Lowercased because that is the case the ids themselves are written in,
			// and the index is looked up by what the text says.
			entry.prefix = strings.ToLower(strings.TrimSpace(cell(cols, r, "value")))
		case strings.HasPrefix(key, memoryPrefix):
			// The slug is stored as written and matched as written: `bd remember
			// --key` is case-sensitive, and two memories differing only in case are
			// two memories.
			if slug := strings.TrimPrefix(key, memoryPrefix); slug != "" {
				entry.slugs = append(entry.slugs, slug)
			}
		}
		// Lowercased because that is the case the ids themselves are written in,
		// and the index is looked up by what the text says.
		entry.prefix = strings.ToLower(strings.TrimSpace(cell(cols, r, "value")))
		return entry, nil
	}
	return entry, nil
}


@@ 253,6 301,32 @@ func (ix *PrefixIndex) Lookup(prefix string) (ReadyDatabase, bool) {
	return d, ok
}

// LookupMemory returns the database holding the memory written under slug. A
// slug no database the caller may browse stores is unknown here, which is what
// keeps a reference to a memory in a database they may not see indistinguishable
// from a reference to one that was never written.
func (ix *PrefixIndex) LookupMemory(slug string) (ReadyDatabase, bool) {
	if ix == nil {
		return ReadyDatabase{}, false
	}
	d, ok := ix.byMemory[slug]
	return d, ok
}

// MemorySlugs lists the slugs the index knows, in order. Like Prefixes, it
// exists for the caller's log line and for tests.
func (ix *PrefixIndex) MemorySlugs() []string {
	if ix == nil {
		return nil
	}
	out := make([]string, 0, len(ix.byMemory))
	for s := range ix.byMemory {
		out = append(out, s)
	}
	sort.Strings(out)
	return out
}

// Prefixes lists the prefixes the index knows, in order. It exists for the
// caller's log line and for tests.
func (ix *PrefixIndex) Prefixes() []string {


M beads/prefixes_test.go => beads/prefixes_test.go +124 -0
@@ 283,3 283,127 @@ func TestNilPrefixIndexKnowsNothing(t *testing.T) {
	empty := PrefixesAcross(t.Context(), nil, nil, &PrefixCache{}, readyNow)
	assert.Nil(t, empty.Scan("artifacts-46c.2"))
}

// --- the memory half of the same read ------------------------------------------

// memoryTracker is a tracker whose config holds memories beside its settings.
// The prefix is a parameter because a tracker that names none still holds
// memories, and the two halves of the projection are independent.
func memoryTracker(head, prefix string, slugs ...string) *fakeReadyDB {
	page := &browse.RowPage{
		Columns: []string{"key", "value"},
		Rows:    [][]string{{"compact_tier2_days", "30"}},
	}
	if prefix != "" {
		page.Rows = append(page.Rows, []string{"issue_prefix", prefix})
	}
	for _, s := range slugs {
		page.Rows = append(page.Rows, []string{memoryPrefix + s, "the body of " + s})
	}
	page.Total = len(page.Rows)
	return &fakeReadyDB{
		branches: []browse.Branch{{Name: "main", Head: head}},
		tables:   beadsTables(),
		rows:     map[string]*browse.RowPage{"config": page},
	}
}

// The slugs come out of the same read the prefix does, and a slug no database
// holds is unknown — which is also the answer for one held by a database this
// caller may not browse, since such a database is never handed to the index.
func TestPrefixesAcrossIndexesMemorySlugs(t *testing.T) {
	in := &readyInstance{dbs: map[int]*fakeReadyDB{
		1: memoryTracker("h-global", "global", "go-vcs-stamp-dirty", "srht-service-release"),
		2: memoryTracker("h-artifacts", "artifacts", "apk-index-cadence"),
	}}
	dbs := []ReadyDatabase{
		{ID: 1, OwnerName: "bigbes", Name: "beads-global"},
		{ID: 2, OwnerName: "bigbes", Name: "sourcehut-artifacts"},
	}

	index := PrefixesAcross(t.Context(), dbs, in.open, &PrefixCache{}, readyNow)

	assert.Equal(t,
		[]string{"apk-index-cadence", "go-vcs-stamp-dirty", "srht-service-release"},
		index.MemorySlugs())
	d, ok := index.LookupMemory("go-vcs-stamp-dirty")
	require.True(t, ok)
	assert.Equal(t, "bigbes/beads-global", d.Slug())
	d, ok = index.LookupMemory("apk-index-cadence")
	require.True(t, ok)
	assert.Equal(t, "bigbes/sourcehut-artifacts", d.Slug())
	_, ok = index.LookupMemory("never-written")
	assert.False(t, ok)

	// One read per database answered both questions.
	assert.Equal(t, 1, in.dbs[1].rowReads)
	assert.Equal(t, []string{"artifacts", "global"}, index.Prefixes())
}

// A slug two trackers hold resolves to the first one listed, unlike a prefix two
// trackers claim, which resolves to neither. The difference is what the
// duplicate means: two trackers claiming one prefix own two different id spaces,
// while the same memory slug in two trackers is normally one memory that was
// re-filed and left a copy behind. The caller lists the page's own database
// first, so this is "the tracker you are reading wins".
func TestPrefixesAcrossResolvesADuplicateSlugToTheFirstDatabase(t *testing.T) {
	in := &readyInstance{dbs: map[int]*fakeReadyDB{
		1: memoryTracker("h1", "here", "shared-note"),
		2: memoryTracker("h2", "there", "shared-note"),
	}}
	dbs := []ReadyDatabase{
		{ID: 1, OwnerName: "alice", Name: "own"},
		{ID: 2, OwnerName: "bob", Name: "other"},
	}

	index := PrefixesAcross(t.Context(), dbs, in.open, &PrefixCache{}, readyNow)

	d, ok := index.LookupMemory("shared-note")
	require.True(t, ok, "a slug two trackers hold is still a link")
	assert.Equal(t, "alice/own", d.Slug())
}

// A tracker that names no issue prefix still contributes its memories: the two
// halves of the projection do not gate each other.
func TestPrefixesAcrossIndexesMemoriesOfAPrefixlessDatabase(t *testing.T) {
	in := &readyInstance{dbs: map[int]*fakeReadyDB{
		1: memoryTracker("h1", "", "only-a-memory"),
	}}
	dbs := []ReadyDatabase{{ID: 1, OwnerName: "alice", Name: "no-prefix"}}

	index := PrefixesAcross(t.Context(), dbs, in.open, &PrefixCache{}, readyNow)

	assert.Empty(t, index.Prefixes())
	d, ok := index.LookupMemory("only-a-memory")
	require.True(t, ok)
	assert.Equal(t, "alice/no-prefix", d.Slug())
}

// The cached projection carries the slugs, so a second build behind the
// head-hash gate answers the memory question without reading a row either.
func TestPrefixesAcrossCachesTheSlugs(t *testing.T) {
	in := &readyInstance{dbs: map[int]*fakeReadyDB{
		1: memoryTracker("h1", "here", "cached-slug"),
	}}
	dbs := []ReadyDatabase{{ID: 1, OwnerName: "alice", Name: "own"}}
	cache := &PrefixCache{}

	PrefixesAcross(t.Context(), dbs, in.open, cache, readyNow)
	reads := in.dbs[1].rowReads
	require.Greater(t, reads, 0)

	second := PrefixesAcross(t.Context(), dbs, in.open, cache, readyNow.Add(time.Second))

	assert.Equal(t, reads, in.dbs[1].rowReads, "the gate holds for the slugs too")
	_, ok := second.LookupMemory("cached-slug")
	assert.True(t, ok)
}

// An index that was never built holds no memory either.
func TestNilPrefixIndexKnowsNoMemory(t *testing.T) {
	var index *PrefixIndex

	assert.Nil(t, index.MemorySlugs())
	_, ok := index.LookupMemory("anything")
	assert.False(t, ok)
}


M docs/DESIGN.views.md => docs/DESIGN.views.md +36 -1
@@ 213,6 213,40 @@ Memory · 9 entries                          master · last commit 4 minutes ago
  uplink). Calibrate downloads …
```

### 2.3 The body is markdown — and its references resolve across databases

*Added after the chapters above shipped; the sketch in 2.2 shows the plain-text
bodies this replaces.*

A memory's value is markdown and always was: `bd remember` stores what was
typed, and what is typed is the same prose the memory files carry — bold
leaders, code spans, fenced recipes, numbered steps, and `[[slug]]` references
to related memories. Printing it as pre-wrapped paragraphs shows the source
rather than the document, and leaves every reference a dead end.

- **The renderer** is `web/markdown.go`, one shared `goldmark.Markdown` (GFM,
  no unsafe HTML, no dangerous URL schemes). Three departures from stock:
  - `[[slug]]` is an **inline parser**, not a text rewrite — which is what makes
    a reference written inside a code span stay text by construction.
  - **Raw HTML is escaped and shown, not dropped.** goldmark's safe default
    omits it, and these memories are full of `<placeholder>` spellings —
    `SRHT_<NAME>_VER` — that CommonMark reads as tags. Omitting one rewrites the
    sentence silently, which is worse than showing markup.
  - **Images render as links.** An `<img>` at another host is a request that
    host makes on behalf of whoever opened the page.
- **Issue ids in the prose** link too, through an AST transformer over the text
  nodes — skipping code spans, links and wikilinks — and through ch. 5's index.
- **The index** is ch. 5's, extended: `issue_prefix` and the `kv.memory.*` keys
  are rows of the same `config` table, so one read per database answers both
  questions and one cached projection carries both. Unlike a prefix two trackers
  both claim, a slug two trackers both hold resolves to the first listed — the
  page's own database — because the same slug in two trackers is normally one
  memory that was re-filed and left a copy behind, not two different things.
- **What an unresolved reference looks like** is one rendering for two cases: a
  slug nobody wrote, and a slug held by a database this caller may not browse.
  A link, a distinct class or a different tooltip would each publish the
  existence of a private database.

## 3. Freshness in the header

Nothing on the board says how fresh it is. `bd` pushes with a 30-second debounce


@@ 283,7 317,8 @@ whose prefix belongs to a database on this instance:
- **The prefix index.** Every beads database stores its own prefix in
  `config` under `issue_prefix` (verified: `global`, `artifacts`, …). Build an
  index prefix → repository, warmed lazily and refreshed on the same head-hash /
  TTL basis as ch. 4's cache, over the databases the *caller* may browse. A
  TTL basis as ch. 4's cache, over the databases the *caller* may browse. (It
  carries the memory slugs of the same `config` read as well; see §2.3.) A
  prefix belonging to a database the caller cannot see is not linked, and the
  page must not reveal that it exists.
- **The pattern.** `<prefix>-<suffix>` where prefix is a known one and suffix is


M go.mod => go.mod +1 -0
@@ 15,6 15,7 @@ require (
	github.com/stretchr/testify v1.11.1
	github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec
	github.com/vektah/gqlparser/v2 v2.5.36
	github.com/yuin/goldmark v1.8.2
	go.bigb.es/auxilia v0.7.0
	google.golang.org/grpc v1.79.3
	gopkg.in/go-jose/go-jose.v2 v2.6.3


M go.sum => go.sum +2 -0
@@ 474,6 474,8 @@ github.com/xtaci/smux v1.5.56/go.mod h1:IGQ9QYrBphmb/4aTnLEcJby0TNr3NV+OslIOMrX8
github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE=
github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg=


M web/beads.go => web/beads.go +47 -28
@@ 126,18 126,12 @@ func beadIssueHref(owner, name, id string) string {
// Building the index opens a store per database, so it is asked for by the one
// rendering that needs it and never as part of the envelope.
//
// The visibility rule is the /ready page's, for the same reason: enumerate with
// ListReposForViewer (the listing rule), then ask core.Allowed/OpBrowse per
// database (the access rule), and do both before a single store is opened. A
// database this caller may not browse never enters the index, so an id belonging
// to it renders as plain text — indistinguishable from one whose prefix matches
// nothing. That indistinguishability is the point: a tooltip, a class or an
// "unknown tracker" marker would each publish the existence of a database this
// caller is not allowed to know about.
//
// A listing that fails costs the page its links and not the page. This is
// decoration on top of an answer, exactly as the freshness line is, and it may
// never be the reason a reader gets a 500 instead of an issue.
// Which databases may enter the index is linkableDatabases' answer and not this
// function's. What that leaves here: a database this caller may not browse never
// enters it, so an id belonging to it renders as plain text — indistinguishable
// from one whose prefix matches nothing. That indistinguishability is the point:
// a tooltip, a class or an "unknown tracker" marker would each publish the
// existence of a database this caller is not allowed to know about.
func (a *app) beadCrossLinks(r *http.Request, view View, repo *core.Repo, data any) *beadLinks {
	bv, ok := view.(*beadsView)
	if !ok {


@@ 148,12 142,46 @@ func (a *app) beadCrossLinks(r *http.Request, view View, repo *core.Repo, data a
		return nil
	}

	dbs, open, ok := a.linkableDatabases(r, repo)
	if !ok {
		return nil
	}

	index := beads.PrefixesAcross(r.Context(), dbs, open, &bv.prefixes, timeNow())

	// A store that cannot be read is a fact about this deployment and belongs in
	// the log with its error. The page says nothing at all: the ids that database
	// owns simply stay text.
	for _, f := range index.Failed {
		slog.Warn("reading a database for the issue link index failed",
			"component", "web", "database", f.Database.Slug(), scribe.Err(f.Err))
	}
	return &beadLinks{index: index}
}

// linkableDatabases is the set of databases a cross-database link index may be
// built over for this request, with the opener that reaches them. Both the issue
// links and the memory links ask for it, and they must ask the same way: the
// index decides what a reader is shown, so the rule for what may enter it lives
// in one place rather than in each caller.
//
// The rule is /ready's. Enumerate with ListReposForViewer (the listing rule),
// then ask core.Allowed/OpBrowse per database (the access rule), and do both
// before a single store is opened.
//
// The false return is a listing that failed, and it costs the page its links and
// not the page: this is decoration on top of an answer, exactly as the freshness
// line is, and it may never be the reason a reader gets a 500.
func (a *app) linkableDatabases(
	r *http.Request,
	repo *core.Repo,
) ([]beads.ReadyDatabase, beads.ReadyOpener, bool) {
	_, caller := callerOf(r.Context())
	repos, err := a.cfg.Repos.ListReposForViewer(r.Context(), caller)
	if err != nil {
		slog.Error("listing databases for the issue link index failed",
		slog.Error("listing databases for the cross-database link index failed",
			"component", "web", scribe.Err(err))
		return nil
		return nil, nil, false
	}

	// The disk path never reaches beads: it is this service's arrangement of its


@@ 168,9 196,10 @@ func (a *app) beadCrossLinks(r *http.Request, view View, repo *core.Repo, data a
		paths[cand.ID] = cand.Path
		entry := beads.ReadyDatabase{ID: cand.ID, OwnerName: cand.OwnerName, Name: cand.Name}
		if cand.ID == repo.ID {
			// The database whose page this is goes first, so the one prefix the
			// page cannot do without — its own, the ids that used to link and must
			// keep linking — can never be the one the ceiling drops.
			// The database whose page this is goes first. It is the one the ceiling
			// may never drop — its own ids have always linked and must keep
			// linking — and it is what makes a [[slug]] this tracker holds resolve
			// here rather than in whichever other tracker also carries a copy.
			dbs = append([]beads.ReadyDatabase{entry}, dbs...)
			continue
		}


@@ 184,15 213,5 @@ func (a *app) beadCrossLinks(r *http.Request, view View, repo *core.Repo, data a
		}
		return a.cfg.Browse.Open(ctx, path)
	}

	index := beads.PrefixesAcross(r.Context(), dbs, open, &bv.prefixes, timeNow())

	// A store that cannot be read is a fact about this deployment and belongs in
	// the log with its error. The page says nothing at all: the ids that database
	// owns simply stay text.
	for _, f := range index.Failed {
		slog.Warn("reading a database for the issue link index failed",
			"component", "web", "database", f.Database.Slug(), scribe.Err(f.Err))
	}
	return &beadLinks{index: index}
	return dbs, open, true
}


M web/handlers_view.go => web/handlers_view.go +8 -0
@@ 93,6 93,13 @@ func (a *app) handleView(w http.ResponseWriter, r *http.Request) {
		// building it opens a store per database the caller may browse. A nil one
		// still renders text; see beadLinks.Text.
		Links *beadLinks
		// Memories renders a memory's stored markdown, with its [[slug]]
		// references and the issue ids in its prose linked to the databases that
		// hold them. It is built by the memory view alone, for the reason Links is
		// built by the detail pane alone: it opens a store per database this
		// caller may browse. A nil one still renders the markdown, with every
		// reference left as text; see memoryLinks.Body.
		Memories *memoryLinks
	}{
		Page:     a.page(r, view.Label()+" — "+repo.OwnerName+"/"+repo.Name),
		Repo:     repo,


@@ 102,6 109,7 @@ func (a *app) handleView(w http.ResponseWriter, r *http.Request) {
		Head:     headCommit(r.Context(), sess, ref),
		Data:     data,
		Links:    a.beadCrossLinks(r, view, repo, data),
		Memories: a.memoryCrossLinks(r, view, repo),
	}
	a.render(w, http.StatusOK, pageName(view.Template()), envelope)
}


A web/markdown.go => web/markdown.go +385 -0
@@ 0,0 1,385 @@
package web

import (
	"bytes"
	"html/template"
	"net/url"

	"github.com/yuin/goldmark"
	"github.com/yuin/goldmark/ast"
	"github.com/yuin/goldmark/extension"
	"github.com/yuin/goldmark/parser"
	"github.com/yuin/goldmark/renderer"
	"github.com/yuin/goldmark/renderer/html"
	"github.com/yuin/goldmark/text"
	"github.com/yuin/goldmark/util"

	"sourcecraft.dev/bigbes/sr-ht-dolt/beads"
)

// --- memories, rendered as the markdown they are -------------------------------
//
// A memory's value is markdown and always was: `bd remember` stores what was
// typed, and what is typed is the same prose the memory files carry — bold
// leaders, code spans, fenced blocks, numbered steps. The view used to print it
// as pre-wrapped paragraphs, which is readable but is the source and not the
// document: a bullet list stays a line starting with a hyphen, a recipe stays
// four spaces of indent, and `**Why:**` keeps its asterisks.
//
// Three things this rendering does that a stock markdown filter would not:
//
//  1. `[[slug]]` becomes a link to the memory it names, in whichever database
//     holds it (see the wikilink parser below). That is the whole point of the
//     notation, and memories reference each other across trackers constantly:
//     the mirroring workflow files a memory by its type, so a related memory is
//     as likely to be in another tracker as in this one.
//  2. An issue id in the prose links to the issue, through the same
//     cross-database index the beads detail pane uses.
//  3. Raw HTML is *escaped and shown*, not dropped. goldmark's safe default
//     omits it, and the memory corpus is full of `<placeholder>` spellings —
//     `SRHT_<NAME>_VER`, `~/data/home/<repo>` — that CommonMark reads as tags.
//     Omitting them would silently rewrite `SRHT_<NAME>_VER` to `SRHT__VER`,
//     which is worse than showing markup: it is showing a different fact.
//
// Safety is goldmark's default posture, kept: no unsafe HTML, no dangerous URL
// schemes in links, everything that came out of the database escaped on its way
// to the browser. The one thing marked template.HTML is the finished document
// this file produced.

// memoryLinkIndexKey carries the per-request link index into the parse. A
// goldmark.Markdown is stateless and shared; what varies per request is which
// databases this caller may browse, and that belongs in the parse context rather
// than in a second renderer built per page.
var memoryLinkIndexKey = parser.NewContextKey()

// memoryMarkdown is the shared renderer. goldmark's own parsers and renderers
// are safe for concurrent use — all per-conversion state lives in the context —
// so this is built once and never rebuilt.
var memoryMarkdown = goldmark.New(
	// GFM for the shapes the memories actually use: tables, strikethrough, task
	// lists, and linkify — a bare https://dolt.srht.bigb.es/~bigbes/<repo> in the
	// prose is a URL the reader wants to follow.
	goldmark.WithExtensions(extension.GFM),
	goldmark.WithParserOptions(
		// Ahead of the link parser (200), behind the task-list marker (0): "[[" is
		// a wikilink before it is a link label. Registered as an inline parser and
		// not as a text rewrite, so a "[[slug]]" written inside a code span is
		// left alone by construction — inline parsers do not run in there.
		parser.WithInlineParsers(util.Prioritized(wikilinkParser{}, 150)),
		parser.WithASTTransformers(util.Prioritized(issueLinkTransformer{}, 900)),
	),
	goldmark.WithRendererOptions(
		renderer.WithNodeRenderers(util.Prioritized(memoryNodeRenderer{}, 100)),
	),
)

// memoryLinks renders memory bodies for one request. It holds the link index —
// prefixes and memory slugs over the databases this caller may browse — and
// nothing else; a nil one still renders markdown, with every reference left as
// text, which is a whole answer and not a degraded one.
type memoryLinks struct {
	index *beads.PrefixIndex
}

// Body renders one memory's stored text as HTML.
//
// It is the only function here that produces template.HTML, and what it marks is
// the document goldmark built: every leaf that came out of the database is
// escaped by the renderer on its way in, including the slug inside a wikilink
// and the href built from it.
func (l *memoryLinks) Body(src string) template.HTML {
	var index *beads.PrefixIndex
	if l != nil {
		index = l.index
	}
	pc := parser.NewContext()
	if index != nil {
		pc.Set(memoryLinkIndexKey, index)
	}
	var buf bytes.Buffer
	if err := memoryMarkdown.Convert([]byte(src), &buf, parser.WithContext(pc)); err != nil {
		// Convert fails only on a write, which this buffer cannot do. The memory is
		// still shown, as the escaped text it was: a rendering that could not run is
		// not a reason to answer with a blank pane.
		return template.HTML(`<p class="mem-raw">` +
			template.HTMLEscapeString(src) + `</p>`)
	}
	return template.HTML(buf.String())
}

// memoryHref is the address of one memory in one database: the memory view
// narrowed to a single slug, which is the page a reference wants to land on.
func memoryHref(owner, name, slug string) string {
	return "/~" + url.PathEscape(owner) + "/" + url.PathEscape(name) +
		"/view/memory?key=" + url.QueryEscape(slug)
}

// --- [[slug]] ------------------------------------------------------------------

// wikilink is a resolved or unresolved memory reference. Href is empty when no
// database this caller may browse holds a memory under that slug — which is
// deliberately the same node as one nobody ever wrote, so the rendering cannot
// disclose the existence of a database the caller may not see.
type wikilink struct {
	ast.BaseInline
	Href string
}

var kindWikilink = ast.NewNodeKind("MemoryWikilink")

func (n *wikilink) Kind() ast.NodeKind { return kindWikilink }

func (n *wikilink) Dump(source []byte, level int) {
	ast.DumpHelper(n, source, level, map[string]string{"Href": n.Href}, nil)
}

// wikilinkParser turns "[[slug]]" into a wikilink node, resolving the slug
// against the request's index as it goes.
type wikilinkParser struct{}

func (wikilinkParser) Trigger() []byte { return []byte{'['} }

// Parse reads a wikilink out of the current line, or nothing at all: a "[[" with
// no closing "]]" on the same line, and anything whose slug is not slug-shaped,
// is left to the ordinary link parser and ends up as the text it was. Memory
// slugs are single-token keys — `bd remember --key` takes one — so a reference
// never spans a line.
func (wikilinkParser) Parse(_ ast.Node, block text.Reader, pc parser.Context) ast.Node {
	line, _ := block.PeekLine()
	if len(line) < 5 || line[0] != '[' || line[1] != '[' {
		return nil
	}
	end := bytes.Index(line, []byte("]]"))
	if end < 3 {
		return nil
	}
	slug := string(line[2:end])
	if !isMemorySlug(slug) {
		return nil
	}
	block.Advance(end + 2)

	node := &wikilink{}
	if index, ok := pc.Get(memoryLinkIndexKey).(*beads.PrefixIndex); ok {
		if d, found := index.LookupMemory(slug); found {
			node.Href = memoryHref(d.OwnerName, d.Name, slug)
		}
	}
	// The slug is carried as a string node rather than as a source segment: a
	// line the reader hands back can be padded — a list item's continuation
	// indent is synthesised, not sliced — and an offset into it is then not an
	// offset into the source. The renderer escapes a string node exactly as it
	// escapes every other leaf.
	node.AppendChild(node, ast.NewString([]byte(slug)))
	return node
}

// memorySlugMax bounds what this will treat as a slug. `bd remember --key` takes
// a short name; a "[[" followed by half a paragraph and a "]]" is prose that
// happens to contain brackets.
const memorySlugMax = 128

// isMemorySlug is the shape a memory key has: the characters `bd remember --key`
// and the memory files' `name:` field use, and no others. It is deliberately
// narrower than "anything without brackets" — a bracketed aside is not a
// reference, and the difference has to be decidable without asking the index,
// since a slug nobody holds must render the same way whether or not it is one.
func isMemorySlug(s string) bool {
	if s == "" || len(s) > memorySlugMax {
		return false
	}
	for _, r := range s {
		switch {
		case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
		case r == '-', r == '_', r == '.', r == '/':
		default:
			return false
		}
	}
	return true
}

// --- issue ids in memory prose -------------------------------------------------

// issueLinkTransformer links the issue ids in a memory's prose to the databases
// that own them, reusing the index this render already built for the wikilinks.
//
// It runs after inline parsing, over the text nodes only, and never descends
// into a code span, a link, an autolink or a wikilink: an id inside `code` is
// being shown rather than cited, and an id inside a link label would nest an
// anchor in an anchor.
type issueLinkTransformer struct{}

func (issueLinkTransformer) Transform(doc *ast.Document, reader text.Reader, pc parser.Context) {
	index, ok := pc.Get(memoryLinkIndexKey).(*beads.PrefixIndex)
	if !ok || index == nil {
		return
	}
	source := reader.Source()

	// Collected first and rewritten after: replacing a node during the walk that
	// found it is how a walk starts stepping over its own edits.
	var texts []*ast.Text
	_ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
		if !entering {
			return ast.WalkContinue, nil
		}
		switch n.Kind() {
		case ast.KindLink, ast.KindImage, ast.KindAutoLink, ast.KindCodeSpan,
			ast.KindRawHTML, ast.KindHTMLBlock, ast.KindCodeBlock,
			ast.KindFencedCodeBlock, kindWikilink:
			return ast.WalkSkipChildren, nil
		case ast.KindText:
			t := n.(*ast.Text)
			// A raw text node is a code span's content, and a padded one carries a
			// block indent its segment offsets do not describe. Neither can be cut
			// on byte offsets taken from the source.
			if !t.IsRaw() && t.Segment.Padding == 0 {
				texts = append(texts, t)
			}
		}
		return ast.WalkContinue, nil
	})

	for _, t := range texts {
		linkIssueIDs(t, source, index)
	}
}

// linkIssueIDs replaces one text node with the sequence of text and link nodes
// its ids imply. A node with no id in it is left exactly as it was.
func linkIssueIDs(t *ast.Text, source []byte, index *beads.PrefixIndex) {
	seg := t.Segment
	refs := index.Scan(string(source[seg.Start:seg.Stop]))
	if len(refs) == 0 {
		return
	}
	parent := t.Parent()
	if parent == nil {
		return
	}

	var nodes []ast.Node
	last := seg.Start
	for _, ref := range refs {
		start, stop := seg.Start+ref.Start, seg.Start+ref.End
		if start > last {
			nodes = append(nodes, ast.NewTextSegment(text.NewSegment(last, start)))
		}
		link := ast.NewLink()
		link.Destination = []byte(beadIssueHref(
			ref.Database.OwnerName, ref.Database.Name, ref.ID))
		link.AppendChild(link, ast.NewTextSegment(text.NewSegment(start, stop)))
		nodes = append(nodes, link)
		last = stop
	}

	// The tail carries the original node's line-break flags. When the id ran to
	// the end of the node the tail is empty and is kept anyway: dropping it drops
	// the newline, and the next line's first word would be glued to the id.
	tail := ast.NewTextSegment(text.NewSegment(last, seg.Stop))
	tail.SetSoftLineBreak(t.SoftLineBreak())
	tail.SetHardLineBreak(t.HardLineBreak())
	nodes = append(nodes, tail)

	for _, n := range nodes {
		parent.InsertBefore(parent, t, n)
	}
	parent.RemoveChild(parent, t)
}

// --- the three nodes this rendering does not leave to goldmark -----------------

// memoryNodeRenderer registers the wikilink renderer and replaces goldmark's
// handling of raw HTML.
type memoryNodeRenderer struct{}

func (memoryNodeRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) {
	reg.Register(kindWikilink, renderWikilink)
	reg.Register(ast.KindRawHTML, renderRawHTMLAsText)
	reg.Register(ast.KindHTMLBlock, renderHTMLBlockAsText)
	reg.Register(ast.KindImage, renderImageAsLink)
}

// renderWikilink writes the anchor, or the muted marker for a slug no database
// this caller may browse holds. The marker says only that the reference does not
// resolve *here*; it cannot say more without disclosing what it must not.
func renderWikilink(w util.BufWriter, _ []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
	n := node.(*wikilink)
	switch {
	case entering && n.Href != "":
		_, _ = w.WriteString(`<a class="mem-link" href="`)
		_, _ = w.Write(util.EscapeHTML(util.URLEscape([]byte(n.Href), true)))
		_, _ = w.WriteString(`">`)
	case entering:
		_, _ = w.WriteString(`<span class="mem-link-out" ` +
			`title="No memory with this slug in a tracker you can browse.">`)
	case n.Href != "":
		_, _ = w.WriteString(`</a>`)
	default:
		_, _ = w.WriteString(`</span>`)
	}
	return ast.WalkContinue, nil
}

// renderImageAsLink renders an image reference as a link to it rather than as an
// <img>.
//
// An <img> pointing at another host is a request that host makes on behalf of
// whoever opened the page, and a memory is prose one account wrote and another
// may read: an image in it would report the reader's address to a server the
// reader never chose to contact. The reference is kept and stays followable; it
// simply is not fetched by opening the page.
func renderImageAsLink(w util.BufWriter, _ []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
	n := node.(*ast.Image)
	if !entering {
		_, _ = w.WriteString(`</a>`)
		return ast.WalkContinue, nil
	}
	_, _ = w.WriteString(`<a class="mem-img" href="`)
	if dest := util.URLEscape(n.Destination, true); !html.IsDangerousURL(dest) {
		_, _ = w.Write(util.EscapeHTML(dest))
	}
	// The alt text is the anchor's text, which is what the children already
	// render as — an image with no alt text is left as a bare link.
	_, _ = w.WriteString(`">`)
	return ast.WalkContinue, nil
}

// renderRawHTMLAsText writes inline raw HTML as the escaped text it reads as.
//
// goldmark's safe default omits it, which is right for a comment feed and wrong
// here: `SRHT_<NAME>_VER` is a placeholder somebody typed, CommonMark sees
// `<NAME>` as a tag, and omitting it turns the memory into a different sentence.
// Escaping shows what was written and is exactly as safe as omitting it.
func renderRawHTMLAsText(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
	if !entering {
		return ast.WalkSkipChildren, nil
	}
	n := node.(*ast.RawHTML)
	for i := 0; i < n.Segments.Len(); i++ {
		seg := n.Segments.At(i)
		_, _ = w.Write(util.EscapeHTML(seg.Value(source)))
	}
	return ast.WalkSkipChildren, nil
}

// renderHTMLBlockAsText is the same treatment for a block that opened with
// something tag-shaped: shown, escaped, in a paragraph of its own rather than
// dropped.
func renderHTMLBlockAsText(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
	n := node.(*ast.HTMLBlock)
	if !entering {
		if n.HasClosure() {
			_, _ = w.Write(util.EscapeHTML(n.ClosureLine.Value(source)))
		}
		_, _ = w.WriteString("</p>\n")
		return ast.WalkContinue, nil
	}
	_, _ = w.WriteString(`<p class="mem-raw">`)
	for i := 0; i < n.Lines().Len(); i++ {
		line := n.Lines().At(i)
		_, _ = w.Write(util.EscapeHTML(line.Value(source)))
	}
	return ast.WalkContinue, nil
}


A web/markdown_test.go => web/markdown_test.go +282 -0
@@ 0,0 1,282 @@
package web

import (
	"errors"
	"net/http"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"sourcecraft.dev/bigbes/sr-ht-dolt/browse"
	"sourcecraft.dev/bigbes/sr-ht-dolt/core"
)

// --- the rendering itself ------------------------------------------------------
//
// These render with no index at all — the nil receiver, which is what a page
// gets when the database listing failed. Everything asserted here is therefore
// about the markdown and about what is allowed to reach the browser, not about
// the links.

// renderMemory is one memory body, rendered by a memoryLinks that knows no
// database.
func renderMemory(src string) string {
	var noLinks *memoryLinks
	return string(noLinks.Body(src))
}

// The shapes the memories are actually written in: a bold leader, code spans, a
// bullet list, a fenced recipe, and hard-wrapped prose that reflows into one
// paragraph rather than keeping the width its author's terminal had.
func TestMemoryBodyRendersMarkdown(t *testing.T) {
	got := renderMemory("**Why:** the networks are declared `external: true` everywhere,\n" +
		"and compose cannot create one.\n" +
		"\n" +
		"- first, `./prepare.sh`\n" +
		"- then `labng push backup`\n" +
		"\n" +
		"```\n" +
		"dolt clone https://dolt.srht.bigb.es/~bigbes/x\n" +
		"```\n")

	assert.Contains(t, got, "<strong>Why:</strong>")
	assert.Contains(t, got, "<code>external: true</code>")
	assert.Contains(t, got, "<ul>")
	assert.Contains(t, got, "<li>first, <code>./prepare.sh</code></li>")
	assert.Contains(t, got, "<pre><code>dolt clone")
	assert.Contains(t, got, "everywhere,\nand compose cannot create one.",
		"a hard-wrapped paragraph reflows: the source's line breaks are not the document's")
	assert.NotContains(t, got, "**", "the asterisks are markup and must not survive as text")
}

// A memory is stored text and is escaped on its way out, exactly as it was when
// the page printed paragraphs. Markup written in one renders as the characters
// somebody typed and never as an element.
func TestMemoryBodyEscapesStoredMarkup(t *testing.T) {
	got := renderMemory("Supersedes the memory that asserted <b>otherwise</b>.\n" +
		"\n" +
		"<script>alert('x')</script>\n")

	assert.NotContains(t, got, "<b>otherwise</b>")
	assert.Contains(t, got, "&lt;b&gt;otherwise&lt;/b&gt;")
	assert.NotContains(t, got, "<script>")
	assert.Contains(t, got, "&lt;script&gt;")
}

// Raw markup is shown rather than dropped, which is what makes a placeholder
// survive: goldmark's safe default omits `<NAME>` as a tag, and the memory then
// says "SRHT__VER" — a different fact, silently.
func TestMemoryBodyKeepsPlaceholdersThatLookLikeTags(t *testing.T) {
	got := renderMemory("Bump SRHT_<NAME>_VER in phoebe-lab/srht/versions.env.\n")

	assert.Contains(t, got, "SRHT_&lt;NAME&gt;_VER")
	assert.NotContains(t, got, "raw HTML omitted")
	assert.NotContains(t, got, "SRHT__VER", "dropping the tag would rewrite the sentence")
}

// A link whose scheme is a script is written with no destination at all, and an
// image is a link rather than a fetch: opening a memory may not make a request
// to a host the reader never chose to contact.
func TestMemoryBodyRefusesDangerousLinksAndRemoteImages(t *testing.T) {
	got := renderMemory("[click](javascript:alert(1)) and ![a pixel](https://tracker.example/p.png)\n")

	assert.NotContains(t, got, "javascript:")
	assert.NotContains(t, got, "<img", "an image reference is not fetched by opening the page")
	assert.Contains(t, got, `<a class="mem-img" href="https://tracker.example/p.png">a pixel</a>`)
}

// A bare URL in the prose is a link the reader can follow.
func TestMemoryBodyLinkifiesBareURLs(t *testing.T) {
	got := renderMemory("Clone from https://dolt.srht.bigb.es/~bigbes/x to check.\n")
	assert.Contains(t, got, `<a href="https://dolt.srht.bigb.es/~bigbes/x">`)
}

// A [[slug]] no database holds is muted and inert — and so is one held by a
// database this reader may not browse, which is the point: the two must be one
// rendering.
func TestMemoryBodyLeavesAnUnresolvedWikilinkInert(t *testing.T) {
	got := renderMemory("Related: [[go-vcs-stamp-dirty]] — the stamp.\n")

	assert.Contains(t, got, `<span class="mem-link-out"`)
	assert.Contains(t, got, ">go-vcs-stamp-dirty</span>")
	assert.NotContains(t, got, "<a", "an unresolved reference is not a link")
	assert.NotContains(t, got, "[[", "the brackets are notation and are consumed")
}

// Bracket notation that is not a reference is left as the text it is: a
// wikilink is a slug, and prose in brackets is prose.
func TestMemoryBodyIgnoresBracketsThatAreNotSlugs(t *testing.T) {
	for _, src := range []string{
		"An aside [[with words in it]] mid-sentence.\n",
		"An unclosed [[reference that never ends.\n",
		"An empty [[]] pair.\n",
	} {
		got := renderMemory(src)
		assert.NotContains(t, got, "mem-link", "%q must not become a reference", src)
		assert.Contains(t, got, "[[", "%q keeps its brackets as text", src)
	}
}

// A reference inside a code span is being shown, not made — the notation is
// what the memory is talking about. Inline parsers do not run inside a code
// span, which is why this is a property of the design rather than a special
// case.
func TestMemoryBodyDoesNotResolveInsideCodeSpans(t *testing.T) {
	got := renderMemory("Link memories with `[[their-name]]` in the body.\n")

	assert.Contains(t, got, "<code>[[their-name]]</code>")
	assert.NotContains(t, got, "mem-link")
}

// --- the links, across databases ----------------------------------------------

// memoryLinkTracker is a tracker as the index reads one: a config naming its
// issue prefix and holding memories. The memories' own bodies matter only for
// the database whose page is rendered; the rest are read for their slugs.
func memoryLinkTracker(head, prefix string, memories ...[2]string) *fakeSession {
	rows := [][]string{{"compact_tier2_days", "30"}, {"issue_prefix", prefix}}
	for _, m := range memories {
		rows = append(rows, []string{"kv.memory." + m[0], m[1]})
	}
	return &fakeSession{
		branches: []browse.Branch{{Name: "main", Head: head}},
		tables:   append(beadsTables(), memoryConfigTable()),
		rowsByTable: map[string]*browse.RowPage{
			"issues":       {Columns: []string{"id", "title", "status"}, Total: 0},
			"dependencies": {Columns: []string{"id", "issue_id", "depends_on_issue_id", "type"}, Total: 0},
			"config":       {Columns: []string{"key", "value"}, Rows: rows, Total: len(rows)},
		},
	}
}

// memoryLinkHarness is the instance these tests read: alice/alpha, whose memory
// page is rendered, a second public tracker (bob/beta) holding a memory alpha's
// prose references, and a PRIVATE one (dave/secrets) holding another. Both alice
// and bob carry a memory under the same slug, which is how the tie is checked.
func memoryLinkHarness(t *testing.T) *harness {
	t.Helper()
	h := newHarness(t)

	body := "See [[gitignore-home]] for the ignore rules, and [[shared-note]] " +
		"for the tie.\n" +
		"\n" +
		"Unknown: [[nowhere-at-all]]. Private: [[secret-note]].\n" +
		"\n" +
		"- a step\n" +
		"  - a nested one, see [[gitignore-home]]\n" +
		"\n" +
		"Filed as beta-46c.2, unlike `beta-nex` which is only quoted.\n" +
		"Superseded by beta-46c.3\nand then reopened.\n"

	addTracker(h, "alice", 1, "alpha", core.VisibilityPublic,
		memoryLinkTracker("h-alpha", "alpha",
			[2]string{"the-page", body},
			[2]string{"shared-note", "alice's copy"}))
	addTracker(h, "bob", 2, "beta", core.VisibilityPublic,
		memoryLinkTracker("h-beta", "beta",
			[2]string{"gitignore-home", "bob's memory"},
			[2]string{"shared-note", "bob's copy"}))
	addTracker(h, "dave", 9, "secrets", core.VisibilityPrivate,
		memoryLinkTracker("h-secret", "secret",
			[2]string{"secret-note", "not for you"}))

	setViews(t, h, &memoryView{})
	return h
}

// The whole point of the notation: a [[slug]] resolves to whichever database
// holds that memory, and a memory in another tracker is as ordinary a target as
// one in this tracker.
func TestMemoryWikilinksResolveAcrossDatabases(t *testing.T) {
	pinClock(t, testNow)
	h := memoryLinkHarness(t)

	rec := h.do("GET", "/~alice/alpha/view/memory?key=the-page", nil, nil)
	require.Equal(t, http.StatusOK, rec.Code, "memory view: %s", rec.Body.String())
	body := rec.Body.String()

	// The sibling database's memory.
	assert.Contains(t, body,
		`<a class="mem-link" href="/~bob/beta/view/memory?key=gitignore-home">gitignore-home</a>`)

	// A slug both trackers hold resolves here: the page's own database is asked
	// first, and a memory mirrored into two trackers is one memory.
	assert.Contains(t, body,
		`<a class="mem-link" href="/~alice/alpha/view/memory?key=shared-note">shared-note</a>`)
	assert.NotContains(t, body, "/~bob/beta/view/memory?key=shared-note")

	// A slug nobody holds, and a slug held by a database this caller may not
	// browse, are the same rendering. The second is the load-bearing one: a link,
	// a distinct class or a different tooltip would each publish the existence of
	// a private database.
	assert.Contains(t, body, `<span class="mem-link-out" title="No memory with this slug in a tracker you can browse.">nowhere-at-all</span>`)
	assert.Contains(t, body, `<span class="mem-link-out" title="No memory with this slug in a tracker you can browse.">secret-note</span>`)
	assert.NotContains(t, body, "/~dave/secrets/")

	// And inside a nested list item, whose lines the parser hands back with a
	// synthesised indent: an offset into such a line is not an offset into the
	// source, and a reference built from one would carry the wrong slug.
	assert.Contains(t, body,
		`a nested one, see <a class="mem-link" href="/~bob/beta/view/memory?key=gitignore-home">gitignore-home</a>`)
}

// The prose in a memory names issues too, and it reaches the same index the
// detail pane's ids do — one read of one table per database answers both.
func TestMemoryProseLinksIssueIDs(t *testing.T) {
	pinClock(t, testNow)
	h := memoryLinkHarness(t)

	rec := h.do("GET", "/~alice/alpha/view/memory?key=the-page", nil, nil)
	require.Equal(t, http.StatusOK, rec.Code, "memory view: %s", rec.Body.String())
	body := rec.Body.String()

	assert.Contains(t, body, `<a href="/~bob/beta/view/beads?issue=beta-46c.2">beta-46c.2</a>`)
	// An id inside a code span is being quoted, not cited.
	assert.Contains(t, body, "<code>beta-nex</code>")
	assert.NotContains(t, body, `issue=beta-nex`)

	// An id that ends a line keeps the line break that followed it. Without it
	// the next line's first word is glued to the id — the link would read
	// "beta-46c.3and then reopened".
	assert.Contains(t, body,
		`<a href="/~bob/beta/view/beads?issue=beta-46c.3">beta-46c.3</a>`+"\nand then reopened.")
}

// The index is built once per request and cached per database on the view, the
// same bound the board's link index has. A second request with unmoved heads
// reads no config row for the databases it only consults.
func TestMemoryLinkIndexIsCachedPerDatabase(t *testing.T) {
	pinClock(t, testNow)
	h := memoryLinkHarness(t)

	first := h.do("GET", "/~alice/alpha/view/memory", nil, nil)
	require.Equal(t, http.StatusOK, first.Code)
	beta := h.browse.byPath["/var/lib/dolt/~bob/beta"]
	require.NotNil(t, beta)
	require.Greater(t, beta.rowReads, 0, "the first render must read the sibling's config")
	reads := beta.rowReads

	second := h.do("GET", "/~alice/alpha/view/memory", nil, nil)
	require.Equal(t, http.StatusOK, second.Code)
	assert.Equal(t, reads, beta.rowReads, "an unmoved head reads no row the second time")
	assert.Contains(t, second.Body.String(), "gitignore-home",
		"and the cached index still resolves the reference")
}

// A memory page renders whether or not the index could be built: the links are
// decoration on top of an answer.
func TestMemoryRendersWithoutAnIndex(t *testing.T) {
	pinClock(t, testNow)
	h := memoryLinkHarness(t)
	h.store.listErr = errors.New("db: list repositories: connection refused")

	rec := h.do("GET", "/~alice/alpha/view/memory?key=the-page", nil, nil)
	require.Equal(t, http.StatusOK, rec.Code, "memory view: %s", rec.Body.String())
	body := rec.Body.String()

	assert.Contains(t, body, "mem-link-out", "every reference is left unresolved")
	assert.NotContains(t, body, `class="mem-link"`)
	assert.True(t, strings.Contains(body, "the-page"), "and the memory itself is still shown")
}


M web/memory.go => web/memory.go +51 -1
@@ 2,8 2,12 @@ package web

import (
	"context"
	"log/slog"
	"net/http"
	"net/url"

	"go.bigb.es/auxilia/scribe"

	"sourcecraft.dev/bigbes/sr-ht-dolt/beads"
	"sourcecraft.dev/bigbes/sr-ht-dolt/browse"
	"sourcecraft.dev/bigbes/sr-ht-dolt/core"


@@ 17,7 21,18 @@ import (
// The reading is not here. It lives in the beads package, which the MCP
// surface's list_memories shares; this type is only the View adapter — slug,
// label, template, and the hand-off of the request's ref and query.
type memoryView struct{}
type memoryView struct {
	// links is the cross-database link index's cache, held for the same reasons
	// beadsView holds one (see web/beads.go): the view is this rendering's one
	// piece of per-process state, a zero value is a working cache, and what is
	// cached is a fact about a database rather than about who may read it.
	//
	// It is a second cache and not the board's, because it is a second page: the
	// two are read at different times and neither should be able to expire the
	// other's entries. What they share is the projection's shape, which is why
	// they share its type.
	links beads.PrefixCache
}

func (*memoryView) Name() string     { return "memory" }
func (*memoryView) Label() string    { return "Memory" }


@@ 43,3 58,38 @@ func (*memoryView) Build(ctx context.Context, sess BrowseSession, _ *core.Repo, 
	}
	return data, nil
}

// memoryCrossLinks builds the link index the memory bodies are rendered against,
// and returns nil for every page that is not the memory view. A nil one still
// renders the markdown; what it costs is the links, which is what an unreadable
// listing may cost and no more (see linkableDatabases).
//
// It is asked for by this page rather than built into the envelope because
// building it opens a store per database the caller may browse — the same reason
// the issue links are asked for by the detail pane alone.
//
// The index answers both halves of what a memory body cites: the [[slug]]
// references and the issue ids in the prose. They come out of one read of one
// table per database, which is why there is one index here and not two.
func (a *app) memoryCrossLinks(r *http.Request, view View, repo *core.Repo) *memoryLinks {
	mv, ok := view.(*memoryView)
	if !ok {
		return nil
	}
	dbs, open, ok := a.linkableDatabases(r, repo)
	if !ok {
		return nil
	}

	index := beads.PrefixesAcross(r.Context(), dbs, open, &mv.links, timeNow())

	// A store that cannot be read is a fact about this deployment and belongs in
	// the log with its error. The page says nothing at all: the references that
	// database holds simply stop resolving, exactly as they do for a database this
	// caller may not browse.
	for _, f := range index.Failed {
		slog.Warn("reading a database for the memory link index failed",
			"component", "web", "database", f.Database.Slug(), scribe.Err(f.Err))
	}
	return &memoryLinks{index: index}
}


M web/templates/memory.html => web/templates/memory.html +61 -7
@@ 49,11 49,63 @@
.mem-rev a { color: var(--bd-muted); }
.mem-rev code { font-size: .72rem; color: inherit; }

/* Paragraphs keep the line breaks their author typed: memories are full of
   bullet lists, and collapsing those into a sentence loses the structure. */
/* The body is rendered markdown (see web/markdown.go), so what is styled here is
   a small document and no longer a run of pre-wrapped paragraphs: lists are
   lists, a fenced recipe is a code block, and `**Why:**` is bold. The type stays
   the page's — flat, square, hairline — and every block loses its last margin so
   the pane closes on the text rather than on a gap. */
.mem-body { padding: .5rem .7rem; }
.mem-body p { white-space: pre-wrap; margin: 0 0 .6rem; }
.mem-body p:last-child { margin-bottom: 0; }
.mem-body > :first-child { margin-top: 0; }
.mem-body > :last-child { margin-bottom: 0; }
.mem-body p { margin: 0 0 .6rem; }

/* Headings inside a memory are subordinate to the slug in the header, so they
   are marked by weight and never by a size that competes with it. */
.mem-body h1, .mem-body h2, .mem-body h3,
.mem-body h4, .mem-body h5, .mem-body h6 {
  font-size: .95rem; font-weight: 700; margin: .9rem 0 .4rem;
}

.mem-body ul, .mem-body ol { margin: 0 0 .6rem; padding-left: 1.3rem; }
.mem-body li { margin-bottom: .2rem; }
.mem-body li > ul, .mem-body li > ol { margin-bottom: 0; }
.mem-body li > p { margin-bottom: .3rem; }

.mem-body code {
  font-size: .82rem; padding: 0 .2rem;
  background: var(--bd-panel); border: 1px solid var(--bd-border);
}
/* A fenced block is one box, so its own code element carries no second border
   inside it. Long recipes scroll on their own rather than widening the pane. */
.mem-body pre {
  margin: 0 0 .6rem; padding: .4rem .55rem; overflow-x: auto;
  background: var(--bd-panel); border: 1px solid var(--bd-border);
}
.mem-body pre code { font-size: .82rem; padding: 0; background: none; border: 0; }

.mem-body blockquote {
  margin: 0 0 .6rem; padding: .1rem .6rem;
  border-left: 2px solid var(--bd-border); color: var(--bd-muted);
}
.mem-body hr { border: 0; border-top: 1px solid var(--bd-border); margin: .8rem 0; }

.mem-body table { border-collapse: collapse; margin: 0 0 .6rem; font-size: .85rem; }
.mem-body th, .mem-body td { border: 1px solid var(--bd-border); padding: .2rem .4rem; text-align: left; }
.mem-body th { background: var(--bd-panel); }

/* Raw markup — a `<placeholder>` CommonMark read as a tag — is shown as the text
   it was, not dropped; it keeps the line breaks it was typed with. */
.mem-body .mem-raw { white-space: pre-wrap; }

/* A [[slug]] that resolves is a link like any other. One that does not is muted
   and inert, and says only that it does not resolve here: whether the memory
   exists in a database this reader may not see is not something this page may
   answer. */
.mem-body .mem-link { font-family: monospace; font-size: .85rem; }
.mem-body .mem-link-out {
  font-family: monospace; font-size: .85rem;
  color: var(--bd-muted); border-bottom: 1px dotted var(--bd-border); cursor: help;
}

.mem-empty { color: var(--bd-muted); font-style: italic; }
</style>


@@ 107,9 159,11 @@
      {{- else}}written before the last {{$.Data.WalkMax}} commits{{end}}
    </span>
  </header>
  <div class="mem-body">
    {{range .Paragraphs}}<p>{{.}}</p>{{end}}
  </div>
  {{/* The body is the stored markdown, rendered. The envelope's .Memories is
       what resolves the [[slug]] references and the issue ids in it against the
       databases this reader may browse; a nil one — a listing that failed —
       still renders the markdown, with those left as text. */}}
  <div class="mem-body">{{$.Memories.Body .Text}}</div>
</article>
{{end}}
{{else if .Data.Key}}