~bigbes/sr-ht-dolt

ref: a41949952f01c47f58cc35fbba02205afd713d41 sr-ht-dolt/beads/memory.go -rw-r--r-- 17.2 KiB
a4194995 — Eugene Blikh mcpsrv: report the projection's clip on get_issue and list_milestones 5 days ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
package beads

import (
	"context"
	"fmt"
	"net/url"
	"regexp"
	"sort"
	"strings"
	"time"

	"sourcecraft.dev/bigbes/sr-ht-dolt/browse"
)

// --- the memories bd remember writes ------------------------------------------

// memoryTable is where bd keeps them: the beads config table, as ordinary
// key/value rows. memoryPrefix marks the keys that are memories; the rest of the
// table is the tracker's settings (issue_prefix, compact_tier2_days, …).
const (
	memoryTable  = "config"
	memoryPrefix = "kv.memory."
)

// memoryWalkMax bounds the revision walk (see walkMemories): at most this many
// commits back from the head of the ref. The most active tracker on this
// instance had 225 commits three weeks in, so this is roughly two months of
// headroom at that rate. A memory not attributed inside the walk carries no date
// at all rather than one the walk cannot support.
const memoryWalkMax = 500

// MemoryStaleAfter is how old a memory has to be before the page questions it.
// It is one constant and not a per-request knob, and what it produces is a
// question ("stale?") rather than a verdict: some memories are meant to be
// permanent, and only the reader knows which.
const MemoryStaleAfter = 60 * 24 * time.Hour

// MemorySession is the seam the memory projection reads through. It is wider
// than BrowseSession because a memory has no timestamp — the config row is
// (key, value) and nothing else — so the date has to come out of the history:
// Log for the commits and TableHash to skip the ones that did not touch config.
// It is still only what this projection calls, and web's BrowseSession and
// mcpsrv's (and *browse.DB itself) satisfy it structurally.
type MemorySession interface {
	BrowseSession
	// Log lists commits from the head of refStr, newest first.
	Log(ctx context.Context, refStr, fromHash string, limit int) ([]browse.CommitInfo, string, error)
	// TableHash is the content hash of a table at a ref, ok=false when the table
	// does not exist there. It reads no rows, which is what makes the walk cheap.
	TableHash(ctx context.Context, refStr, table string) (string, bool, error)
}

// MemoryRevision is when a memory's value last changed: the commit that wrote
// it, taken from the history rather than from the row.
type MemoryRevision struct {
	Commit string
	Date   time.Time
	Author string
}

// Memory is one `bd remember` entry.
type Memory struct {
	Slug string // the config key with the "kv.memory." prefix stripped
	Text string // the value, with both newline spellings normalised
	// Paragraphs is Text split on blank lines. Values are typed into shell
	// strings, so they arrive with a mix of real newlines and literal "\n"
	// escapes; a page that dumped the raw value would render either as one blob.
	Paragraphs []string
	// Revision is the commit that last changed this key's value, or nil when the
	// walk could not reach it (see MemoryView.WalkTruncated).
	Revision *MemoryRevision
	// Stale reports that the memory is older than MemoryStaleAfter. For a memory
	// with no Revision it is set only when the walk's own oldest commit is
	// already past the threshold — that much is known even without a date.
	Stale bool
}

// MemoryView is the opaque .Data handed to memory.html.
type MemoryView struct {
	Memories []Memory
	Total    int    // memories in the tracker, before ?q= / ?key= narrowing
	Search   string // ?q=, sticky form state
	Key      string // ?key=<slug>, a single memory
	Sort     string // MemorySortSlug (default) | MemorySortAge
	// Query is the request's query as parsed, carried so the sort toggle can
	// rebuild this exact URL with one key replaced (web's withQuery) instead of
	// re-listing the parameters it happens to know about.
	Query url.Values
	// WalkTruncated says the revision walk stopped at WalkMax commits with the
	// history still going. It is the only way a Memory can carry no Revision, and
	// it is what the page says instead of a date.
	WalkTruncated bool
	WalkMax       int // memoryWalkMax, so the page can name the number it hit

	// ConfigTruncated says a read of the config table came back clipped at
	// beads.Max. That is a different clip from WalkTruncated, which bounds the
	// history in commits; this one bounds a single read in rows, and it covers
	// both reads this view makes: the one at ref that the memories themselves
	// come from, and the per-commit ones the walk dates them by. A clipped read
	// in the walk leaves the dates unsafe rather than the list short — a key
	// whose row fell past the cap reads as absent there, which is
	// indistinguishable from a key that had not been written yet.
	ConfigTruncated bool
	// ConfigShownOf is the config table's reported total at ref, clipped or not:
	// rows that exist, against the at most Max that were read. It counts the
	// whole table, memory keys and tracker settings alike, because that is what
	// the cap applies to.
	ConfigShownOf int
}

// ConfigClipped reports that the config read the memories themselves come from
// exceeded Max, so Memories and Total cover its first Max rows only and a
// memory may be missing from the list entirely. ConfigTruncated is the wider
// fact (any config read, here or in the walk, was clipped).
func (v *MemoryView) ConfigClipped() bool { return v.ConfigShownOf > Max }

// Memory sort orders. Slug is the default; Age is the review queue.
const (
	MemorySortSlug = "slug"
	MemorySortAge  = "age"
)

// AppliesMemories fingerprints a tracker that can carry memories: the beads
// fingerprint plus a config table with key and value columns. Like every
// Applies, it sees table shapes and never rows, so a tracker whose config holds
// no memory at all still gets the tab and renders an empty state.
func AppliesMemories(tables []browse.TableInfo) bool {
	if !Applies(tables) {
		return false
	}
	for _, t := range tables {
		if t.Name != memoryTable {
			continue
		}
		var haveKey, haveValue bool
		for _, c := range t.Columns {
			switch c.Name {
			case "key":
				haveKey = true
			case "value":
				haveValue = true
			}
		}
		return haveKey && haveValue
	}
	return false
}

// BuildMemories reads the memories at ref and dates each one from the history.
// now is the clock staleness is measured against, passed in rather than read
// here: this package renders nothing and reads no hidden clock, and a caller
// that pins its own clock (the web view, its tests) gets a deterministic answer.
//
// A missing config table degrades to no memories — the same treatment the other
// optional tables get — but a history that cannot be read is an error, not an
// empty answer: the date is what this view is for, and a page that silently
// dropped every date would look exactly like a tracker whose memories are all
// older than the walk.
func BuildMemories(ctx context.Context, sess MemorySession, ref string, query url.Values, now time.Time) (*MemoryView, error) {
	view := &MemoryView{
		Search:  strings.TrimSpace(query.Get("q")),
		Key:     strings.TrimSpace(query.Get("key")),
		Sort:    parseMemorySort(query.Get("sort")),
		Query:   query,
		WalkMax: memoryWalkMax,
	}

	rows, configTotal, err := readRowsOptional(ctx, sess, ref, memoryTable)
	if err != nil {
		return nil, err
	}
	view.ConfigShownOf = configTotal
	view.ConfigTruncated = configTotal > Max
	if rows == nil {
		return view, nil
	}

	// The whole config table, narrowed to the memory keys. raw keeps the stored
	// value under its full key: the walk compares values as they are stored, and
	// normalising first would make two spellings of the same text look equal.
	cols := indexCols(rows.Columns)
	raw := map[string]string{}
	texts := map[string]string{}
	for _, r := range rowsOf(rows) {
		key := cell(cols, r, "key")
		if !strings.HasPrefix(key, memoryPrefix) {
			continue
		}
		slug := strings.TrimPrefix(key, memoryPrefix)
		if slug == "" {
			continue
		}
		raw[key] = cell(cols, r, "value")
		texts[key] = normalizeMemoryText(raw[key])
	}
	view.Total = len(raw)

	// Narrow before walking: the walk is per-key work, and a ?key= page has no
	// business dating the other eight memories.
	tracked := map[string]string{}
	for key, value := range raw {
		slug := strings.TrimPrefix(key, memoryPrefix)
		if view.Key != "" && slug != view.Key {
			continue
		}
		if !matchesMemorySearch(view.Search, slug, texts[key]) {
			continue
		}
		tracked[key] = value
	}
	if len(tracked) == 0 {
		return view, nil
	}

	walk, err := walkMemories(ctx, sess, ref, tracked)
	if err != nil {
		return nil, err
	}
	view.WalkTruncated = walk.truncated
	view.ConfigTruncated = view.ConfigTruncated || walk.configClipped

	view.Memories = make([]Memory, 0, len(tracked))
	for key := range tracked {
		m := Memory{
			Slug:       strings.TrimPrefix(key, memoryPrefix),
			Text:       texts[key],
			Paragraphs: memoryParagraphs(texts[key]),
		}
		if rev, ok := walk.revisions[key]; ok {
			m.Revision = &rev
			m.Stale = now.Sub(rev.Date) > MemoryStaleAfter
		} else if !walk.oldest.IsZero() {
			// No date, but a floor: the memory is at least as old as the oldest
			// commit the walk examined. When that alone is past the threshold the
			// question is supportable; otherwise it is not asked.
			m.Stale = now.Sub(walk.oldest) > MemoryStaleAfter
		}
		view.Memories = append(view.Memories, m)
	}
	sortMemories(view.Memories, view.Sort)

	return view, nil
}

// memoryWalk is what walkMemories learned: the commit each key was last written
// by, whether the walk ran out of budget before the history ran out, and the
// date of the oldest commit it examined.
type memoryWalk struct {
	revisions map[string]MemoryRevision
	truncated bool
	oldest    time.Time
	// configClipped says one of the per-commit config reads exceeded Max. It is
	// a different bound from truncated — rows rather than commits — and it costs
	// the attribution its footing: a key the read never reached looks like a key
	// that commit had not written yet, which is precisely what the comparison
	// below treats as a write.
	configClipped bool
}

// walkMemories attributes each tracked key to the commit that last changed its
// value, walking the history of ref newest to oldest, at most memoryWalkMax
// commits.
//
// The trick that makes it affordable is the table hash. At each commit the
// content hash of config is O(1) and reads no rows; when it equals the hash at
// the newer neighbour, config is byte-identical across that step and the newer
// commit wrote no memory — skip, read nothing. Only a commit that actually
// touched config costs a row read, and then each still-unresolved key whose
// value differs from the newer neighbour's was written *by that newer commit*.
//
// The commit message is not the signal. `bd remember` does write
// "bd: remember (auto-commit) by <author>", but that is a claim by whoever wrote
// it; the table hash is the fact.
//
// A key still unresolved when the history itself runs out was present with this
// value at the root commit, so the root is what wrote it — that is a date the
// walk supports. A key unresolved because the walk hit its budget gets no date
// at all, and truncated says so.
//
// The walk reads the log's linearization (Log is reverse-topological). Beads
// histories are linear chains of auto-commits, which this is exact for; across a
// merge, attribution is to the nearest commit in that order.
func walkMemories(ctx context.Context, sess MemorySession, ref string, tracked map[string]string) (memoryWalk, error) {
	out := memoryWalk{revisions: map[string]MemoryRevision{}}

	commits, next, err := sess.Log(ctx, ref, "", memoryWalkMax)
	if err != nil {
		return memoryWalk{}, fmt.Errorf("beads: read history of %q: %w", ref, err)
	}
	if len(commits) == 0 {
		return out, nil
	}
	out.oldest = commits[len(commits)-1].Date
	out.truncated = next != ""

	// unresolved carries each key's value at the newer neighbour of the commit
	// being examined; it starts as the value at the head, which is where the
	// memories themselves were read.
	unresolved := make(map[string]string, len(tracked))
	for k, v := range tracked {
		unresolved[k] = v
	}

	newerHash, err := memoryTableHash(ctx, sess, commits[0].Hash)
	if err != nil {
		return memoryWalk{}, err
	}

	for i := 1; i < len(commits) && len(unresolved) > 0; i++ {
		hash, err := memoryTableHash(ctx, sess, commits[i].Hash)
		if err != nil {
			return memoryWalk{}, err
		}
		if hash == newerHash {
			continue // config unchanged across this step: nothing to read
		}

		older, olderTotal, err := memoryValuesAt(ctx, sess, commits[i].Hash)
		if err != nil {
			return memoryWalk{}, err
		}
		out.configClipped = out.configClipped || olderTotal > Max
		newer := commits[i-1]
		for key, newerValue := range unresolved {
			if older[key] == newerValue {
				continue
			}
			out.revisions[key] = MemoryRevision{
				Commit: newer.Hash,
				Date:   newer.Date,
				Author: newer.Author,
			}
			delete(unresolved, key)
		}
		for key := range unresolved {
			unresolved[key] = older[key]
		}
		newerHash = hash
	}

	if !out.truncated {
		// The history ended with these keys never changing: the oldest commit
		// walked is the root, and it carries the value we are looking at.
		root := commits[len(commits)-1]
		for key := range unresolved {
			out.revisions[key] = MemoryRevision{Commit: root.Hash, Date: root.Date, Author: root.Author}
		}
	}

	return out, nil
}

// memoryTableHash is the config table's content hash at one commit. A table that
// does not exist there is "" — an ordinary answer while walking backwards past
// the commit that created it, and one that compares correctly against another
// commit where it is equally absent.
func memoryTableHash(ctx context.Context, sess MemorySession, at string) (string, error) {
	hash, ok, err := sess.TableHash(ctx, at, memoryTable)
	if err != nil {
		return "", fmt.Errorf("beads: hash of %s at %s: %w", memoryTable, at, err)
	}
	if !ok {
		return "", nil
	}
	return hash, nil
}

// memoryValuesAt reads the config table at one commit as key → value, and the
// total that read reported. A missing table is an empty map, which is what it
// means here: no key had a value yet.
//
// The total is returned rather than dropped because "no key had a value yet"
// and "the key sits past Max" arrive at this caller as the same empty slot, and
// only the total tells them apart.
func memoryValuesAt(ctx context.Context, sess MemorySession, at string) (map[string]string, int, error) {
	rows, total, err := readRowsOptional(ctx, sess, at, memoryTable)
	if err != nil {
		return nil, 0, err
	}
	out := map[string]string{}
	if rows == nil {
		return out, total, nil
	}
	cols := indexCols(rows.Columns)
	for _, r := range rowsOf(rows) {
		if key := cell(cols, r, "key"); key != "" {
			out[key] = cell(cols, r, "value")
		}
	}
	return out, total, nil
}

// memoryEscapedNewline matches the two-character escapes that reach the value
// because the memory was typed into a shell string: "\r\n" and "\n" written out
// as backslash sequences rather than as newlines.
var memoryEscapedNewline = regexp.MustCompile(`\\r\\n|\\n`)

// memoryBlankLine splits a memory into paragraphs on runs of blank lines.
var memoryBlankLine = regexp.MustCompile(`\n[ \t]*\n[ \t\n]*`)

// normalizeMemoryText brings a stored value's two newline spellings together.
// The same memory arrives with real newlines when it was written from a file or
// a heredoc and with literal backslash-n when it was typed into a shell string,
// and both spellings turn up in the same tracker.
func normalizeMemoryText(v string) string {
	v = strings.ReplaceAll(v, "\r\n", "\n")
	v = strings.ReplaceAll(v, "\r", "\n")
	v = memoryEscapedNewline.ReplaceAllString(v, "\n")
	return strings.TrimSpace(v)
}

// memoryParagraphs splits normalised text on blank lines. Line breaks *inside* a
// paragraph are kept — memories are full of bullet lists, and joining those into
// a sentence is not rendering, it is losing the structure the author typed.
func memoryParagraphs(text string) []string {
	if text == "" {
		return nil
	}
	parts := memoryBlankLine.Split(text, -1)
	out := make([]string, 0, len(parts))
	for _, p := range parts {
		if strings.TrimSpace(p) == "" {
			continue
		}
		out = append(out, strings.Trim(p, "\n"))
	}
	return out
}

// matchesMemorySearch is the ?q= rule: a case-insensitive substring of the slug
// or of the text. An empty query matches everything.
func matchesMemorySearch(q, slug, text string) bool {
	if q == "" {
		return true
	}
	q = strings.ToLower(q)
	return strings.Contains(strings.ToLower(slug), q) || strings.Contains(strings.ToLower(text), q)
}

// parseMemorySort reads ?sort=, defaulting (and falling back from an unknown
// value) to slug order.
func parseMemorySort(v string) string {
	if strings.ToLower(strings.TrimSpace(v)) == MemorySortAge {
		return MemorySortAge
	}
	return MemorySortSlug
}

// sortMemories orders the list: by slug, or oldest first for the review queue.
// In age order a memory with no revision leads — it is older than anything the
// walk could date — and ties fall back to the slug, so the order is total.
func sortMemories(ms []Memory, order string) {
	sort.SliceStable(ms, func(i, j int) bool {
		a, b := ms[i], ms[j]
		if order == MemorySortAge {
			switch {
			case a.Revision == nil && b.Revision != nil:
				return true
			case a.Revision != nil && b.Revision == nil:
				return false
			case a.Revision != nil && b.Revision != nil && !a.Revision.Date.Equal(b.Revision.Date):
				return a.Revision.Date.Before(b.Revision.Date)
			}
		}
		return a.Slug < b.Slug
	})
}