From 8dfe078bcda61dc7ec82d607468747771d9dfa41 Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Thu, 13 Aug 2026 10:48:16 +0300 Subject: [PATCH] beads: report a clipped read from the remaining projections --- beads/memory.go | 54 ++- beads/prefixes.go | 90 +++-- beads/ready.go | 116 ++++++- beads/truncation_projections_test.go | 469 +++++++++++++++++++++++++++ 4 files changed, 681 insertions(+), 48 deletions(-) create mode 100644 beads/truncation_projections_test.go diff --git a/beads/memory.go b/beads/memory.go index d7aa2e5f93ba6f189c2eab09b315ef1c8f540377..5781a5667a19f57ae749c93c2feb687bad4f2844 100644 --- a/beads/memory.go +++ b/beads/memory.go @@ -91,8 +91,29 @@ type MemoryView struct { // it is what the page says instead of a date. WalkTruncated bool WalkMax int // memoryWalkMax, so the page can name the number it hit + + // ConfigTruncated says a read of the config table came back clipped at + // beads.Max. That is a different clip from WalkTruncated, which bounds the + // history in commits; this one bounds a single read in rows, and it covers + // both reads this view makes: the one at ref that the memories themselves + // come from, and the per-commit ones the walk dates them by. A clipped read + // in the walk leaves the dates unsafe rather than the list short — a key + // whose row fell past the cap reads as absent there, which is + // indistinguishable from a key that had not been written yet. + ConfigTruncated bool + // ConfigShownOf is the config table's reported total at ref, clipped or not: + // rows that exist, against the at most Max that were read. It counts the + // whole table, memory keys and tracker settings alike, because that is what + // the cap applies to. + ConfigShownOf int } +// ConfigClipped reports that the config read the memories themselves come from +// exceeded Max, so Memories and Total cover its first Max rows only and a +// memory may be missing from the list entirely. ConfigTruncated is the wider +// fact (any config read, here or in the walk, was clipped). +func (v *MemoryView) ConfigClipped() bool { return v.ConfigShownOf > Max } + // Memory sort orders. Slug is the default; Age is the review queue. const ( MemorySortSlug = "slug" @@ -144,10 +165,12 @@ func BuildMemories(ctx context.Context, sess MemorySession, ref string, query ur WalkMax: memoryWalkMax, } - rows, _, err := readRowsOptional(ctx, sess, ref, memoryTable) + rows, configTotal, err := readRowsOptional(ctx, sess, ref, memoryTable) if err != nil { return nil, err } + view.ConfigShownOf = configTotal + view.ConfigTruncated = configTotal > Max if rows == nil { return view, nil } @@ -194,6 +217,7 @@ func BuildMemories(ctx context.Context, sess MemorySession, ref string, query ur return nil, err } view.WalkTruncated = walk.truncated + view.ConfigTruncated = view.ConfigTruncated || walk.configClipped view.Memories = make([]Memory, 0, len(tracked)) for key := range tracked { @@ -225,6 +249,12 @@ type memoryWalk struct { revisions map[string]MemoryRevision truncated bool oldest time.Time + // configClipped says one of the per-commit config reads exceeded Max. It is + // a different bound from truncated — rows rather than commits — and it costs + // the attribution its footing: a key the read never reached looks like a key + // that commit had not written yet, which is precisely what the comparison + // below treats as a write. + configClipped bool } // walkMemories attributes each tracked key to the commit that last changed its @@ -285,10 +315,11 @@ func walkMemories(ctx context.Context, sess MemorySession, ref string, tracked m continue // config unchanged across this step: nothing to read } - older, err := memoryValuesAt(ctx, sess, commits[i].Hash) + older, olderTotal, err := memoryValuesAt(ctx, sess, commits[i].Hash) if err != nil { return memoryWalk{}, err } + out.configClipped = out.configClipped || olderTotal > Max newer := commits[i-1] for key, newerValue := range unresolved { if older[key] == newerValue { @@ -334,16 +365,21 @@ func memoryTableHash(ctx context.Context, sess MemorySession, at string) (string return hash, nil } -// memoryValuesAt reads the config table at one commit as key → value. A missing -// table is an empty map, which is what it means here: no key had a value yet. -func memoryValuesAt(ctx context.Context, sess MemorySession, at string) (map[string]string, error) { - rows, _, err := readRowsOptional(ctx, sess, at, memoryTable) +// memoryValuesAt reads the config table at one commit as key → value, and the +// total that read reported. A missing table is an empty map, which is what it +// means here: no key had a value yet. +// +// The total is returned rather than dropped because "no key had a value yet" +// and "the key sits past Max" arrive at this caller as the same empty slot, and +// only the total tells them apart. +func memoryValuesAt(ctx context.Context, sess MemorySession, at string) (map[string]string, int, error) { + rows, total, err := readRowsOptional(ctx, sess, at, memoryTable) if err != nil { - return nil, err + return nil, 0, err } out := map[string]string{} if rows == nil { - return out, nil + return out, total, nil } cols := indexCols(rows.Columns) for _, r := range rowsOf(rows) { @@ -351,7 +387,7 @@ func memoryValuesAt(ctx context.Context, sess MemorySession, at string) (map[str out[key] = cell(cols, r, "value") } } - return out, nil + return out, total, nil } // memoryEscapedNewline matches the two-character escapes that reach the value diff --git a/beads/prefixes.go b/beads/prefixes.go index 2badc56112f02beea17a7364d161c4db5cd3bdae..85315f0b0889dcf204b406c367a119b1925ccbc1 100644 --- a/beads/prefixes.go +++ b/beads/prefixes.go @@ -37,7 +37,19 @@ const prefixKey = "issue_prefix" // a second set (see cache.go). The zero value is usable, so a holder that has no // constructor can carry one as a field. type PrefixCache struct { - projectionCache[string] + projectionCache[prefixEntry] +} + +// prefixEntry is one database's cached prefix projection: the prefix its config +// names, and the total that read reported. The total is cached with the prefix +// rather than recomputed because a cache hit reads no row — it must say what +// the read that produced it said, including that the read was partial. +type prefixEntry struct { + prefix string + // configTotal is the config table's reported total, clipped or not. Over Max + // it means the read stopped short, and an absent prefix is then a row that + // was never reached rather than a row that does not exist. + configTotal int } // PrefixIndex is prefix → the database that owns it, over the databases one @@ -50,6 +62,21 @@ type PrefixIndex struct { // caller's log and for nothing else: a database that could not be read costs // the ids it owns their link, and a page may not say more than that. Failed []ReadyFailure + // Truncated lists the databases whose config table exceeded Max and came back + // clipped, in the order the caller listed them. A clipped config is how a + // tracker loses its prefix without failing: the issue_prefix row simply was + // not among the rows read, so every id pointing at that tracker stops linking + // and the index looks complete. It is a row clip, unrelated to the ceiling on + // how many databases one build opens. + Truncated []PrefixTruncation +} + +// PrefixTruncation is one database whose config table came back clipped. +type PrefixTruncation struct { + Database ReadyDatabase + // ShownOf is that config table's reported total: rows that exist, against the + // at most Max that were read. + ShownOf int } // Reference is one id found in a text: where it sits, what it says, and which @@ -112,11 +139,21 @@ func PrefixesAcross( // guess rendered as a fact, and the id is readable as text either way. ambiguous := map[string]bool{} for _, d := range dbs { - prefix, err := databasePrefix(ctx, d, open, cache, now) + entry, err := databasePrefix(ctx, d, open, cache, now) if err != nil { index.Failed = append(index.Failed, ReadyFailure{Database: d, Err: err}) continue } + if entry.configTotal > Max { + // Recorded before the empty-prefix skip below, because a clipped read is + // the one case where an empty prefix is not an answer: the row may be + // sitting in the tail this build never saw. + index.Truncated = append(index.Truncated, PrefixTruncation{ + Database: d, + ShownOf: entry.configTotal, + }) + } + prefix := entry.prefix if prefix == "" { continue } @@ -133,8 +170,9 @@ func PrefixesAcross( return index } -// databasePrefix returns one database's issue prefix, from the cache when the -// head has not moved and by reading config otherwise. +// databasePrefix returns one database's issue prefix and the total its config +// read reported, from the cache when the head has not moved and by reading +// config otherwise. // // The order is the whole point of the gate: open, list branches, and only then // consult the cache. Opening a session and reading the branch list is cheap; @@ -145,46 +183,51 @@ func databasePrefix( open ReadyOpener, cache *PrefixCache, now time.Time, -) (string, error) { +) (prefixEntry, error) { sess, err := open(ctx, d) if err != nil { - return "", err + return prefixEntry{}, err } defer sess.Close() branches, err := sess.Branches(ctx) if err != nil { - return "", err + return prefixEntry{}, err } ref := browse.DefaultBranch(branches) if ref == "" { // A store with no branches carries no config either: nothing to read and // nothing to cache. - return "", nil + return prefixEntry{}, nil } head := headHashOf(branches, ref) - if prefix, ok := cache.lookup(d.ID, head, now); ok { - return prefix, nil + if entry, ok := cache.lookup(d.ID, head, now); ok { + return entry, nil } - prefix, err := readPrefix(ctx, sess, ref) + entry, err := readPrefix(ctx, sess, ref) if err != nil { - return "", err + return prefixEntry{}, err } - cache.store(d.ID, head, now, prefix) - return prefix, nil + cache.store(d.ID, head, now, entry) + return entry, nil } -// readPrefix reads the issue_prefix row out of a database's config table. A -// missing table — this is not a bd tracker — is no prefix, the treatment every -// optional table gets here. -func readPrefix(ctx context.Context, sess BrowseSession, ref string) (string, error) { - rows, _, err := readRowsOptional(ctx, sess, ref, memoryTable) +// readPrefix reads the issue_prefix row out of a database's config table, and +// the total that read reported. A missing table — this is not a bd tracker — is +// no prefix, the treatment every optional table gets here. +// +// The total comes back with the prefix because the two answers this can produce +// are otherwise identical: a config with no issue_prefix row and a config whose +// issue_prefix row was left past Max both arrive here as no prefix at all. +func readPrefix(ctx context.Context, sess BrowseSession, ref string) (prefixEntry, error) { + rows, total, err := readRowsOptional(ctx, sess, ref, memoryTable) if err != nil { - return "", err + return prefixEntry{}, err } + entry := prefixEntry{configTotal: total} if rows == nil { - return "", nil + return entry, nil } cols := indexCols(rows.Columns) for _, r := range rowsOf(rows) { @@ -193,9 +236,10 @@ func readPrefix(ctx context.Context, sess BrowseSession, ref string) (string, er } // Lowercased because that is the case the ids themselves are written in, // and the index is looked up by what the text says. - return strings.ToLower(strings.TrimSpace(cell(cols, r, "value"))), nil + entry.prefix = strings.ToLower(strings.TrimSpace(cell(cols, r, "value"))) + return entry, nil } - return "", nil + return entry, nil } // Lookup returns the database owning prefix. A nil index — no database was diff --git a/beads/ready.go b/beads/ready.go index a4ca796026db3697f0448f9b236fc54d31c3b0f0..c2189e4800c9e0542247d7626fc3058d3851c62a 100644 --- a/beads/ready.go +++ b/beads/ready.go @@ -92,8 +92,42 @@ type ReadyGroup struct { Ref string // the branch the ready set was read from Head *browse.CommitInfo // that branch's head, or nil when it cannot be read Cards []Card // ready issues, priority then id + + // Truncated says one of the tables this group was projected from — issues, + // dependencies, labels, custom_statuses — exceeded Max and came back clipped. + // The cards below are then the ready work among the rows that were read, + // which is not the same claim as this database's ready work. + // + // It is a row clip and has nothing to do with ReadyView.Capped, which counts + // databases. + Truncated bool + // ShownOf is this database's issues total, clipped or not: what exists, + // against the at most Max rows the projection read. IssuesClipped is the + // comparison callers usually want. + ShownOf int +} + +// IssuesClipped reports that this database's issues table itself exceeded Max, +// so the ready rule was applied to its first Max rows only. Truncated is the +// wider fact (any input table was clipped); this is the one that says ready work +// may be missing from the group rather than merely mislabelled. +func (g ReadyGroup) IssuesClipped() bool { return g.ShownOf > Max } + +// ReadyTruncation is one database whose ready set was projected from a clipped +// read. It exists separately from ReadyGroup because a database with no group is +// exactly the case that needs saying: a tracker whose ready work sits past Max +// is absent from Groups for the same reason a tracker with no ready work is, and +// without this the two are indistinguishable. +type ReadyTruncation struct { + Database ReadyDatabase + // ShownOf is that database's issues total, clipped or not. A truncation whose + // ShownOf is within Max was caused by one of the other three tables. + ShownOf int } +// IssuesClipped reports that this database's issues table itself exceeded Max. +func (t ReadyTruncation) IssuesClipped() bool { return t.ShownOf > Max } + // ReadyFailure is one database that could not be read. It carries the error for // the caller's log and nothing for the reader: an error string on a page is how // a store path and a dolt internal end up in a browser (sr-ht-dolt-7ta). @@ -102,9 +136,9 @@ type ReadyFailure struct { Err error } -// ReadyView is the whole answer: the groups, what was considered, and the two -// facts a reader needs in order not to over-read it (the ceiling, and that some -// databases could not be read). +// ReadyView is the whole answer: the groups, what was considered, and the three +// facts a reader needs in order not to over-read it (the ceiling, that some +// databases could not be read, and that some were read only in part). type ReadyView struct { Groups []ReadyGroup Total int // ready cards across every group, after filtering @@ -114,6 +148,17 @@ type ReadyView struct { Failed []ReadyFailure Filter ReadyFilter Options ReadyOptions + + // Truncated lists every database considered whose rows came back clipped at + // beads.Max, in the order the caller listed them. It is the complete set, and + // deliberately wider than the groups: a database whose ready work sits past + // the cap produces no group at all, and that is the case a per-group flag + // cannot report. + // + // Capped and this are two different bounds and neither implies the other. + // Capped counts databases — there were more trackers than one call may open. + // This counts rows inside a database that was opened and read. + Truncated []ReadyTruncation // Query is the request's query as parsed, carried so a link can rebuild this // exact URL with one key replaced (web's withQuery) instead of re-listing the // parameters it happens to know about. @@ -211,6 +256,20 @@ type readyEntry struct { // reads no log either. commit *browse.CommitInfo cards []Card + // read is what the row reads reported about their own completeness. It is + // cached beside the cards for the same reason: a projection served from the + // cache has to say what the read that produced it said, and a cache hit + // reads no row it could learn this from a second time. + read readyRead +} + +// readyRead is what one database's row reads reported about their own +// completeness: whether any table came back clipped at Max, and the issues +// table's true total. Every number in it comes from the reads readyCards +// already makes. +type readyRead struct { + truncated bool // some table this projection reads exceeded Max + issuesTotal int // the issues table's reported total, clipped or not } // NewReadyCache returns an empty cache. The zero value works too — the map is @@ -272,6 +331,16 @@ func ReadyAcross( // silently, exactly as the view tabs skip it. continue } + if entry.read.truncated { + // Recorded before the card filters and before the empty-group skip + // below: a database whose ready work was left past the cap has no + // group to carry the fact, and it is that database the reader most + // needs named. + view.Truncated = append(view.Truncated, ReadyTruncation{ + Database: d, + ShownOf: entry.read.issuesTotal, + }) + } cards := make([]Card, 0, len(entry.cards)) for _, c := range entry.cards { if c.Assignee != "" { @@ -289,10 +358,12 @@ func ReadyAcross( continue } view.Groups = append(view.Groups, ReadyGroup{ - Database: d, - Ref: entry.ref, - Head: entry.commit, - Cards: cards, + Database: d, + Ref: entry.ref, + Head: entry.commit, + Cards: cards, + Truncated: entry.read.truncated, + ShownOf: entry.read.issuesTotal, }) view.Total += len(cards) } @@ -358,11 +429,12 @@ func readyProjection( return entry, nil } - cards, err := readyCards(ctx, sess, ref) + cards, read, err := readyCards(ctx, sess, ref) if err != nil { return readyEntry{}, err } entry.cards = cards + entry.read = read entry.commit = readyHead(ctx, sess, ref) cache.store(d.ID, head, now, entry) return entry, nil @@ -396,17 +468,29 @@ func readyHead(ctx context.Context, sess ReadySession, ref string) *browse.Commi // readyCards projects one database's ready set: the same tables the board reads, // through the same ready rule (readyRow), sorted priority then id. -func readyCards(ctx context.Context, sess BrowseSession, ref string) ([]Card, error) { - issues, _, err := readRows(ctx, sess, ref, "issues") +// +// It also returns what those reads said about their own completeness. All four +// tables count towards it: a clipped issues table leaves ready work unread, and +// a clipped dependencies, labels or custom_statuses table changes the verdict on +// the issues that were read — a blocking edge past the cap is a card called +// ready that is not. +func readyCards(ctx context.Context, sess BrowseSession, ref string) ([]Card, readyRead, error) { + issues, issuesTotal, err := readRows(ctx, sess, ref, "issues") if err != nil { - return nil, err + return nil, readyRead{}, err } - deps, _, err := readRows(ctx, sess, ref, "dependencies") + deps, depsTotal, err := readRows(ctx, sess, ref, "dependencies") if err != nil { - return nil, err + return nil, readyRead{}, err + } + labels, labelsTotal, _ := readRowsOptional(ctx, sess, ref, "labels") + statuses, statusesTotal, _ := readRowsOptional(ctx, sess, ref, "custom_statuses") + + read := readyRead{ + truncated: issuesTotal > Max || depsTotal > Max || + labelsTotal > Max || statusesTotal > Max, + issuesTotal: issuesTotal, } - labels, _, _ := readRowsOptional(ctx, sess, ref, "labels") - statuses, _, _ := readRowsOptional(ctx, sess, ref, "custom_statuses") catByStatus := indexStatusCategories(statuses) issueCols := indexCols(issues.Columns) @@ -436,7 +520,7 @@ func readyCards(ctx context.Context, sess BrowseSession, ref string) ([]Card, er }) } sortReadyCards(cards) - return cards, nil + return cards, read, nil } // sortReadyCards orders a group by priority (0 = highest first), then id. The diff --git a/beads/truncation_projections_test.go b/beads/truncation_projections_test.go new file mode 100644 index 0000000000000000000000000000000000000000..16fc056e4c18e58cdf400ddb2d9a6d6eacfb4cc8 --- /dev/null +++ b/beads/truncation_projections_test.go @@ -0,0 +1,469 @@ +package beads + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "sourcecraft.dev/bigbes/sr-ht-dolt/browse" +) + +// --- the three cross-cutting projections, read past the cap ------------------- +// +// The seam is truncation_test.go's clippingSession: it honours offset/limit and +// reports the table's true row count, which is what a real read at limit=Max +// does. Everything here builds on it rather than lowering Max or hand-writing a +// page whose Total merely disagrees with its own rows — the row that was left +// behind has to be genuinely absent from what the projection is handed, or the +// test asserts a flag without ever producing the situation the flag is about. + +// clippingReadyDB is a ReadySession over that seam: the clipping row reads, plus +// the branch list, table list and log the cross-database readings need. It +// counts row reads, because the head-hash gate is a claim about reads not +// happening and only a counter can check it. +type clippingReadyDB struct { + *clippingSession + branches []browse.Branch + tables []browse.TableInfo + commits []browse.CommitInfo + + opens int + closes int + rowReads int +} + +func (f *clippingReadyDB) Rows(ctx context.Context, ref, table string, offset, limit int) (*browse.RowPage, error) { + f.rowReads++ + return f.clippingSession.Rows(ctx, ref, table, offset, limit) +} + +func (f *clippingReadyDB) Branches(context.Context) ([]browse.Branch, error) { + return f.branches, nil +} + +func (f *clippingReadyDB) Tables(context.Context, string) ([]browse.TableInfo, error) { + return f.tables, nil +} + +func (f *clippingReadyDB) Log(_ context.Context, _, _ string, _ int) ([]browse.CommitInfo, string, error) { + return f.commits, "", nil +} + +func (f *clippingReadyDB) Close() error { f.closes++; return nil } + +// clippingTracker is one beads database at head, over the given tables. +func clippingTracker(head string, tables map[string]*browse.RowPage) *clippingReadyDB { + return &clippingReadyDB{ + clippingSession: &clippingSession{rowsByTable: tables}, + branches: []browse.Branch{{Name: "main", Head: head}}, + tables: beadsTables(), + commits: []browse.CommitInfo{{ + Hash: head, Author: "bigbes", Date: readyNow.Add(-4 * time.Minute), + Message: "bd: create (auto-commit)", + }}, + } +} + +// clippingInstance is a set of clipping databases addressed by repository id, +// plus the opener over them. +type clippingInstance struct { + dbs map[int]*clippingReadyDB +} + +func (in *clippingInstance) open(_ context.Context, d ReadyDatabase) (ReadySession, error) { + f, ok := in.dbs[d.ID] + if !ok { + return nil, fmt.Errorf("no such database: %s", d.Slug()) + } + f.opens++ + return f, nil +} + +// issueColumns is manyIssues' column order, needed by the fixtures below that +// build their own issue rows. +var issueColumns = []string{ + "id", "title", "status", "priority", "issue_type", "assignee", "created_at", "is_blocked", +} + +// emptyDeps is a present but empty dependencies table. The table is required — +// a ready projection that cannot read it fails rather than answering — so every +// tracker here carries one. +func emptyDeps() *browse.RowPage { + return &browse.RowPage{Columns: []string{"id", "issue_id", "depends_on_issue_id", "type"}} +} + +// readyTables is the two tables a ready projection requires, plus the issues +// page under test. +func readyTables(issues *browse.RowPage) map[string]*browse.RowPage { + return map[string]*browse.RowPage{"issues": issues, "dependencies": emptyDeps()} +} + +// tailReadyIssues builds n issue rows in which only the rows from index `from` +// on are open; everything before it is closed. With from == Max the tracker's +// entire ready set sits past the cap, so a clipped read finds nothing ready — +// and a database with nothing ready is absent from the groups, which is exactly +// the case a per-group flag cannot report. +func tailReadyIssues(n, from int) *browse.RowPage { + rows := make([][]string, 0, n) + for i := 0; i < n; i++ { + status := "closed" + if i >= from { + status = "open" + } + rows = append(rows, []string{ + issueID(i), fmt.Sprintf("Issue %d", i), status, + "1", "task", "alice", "2024-01-01 00:00:00", "0", + }) + } + return &browse.RowPage{Columns: issueColumns, Rows: rows, Total: n} +} + +// groupFor returns the group for a database slug, failing the test when the +// answer carries none. +func groupFor(t *testing.T, view *ReadyView, slug string) ReadyGroup { + t.Helper() + for _, g := range view.Groups { + if g.Database.Slug() == slug { + return g + } + } + require.FailNowf(t, "no group", "the answer carries no group for %s", slug) + return ReadyGroup{} +} + +// --- the cross-database ready set --------------------------------------------- + +func TestReadyAcrossReportsAClippedRead(t *testing.T) { + in := &clippingInstance{dbs: map[int]*clippingReadyDB{ + 1: clippingTracker("h-alpha", readyTables(manyIssues(Max+5))), + }} + dbs := []ReadyDatabase{{ID: 1, OwnerName: "alice", Name: "alpha"}} + + view := ReadyAcross(t.Context(), dbs, in.open, NewReadyCache(), ReadyFilter{}, readyNow) + + require.Len(t, view.Truncated, 1, "the database's read was partial and the answer must say so") + assert.Equal(t, "alice/alpha", view.Truncated[0].Database.Slug()) + assert.Equal(t, Max+5, view.Truncated[0].ShownOf, "the tracker's true issue count, not the number read") + assert.True(t, view.Truncated[0].IssuesClipped()) + + // The group carries the same fact, so a rendered group needs no lookup. + g := groupFor(t, view, "alice/alpha") + assert.True(t, g.Truncated) + assert.True(t, g.IssuesClipped()) + assert.Equal(t, Max+5, g.ShownOf) + + // And the other bound is untouched: one database is not a ceiling on + // databases. + assert.False(t, view.Capped) + assert.Equal(t, 1, view.Considered) +} + +func TestReadyAcrossOnACompleteReadReportsNoClip(t *testing.T) { + in := &clippingInstance{dbs: map[int]*clippingReadyDB{ + 1: clippingTracker("h-alpha", readyTables(manyIssues(9))), + }} + dbs := []ReadyDatabase{{ID: 1, OwnerName: "alice", Name: "alpha"}} + + view := ReadyAcross(t.Context(), dbs, in.open, NewReadyCache(), ReadyFilter{}, readyNow) + + assert.Empty(t, view.Truncated) + g := groupFor(t, view, "alice/alpha") + assert.False(t, g.Truncated) + assert.False(t, g.IssuesClipped()) + assert.Equal(t, 9, g.ShownOf, "nothing was left behind, so this is what was read") + assert.Len(t, g.Cards, 3, "one issue in three is open in this fixture") +} + +// The attribution: two databases, one clipped. The answer names that one and +// says nothing about the other — "some read somewhere was partial" is not +// actionable on an instance with sixty trackers. +func TestReadyAcrossAttributesTheClipToItsDatabase(t *testing.T) { + in := &clippingInstance{dbs: map[int]*clippingReadyDB{ + 1: clippingTracker("h-alpha", readyTables(manyIssues(Max+5))), + 2: clippingTracker("h-beta", readyTables(manyIssues(6))), + }} + dbs := []ReadyDatabase{ + {ID: 1, OwnerName: "alice", Name: "alpha"}, + {ID: 2, OwnerName: "bob", Name: "beta"}, + } + + view := ReadyAcross(t.Context(), dbs, in.open, NewReadyCache(), ReadyFilter{}, readyNow) + + require.Len(t, view.Groups, 2, "both databases have ready work") + require.Len(t, view.Truncated, 1, "only one of the two was read in part") + assert.Equal(t, "alice/alpha", view.Truncated[0].Database.Slug()) + assert.Equal(t, Max+5, view.Truncated[0].ShownOf) + + assert.True(t, groupFor(t, view, "alice/alpha").Truncated) + beta := groupFor(t, view, "bob/beta") + assert.False(t, beta.Truncated, "the sibling's read was complete") + assert.Equal(t, 6, beta.ShownOf) +} + +// The case the group cannot carry: every ready issue sits past the cap, so the +// database has no group at all. Without the view's list, a tracker missing its +// ready work is indistinguishable from a tracker with none. +func TestReadyAcrossReportsAClippedDatabaseThatProducedNoGroup(t *testing.T) { + in := &clippingInstance{dbs: map[int]*clippingReadyDB{ + 1: clippingTracker("h-alpha", readyTables(tailReadyIssues(Max+5, Max))), + }} + dbs := []ReadyDatabase{{ID: 1, OwnerName: "alice", Name: "alpha"}} + + view := ReadyAcross(t.Context(), dbs, in.open, NewReadyCache(), ReadyFilter{}, readyNow) + + assert.Empty(t, view.Groups, "nothing in the rows that were read is ready") + assert.Zero(t, view.Total) + assert.Empty(t, view.Failed) + require.Len(t, view.Truncated, 1, "and the reason the database is absent is on the record") + assert.Equal(t, "alice/alpha", view.Truncated[0].Database.Slug()) + assert.Equal(t, Max+5, view.Truncated[0].ShownOf) +} + +// A clip in one of the other tables the ready rule is computed from: the cards +// are all there, but the verdict on them was reached over a partial read — a +// blocking edge past the cap is a card called ready that is not. +func TestReadyAcrossReportsAClippedDependenciesRead(t *testing.T) { + deps := emptyDeps() + for i := 0; i < Max+1; i++ { + deps.Rows = append(deps.Rows, []string{fmt.Sprintf("d%d", i), "i-0002", "i-0000", "blocks"}) + } + in := &clippingInstance{dbs: map[int]*clippingReadyDB{ + 1: clippingTracker("h-alpha", map[string]*browse.RowPage{ + "issues": manyIssues(9), "dependencies": deps, + }), + }} + dbs := []ReadyDatabase{{ID: 1, OwnerName: "alice", Name: "alpha"}} + + view := ReadyAcross(t.Context(), dbs, in.open, NewReadyCache(), ReadyFilter{}, readyNow) + + require.Len(t, view.Truncated, 1) + assert.Equal(t, 9, view.Truncated[0].ShownOf) + assert.False(t, view.Truncated[0].IssuesClipped(), "the issue set itself is whole") + g := groupFor(t, view, "alice/alpha") + assert.True(t, g.Truncated, "dependencies decide which of those issues are ready") + assert.False(t, g.IssuesClipped()) +} + +// The clip is cached with the cards. A second call under an unmoved head reads +// no row it could learn this from, so an uncached flag would quietly become +// false on the second request. +func TestReadyAcrossClipSurvivesTheCache(t *testing.T) { + in := &clippingInstance{dbs: map[int]*clippingReadyDB{ + 1: clippingTracker("h-alpha", readyTables(manyIssues(Max+5))), + }} + dbs := []ReadyDatabase{{ID: 1, OwnerName: "alice", Name: "alpha"}} + cache := NewReadyCache() + + first := ReadyAcross(t.Context(), dbs, in.open, cache, ReadyFilter{}, readyNow) + require.Len(t, first.Truncated, 1) + reads := in.dbs[1].rowReads + require.Greater(t, reads, 0, "the first call must read rows") + + second := ReadyAcross(t.Context(), dbs, in.open, cache, ReadyFilter{}, readyNow.Add(30*time.Second)) + + assert.Equal(t, reads, in.dbs[1].rowReads, "a second call with an unmoved head must read no rows") + require.Len(t, second.Truncated, 1, "and must still report the clipped read") + assert.Equal(t, Max+5, second.Truncated[0].ShownOf) + assert.True(t, groupFor(t, second, "alice/alpha").Truncated) +} + +// --- the memory list ---------------------------------------------------------- + +// clippingHistory is a MemorySession whose config reads clip through the same +// seam: one clippingSession per ref, plus the table hashes and the log the +// revision walk runs on. +type clippingHistory struct { + refs map[string]*clippingSession // ref → the tables there + hashes map[string]string // ref → config's content hash; "" = table absent + commits []browse.CommitInfo // newest first, as Log returns them +} + +func (f *clippingHistory) Rows(ctx context.Context, ref, table string, offset, limit int) (*browse.RowPage, error) { + s, ok := f.refs[ref] + if !ok { + return nil, fmt.Errorf("%w: %s", browse.ErrTableNotFound, table) + } + return s.Rows(ctx, ref, table, offset, limit) +} + +func (f *clippingHistory) Log(_ context.Context, _, _ string, limit int) ([]browse.CommitInfo, string, error) { + if limit < len(f.commits) { + return f.commits[:limit], f.commits[limit].Hash, nil + } + return f.commits, "", nil +} + +func (f *clippingHistory) TableHash(_ context.Context, ref, table string) (string, bool, error) { + if table != memoryTable { + return "", false, nil + } + h, ok := f.hashes[ref] + if !ok || h == "" { + return "", false, nil + } + return h, true, nil +} + +// configPage builds a config table out of key/value pairs, in the key order it +// is given — a store returns them ordered by key, and which rows fall past the +// cap is a function of that order. +func configPage(rows [][]string) *browse.RowPage { + return &browse.RowPage{Columns: []string{"key", "value"}, Rows: rows, Total: len(rows)} +} + +// manyMemoryRows builds n memory rows, keys kv.memory.m0000… in order. +func manyMemoryRows(n int) [][]string { + rows := make([][]string, 0, n) + for i := 0; i < n; i++ { + rows = append(rows, []string{fmt.Sprintf("%sm%04d", memoryPrefix, i), fmt.Sprintf("memory %d", i)}) + } + return rows +} + +func TestMemoriesReportAClippedConfigRead(t *testing.T) { + rows := manyMemoryRows(Max + 3) + config := &clippingSession{rowsByTable: map[string]*browse.RowPage{memoryTable: configPage(rows)}} + sess := &clippingHistory{ + refs: map[string]*clippingSession{"main": config, "c0": config}, + hashes: map[string]string{"main": "h-a", "c0": "h-a"}, + commits: []browse.CommitInfo{{Hash: "c0", Author: "bigbes", Date: memoryNow.Add(-time.Hour)}}, + } + + v := buildMemories(t, sess, "") + + assert.True(t, v.ConfigTruncated, "the config table exceeded Max") + assert.True(t, v.ConfigClipped(), "and it is the read the memories themselves come from") + assert.Equal(t, Max+3, v.ConfigShownOf, "the table's true row count, not the number read") + assert.Equal(t, Max, v.Total, "three memories exist that this list does not carry") + assert.False(t, v.WalkTruncated, "which is a bound on commits and was not hit") +} + +func TestMemoriesOnACompleteReadReportNoClip(t *testing.T) { + v := buildMemories(t, memoryHistory(), "") + + assert.False(t, v.ConfigTruncated) + assert.False(t, v.ConfigClipped()) + assert.Equal(t, 4, v.ConfigShownOf, "three memories and the tracker's issue_prefix") + assert.Equal(t, 3, v.Total, "the settings row is not a memory") +} + +// The other config read: the walk's own, at each commit it has to compare. A +// clip there leaves the list whole and the dates unsafe, and the two facts are +// told apart rather than merged. +func TestMemoriesReportAClippedConfigReadInsideTheWalk(t *testing.T) { + head := &clippingSession{rowsByTable: map[string]*browse.RowPage{ + memoryTable: configPage([][]string{{memoryPrefix + "alpha", "second version"}}), + }} + // The root's config is the oversized one: the walk reads it to find out what + // alpha said there, and gets its first Max rows. + rootRows := append(manyMemoryRows(Max), []string{memoryPrefix + "alpha", "first version"}) + root := &clippingSession{rowsByTable: map[string]*browse.RowPage{memoryTable: configPage(rootRows)}} + sess := &clippingHistory{ + refs: map[string]*clippingSession{"main": head, "c1": head, "c0": root}, + hashes: map[string]string{"main": "h-b", "c1": "h-b", "c0": "h-a"}, + commits: []browse.CommitInfo{ + {Hash: "c1", Author: "bigbes", Date: memoryNow.Add(-time.Hour)}, + {Hash: "c0", Author: "alice", Date: memoryNow.Add(-48 * time.Hour)}, + }, + } + + v := buildMemories(t, sess, "") + + assert.True(t, v.ConfigTruncated, "a config read inside the walk was clipped") + assert.False(t, v.ConfigClipped(), "though the list of memories itself is whole") + assert.Equal(t, 1, v.ConfigShownOf) + assert.Equal(t, 1, v.Total) + assert.False(t, v.WalkTruncated, "the history is two commits and the walk saw both") +} + +// --- the prefix index --------------------------------------------------------- + +// prefixConfigRows is a config table of n rows whose issue_prefix sits last, so +// a read that stops at Max never reaches it: the tracker keeps its prefix and +// the index loses it. +func prefixConfigRows(n int, prefix string) [][]string { + rows := make([][]string, 0, n) + for i := 0; i < n-1; i++ { + rows = append(rows, []string{fmt.Sprintf("%sm%04d", memoryPrefix, i), "some memory"}) + } + return append(rows, []string{prefixKey, prefix}) +} + +func clippingPrefixTracker(head string, rows [][]string) *clippingReadyDB { + return clippingTracker(head, map[string]*browse.RowPage{memoryTable: configPage(rows)}) +} + +func TestPrefixesAcrossReportsAClippedConfigRead(t *testing.T) { + in := &clippingInstance{dbs: map[int]*clippingReadyDB{ + 1: clippingPrefixTracker("h-global", prefixConfigRows(Max+2, "global")), + }} + dbs := []ReadyDatabase{{ID: 1, OwnerName: "bigbes", Name: "beads-global"}} + + index := PrefixesAcross(t.Context(), dbs, in.open, &PrefixCache{}, readyNow) + + // The prefix is in the table and not in the index: this is the silent loss. + assert.Empty(t, index.Prefixes()) + _, ok := index.Lookup("global") + assert.False(t, ok) + assert.Empty(t, index.Failed, "a clipped read is not a failed one") + + require.Len(t, index.Truncated, 1, "and the index must not pass that off as an answer") + assert.Equal(t, "bigbes/beads-global", index.Truncated[0].Database.Slug()) + assert.Equal(t, Max+2, index.Truncated[0].ShownOf, "the config table's true row count") +} + +func TestPrefixesAcrossOnACompleteReadReportsNoClip(t *testing.T) { + in := &clippingInstance{dbs: map[int]*clippingReadyDB{ + 1: clippingPrefixTracker("h-global", prefixConfigRows(3, "global")), + }} + dbs := []ReadyDatabase{{ID: 1, OwnerName: "bigbes", Name: "beads-global"}} + + index := PrefixesAcross(t.Context(), dbs, in.open, &PrefixCache{}, readyNow) + + assert.Equal(t, []string{"global"}, index.Prefixes()) + assert.Empty(t, index.Truncated) +} + +func TestPrefixesAcrossAttributesTheClipToItsDatabase(t *testing.T) { + in := &clippingInstance{dbs: map[int]*clippingReadyDB{ + 1: clippingPrefixTracker("h-global", prefixConfigRows(Max+2, "global")), + 2: clippingPrefixTracker("h-artifacts", prefixConfigRows(3, "artifacts")), + }} + dbs := []ReadyDatabase{ + {ID: 1, OwnerName: "bigbes", Name: "beads-global"}, + {ID: 2, OwnerName: "bigbes", Name: "sourcehut-artifacts"}, + } + + index := PrefixesAcross(t.Context(), dbs, in.open, &PrefixCache{}, readyNow) + + assert.Equal(t, []string{"artifacts"}, index.Prefixes(), "the readable sibling is unaffected") + require.Len(t, index.Truncated, 1, "only one of the two was read in part") + assert.Equal(t, "bigbes/beads-global", index.Truncated[0].Database.Slug()) + assert.Equal(t, Max+2, index.Truncated[0].ShownOf) +} + +// The clip is cached with the prefix, for the reason the ready set's is: a +// second build under an unmoved head reads no row, and a projection served from +// the cache has to say what the read that produced it said. +func TestPrefixesAcrossClipSurvivesTheCache(t *testing.T) { + in := &clippingInstance{dbs: map[int]*clippingReadyDB{ + 1: clippingPrefixTracker("h-global", prefixConfigRows(Max+2, "global")), + }} + dbs := []ReadyDatabase{{ID: 1, OwnerName: "bigbes", Name: "beads-global"}} + cache := &PrefixCache{} + + first := PrefixesAcross(t.Context(), dbs, in.open, cache, readyNow) + require.Len(t, first.Truncated, 1) + reads := in.dbs[1].rowReads + require.Greater(t, reads, 0, "the first build must read config") + + second := PrefixesAcross(t.Context(), dbs, in.open, cache, readyNow.Add(30*time.Second)) + + assert.Equal(t, reads, in.dbs[1].rowReads, "a second build with an unmoved head must read no rows") + require.Len(t, second.Truncated, 1, "and must still report the clipped read") + assert.Equal(t, Max+2, second.Truncated[0].ShownOf) +}