package beads
import (
"context"
"net/url"
"sort"
"strings"
"time"
"sourcecraft.dev/bigbes/sr-ht-dolt/browse"
)
// --- the cross-database ready set ---------------------------------------------
//
// "What is ready to work" is answerable in each beads database on the instance
// and, until this, nowhere across them — which is the question the split into a
// global tracker plus per-project trackers was supposed to make askable.
//
// This is one function with two consumers: the /ready page renders it, and the
// MCP surface's ready_work with no database named calls it. Two implementations
// would answer differently the first time the ready rule moved.
//
// Authorization is not here. By the time a caller reaches this function it has
// already decided which databases this caller may browse (see the package
// comment: this package renders nothing and authorizes nothing); handing it a
// database is the statement that the caller may read it.
// The bounds every cross-database reading here is held to. They are named for
// /ready because that is the page that needed them first; the prefix index
// behind cross-database issue links is bounded by these same three numbers
// rather than by a second set of its own (see cache.go).
const (
// ReadyMaxDatabases bounds how many databases one call opens. Opening N
// stores per request is exactly what the per-request browse discipline does
// not scale to, and a page that hit this ceiling says so: a silent cap reads
// as "that is everything".
ReadyMaxDatabases = 64
// ReadyCacheTTL is how long a cached projection stands regardless of the head
// hash. The head-hash gate is what makes the cache correct; the TTL is what
// makes it impossible for the cache to be the reason a reader sees yesterday's
// answer.
ReadyCacheTTL = 60 * time.Second
// readyCacheMaxEntries bounds the cache by entry count. A cache is not a
// store: over the ceiling it drops what has expired and, failing that, starts
// again, rather than growing with the instance.
readyCacheMaxEntries = 256
)
// ReadyDatabase names one database to consider. ID is the identity the cache is
// keyed on — the repository row id, which no two databases share and which
// survives a rename.
//
// The field names are OwnerName and Name deliberately: the freshness partial the
// beads views render is handed a database here rather than a repository, and a
// partial that reads .OwnerName must find it on both.
type ReadyDatabase struct {
ID int
OwnerName string
Name string
}
// Slug is the database's "owner/name" address — what ?db= names and what the
// group headers show.
func (d ReadyDatabase) Slug() string { return d.OwnerName + "/" + d.Name }
// ReadySession is the read-only surface one database is read through. It is
// BrowseSession (the rows) plus the three things this aggregation needs that a
// projection of a single, already-opened database does not: the branch list (for
// the head hash the cache gates on), the table list (for the fingerprint), the
// log (for the group's own freshness line) — and Close, because here the
// aggregation owns the session's lifetime.
//
// web's BrowseSession and *browse.DB satisfy it structurally.
type ReadySession interface {
BrowseSession
Branches(ctx context.Context) ([]browse.Branch, error)
Tables(ctx context.Context, refStr string) ([]browse.TableInfo, error)
Log(ctx context.Context, refStr, fromHash string, limit int) ([]browse.CommitInfo, string, error)
Close() error
}
// ReadyOpener opens one database. The returned session is closed by this
// package — a caller that kept it would be hoarding a file handle and a memory
// mapping, which is what per-request opening exists to avoid.
type ReadyOpener func(ctx context.Context, db ReadyDatabase) (ReadySession, error)
// ReadyGroup is one database's ready work.
type ReadyGroup struct {
Database ReadyDatabase
Ref string // the branch the ready set was read from
Head *browse.CommitInfo // that branch's head, or nil when it cannot be read
Cards []Card // ready issues, priority then id
// Truncated says one of the tables this group was projected from — issues,
// dependencies, labels, custom_statuses — exceeded Max and came back clipped.
// The cards below are then the ready work among the rows that were read,
// which is not the same claim as this database's ready work.
//
// It is a row clip and has nothing to do with ReadyView.Capped, which counts
// databases.
Truncated bool
// ShownOf is this database's issues total, clipped or not: what exists,
// against the at most Max rows the projection read. IssuesClipped is the
// comparison callers usually want.
ShownOf int
}
// IssuesClipped reports that this database's issues table itself exceeded Max,
// so the ready rule was applied to its first Max rows only. Truncated is the
// wider fact (any input table was clipped); this is the one that says ready work
// may be missing from the group rather than merely mislabelled.
func (g ReadyGroup) IssuesClipped() bool { return g.ShownOf > Max }
// ReadyTruncation is one database whose ready set was projected from a clipped
// read. It exists separately from ReadyGroup because a database with no group is
// exactly the case that needs saying: a tracker whose ready work sits past Max
// is absent from Groups for the same reason a tracker with no ready work is, and
// without this the two are indistinguishable.
type ReadyTruncation struct {
Database ReadyDatabase
// ShownOf is that database's issues total, clipped or not. A truncation whose
// ShownOf is within Max was caused by one of the other three tables.
ShownOf int
}
// IssuesClipped reports that this database's issues table itself exceeded Max.
func (t ReadyTruncation) IssuesClipped() bool { return t.ShownOf > Max }
// ReadyFailure is one database that could not be read. It carries the error for
// the caller's log and nothing for the reader: an error string on a page is how
// a store path and a dolt internal end up in a browser (sr-ht-dolt-7ta).
type ReadyFailure struct {
Database ReadyDatabase
Err error
}
// ReadyView is the whole answer: the groups, what was considered, and the three
// facts a reader needs in order not to over-read it (the ceiling, that some
// databases could not be read, and that some were read only in part).
type ReadyView struct {
Groups []ReadyGroup
Total int // ready cards across every group, after filtering
Considered int // databases actually opened (or served from cache)
Capped bool // there were more candidates than Max
Max int // ReadyMaxDatabases, so the page can name the number it hit
Failed []ReadyFailure
Filter ReadyFilter
Options ReadyOptions
// Truncated lists every database considered whose rows came back clipped at
// beads.Max, in the order the caller listed them. It is the complete set, and
// deliberately wider than the groups: a database whose ready work sits past
// the cap produces no group at all, and that is the case a per-group flag
// cannot report.
//
// Capped and this are two different bounds and neither implies the other.
// Capped counts databases — there were more trackers than one call may open.
// This counts rows inside a database that was opened and read.
Truncated []ReadyTruncation
// Query is the request's query as parsed, carried so a link can rebuild this
// exact URL with one key replaced (web's withQuery) instead of re-listing the
// parameters it happens to know about.
Query url.Values
}
// ReadyOptions lists the distinct values present across the ready set, so the
// filter dropdowns offer only real choices. Collected before the card filters
// narrow anything, so the options do not shrink as a filter is applied.
type ReadyOptions struct {
Assignees []string
Priorities []string
}
// ReadyFilter is the page's filter state: ?q=, ?assignee=, ?priority= over the
// cards, and a repeatable ?db=<owner>/<name> over the databases.
type ReadyFilter struct {
Query string
Assignee string
Priority string
Databases []string // empty means every database the caller was given
}
// ParseReadyFilter reads the filter out of a request query.
func ParseReadyFilter(q url.Values) ReadyFilter {
f := ReadyFilter{
Query: strings.TrimSpace(q.Get("q")),
Assignee: strings.TrimSpace(q.Get("assignee")),
Priority: strings.TrimSpace(q.Get("priority")),
}
for _, d := range q["db"] {
if d = strings.TrimSpace(d); d != "" {
f.Databases = append(f.Databases, d)
}
}
return f
}
// Active reports whether any filter is set (drives the "Clear" link and the
// empty-page wording).
func (f ReadyFilter) Active() bool {
return f.Query != "" || f.Assignee != "" || f.Priority != "" || len(f.Databases) > 0
}
// selects reports whether a database is one of the ones asked for. No ?db= at
// all means every database the caller was handed.
func (f ReadyFilter) selects(slug string) bool {
if len(f.Databases) == 0 {
return true
}
for _, d := range f.Databases {
if d == slug {
return true
}
}
return false
}
// matches reports whether one ready card passes every set card filter.
func (f ReadyFilter) matches(c Card) bool {
if f.Assignee != "" && c.Assignee != f.Assignee {
return false
}
if f.Priority != "" && c.Priority != f.Priority {
return false
}
if f.Query != "" {
hay := strings.ToLower(c.ID + " " + c.Title)
if !strings.Contains(hay, strings.ToLower(f.Query)) {
return false
}
}
return true
}
// ReadyCache holds one ready projection per database, keyed by the repository id
// and gated on the head hash. The gate, the TTL and the entry ceiling are the
// shared projectionCache's (cache.go), which the prefix index is bounded by too:
// one set of rules, one lifetime.
//
// What it holds is the projection — the ready cards — and never an open
// session. An open store is a file handle and a memory mapping; caching those is
// the thing per-request opening exists to prevent.
type ReadyCache struct {
projectionCache[readyEntry]
}
// readyEntry is one database's cached ready projection. The head it was read at
// and the time it was stored are the cache's, not this struct's.
type readyEntry struct {
ref string // the branch it was read from
beads bool // the fingerprint held — a false entry is a database to skip
// commit is the head commit for the group's freshness line. It cannot go
// stale under an unmoved head, so it is cached with the cards and a cache hit
// reads no log either.
commit *browse.CommitInfo
cards []Card
// read is what the row reads reported about their own completeness. It is
// cached beside the cards for the same reason: a projection served from the
// cache has to say what the read that produced it said, and a cache hit
// reads no row it could learn this from a second time.
read readyRead
}
// readyRead is what one database's row reads reported about their own
// completeness: whether any table came back clipped at Max, and the issues
// table's true total. Every number in it comes from the reads readyCards
// already makes.
type readyRead struct {
truncated bool // some table this projection reads exceeded Max
issuesTotal int // the issues table's reported total, clipped or not
}
// NewReadyCache returns an empty cache. The zero value works too — the map is
// allocated on first store — and this exists for the callers that hold one by
// pointer.
func NewReadyCache() *ReadyCache {
return &ReadyCache{}
}
// ReadyAcross collects the ready set of every database it is handed, grouped by
// database: groups ordered by ready count desc then name, cards inside a group
// by priority then id, and a database with no ready work absent rather than
// shown empty.
//
// now is the clock the TTL is measured against, passed in rather than read here
// for the reason BuildMemories takes one: this package reads no hidden clock,
// and a caller that pins its own gets a deterministic answer.
//
// It returns no error. One database that cannot be opened or read costs itself
// only — it lands in Failed for the caller's log — because a page that 500s
// because the seventeenth store is corrupt answers nothing about the other
// sixteen.
func ReadyAcross(
ctx context.Context,
dbs []ReadyDatabase,
open ReadyOpener,
cache *ReadyCache,
filter ReadyFilter,
now time.Time,
) *ReadyView {
view := &ReadyView{Filter: filter, Max: ReadyMaxDatabases}
// ?db= narrows the candidates before the ceiling applies: a database the
// caller named is the one thing the cap must not be able to drop.
candidates := make([]ReadyDatabase, 0, len(dbs))
for _, d := range dbs {
if filter.selects(d.Slug()) {
candidates = append(candidates, d)
}
}
if len(candidates) > ReadyMaxDatabases {
// The first Max in the order the caller listed them, which is the caller's
// own ordering (newest first, as the listing produces it) and stable
// across requests.
candidates = candidates[:ReadyMaxDatabases]
view.Capped = true
}
view.Considered = len(candidates)
assignees, priorities := map[string]bool{}, map[string]bool{}
for _, d := range candidates {
entry, err := readyProjection(ctx, d, open, cache, now)
if err != nil {
view.Failed = append(view.Failed, ReadyFailure{Database: d, Err: err})
continue
}
if !entry.beads {
// Not a beads database (or a store with no branches at all): skipped
// silently, exactly as the view tabs skip it.
continue
}
if entry.read.truncated {
// Recorded before the card filters and before the empty-group skip
// below: a database whose ready work was left past the cap has no
// group to carry the fact, and it is that database the reader most
// needs named.
view.Truncated = append(view.Truncated, ReadyTruncation{
Database: d,
ShownOf: entry.read.issuesTotal,
})
}
cards := make([]Card, 0, len(entry.cards))
for _, c := range entry.cards {
if c.Assignee != "" {
assignees[c.Assignee] = true
}
if c.Priority != "" {
priorities[c.Priority] = true
}
if filter.matches(c) {
cards = append(cards, c)
}
}
if len(cards) == 0 {
// Nothing ready here: absent from the page rather than an empty group.
continue
}
view.Groups = append(view.Groups, ReadyGroup{
Database: d,
Ref: entry.ref,
Head: entry.commit,
Cards: cards,
Truncated: entry.read.truncated,
ShownOf: entry.read.issuesTotal,
})
view.Total += len(cards)
}
sort.SliceStable(view.Groups, func(i, j int) bool {
ni, nj := len(view.Groups[i].Cards), len(view.Groups[j].Cards)
if ni != nj {
return ni > nj
}
return view.Groups[i].Database.Slug() < view.Groups[j].Database.Slug()
})
view.Options = ReadyOptions{
Assignees: sortedKeys(assignees),
Priorities: sortedKeys(priorities), // single digits sort numerically as strings
}
return view
}
// readyProjection returns one database's ready cards, from the cache when the
// head has not moved and by reading the store otherwise.
//
// The order is the whole point of the gate: open, list branches, and only then
// consult the cache. Opening a session and reading the branch list is cheap;
// reading and projecting issues + dependencies is not, and on a hit neither the
// tables, nor the rows, nor the log are touched.
func readyProjection(
ctx context.Context,
d ReadyDatabase,
open ReadyOpener,
cache *ReadyCache,
now time.Time,
) (readyEntry, error) {
sess, err := open(ctx, d)
if err != nil {
return readyEntry{}, err
}
defer sess.Close()
branches, err := sess.Branches(ctx)
if err != nil {
return readyEntry{}, err
}
ref := browse.DefaultBranch(branches)
if ref == "" {
// A store with no branches carries no tables either: nothing to skip past
// and nothing to cache.
return readyEntry{}, nil
}
head := headHashOf(branches, ref)
if e, ok := cache.lookup(d.ID, head, now); ok {
return e, nil
}
tables, err := sess.Tables(ctx, ref)
if err != nil {
return readyEntry{}, err
}
entry := readyEntry{ref: ref, beads: Applies(tables)}
if !entry.beads {
// Cached too: a database that is not a tracker is not one on the next
// request either, and the fingerprint is worth exactly one table listing.
cache.store(d.ID, head, now, entry)
return entry, nil
}
cards, read, err := readyCards(ctx, sess, ref)
if err != nil {
return readyEntry{}, err
}
entry.cards = cards
entry.read = read
entry.commit = readyHead(ctx, sess, ref)
cache.store(d.ID, head, now, entry)
return entry, nil
}
// headHashOf returns the head hash of the named branch, or "" when the branch
// list does not carry it. An empty hash disables the cache for that database
// rather than letting an entry stand un-gated.
func headHashOf(branches []browse.Branch, ref string) string {
for _, b := range branches {
if b.Name == ref {
return b.Head
}
}
return ""
}
// readyHead reads the head commit of ref for the group's freshness line, and
// returns nil rather than an error on both failure arms. "Ready" read from a
// store that stopped receiving pushes is exactly the claim this page must not
// make silently — but a log that cannot be read may not be the reason the ready
// set is withheld either, so the line is simply absent (the partial renders
// nothing for nil).
func readyHead(ctx context.Context, sess ReadySession, ref string) *browse.CommitInfo {
commits, _, err := sess.Log(ctx, ref, "", 1)
if err != nil || len(commits) == 0 {
return nil
}
return &commits[0]
}
// readyCards projects one database's ready set: the same tables the board reads,
// through the same ready rule (readyRow), sorted priority then id.
//
// It also returns what those reads said about their own completeness. All four
// tables count towards it: a clipped issues table leaves ready work unread, and
// a clipped dependencies, labels or custom_statuses table changes the verdict on
// the issues that were read — a blocking edge past the cap is a card called
// ready that is not.
func readyCards(ctx context.Context, sess BrowseSession, ref string) ([]Card, readyRead, error) {
issues, issuesTotal, err := readRows(ctx, sess, ref, "issues")
if err != nil {
return nil, readyRead{}, err
}
deps, depsTotal, err := readRows(ctx, sess, ref, "dependencies")
if err != nil {
return nil, readyRead{}, err
}
labels, labelsTotal, _ := readRowsOptional(ctx, sess, ref, "labels")
statuses, statusesTotal, _ := readRowsOptional(ctx, sess, ref, "custom_statuses")
read := readyRead{
truncated: issuesTotal > Max || depsTotal > Max ||
labelsTotal > Max || statusesTotal > Max,
issuesTotal: issuesTotal,
}
catByStatus := indexStatusCategories(statuses)
issueCols := indexCols(issues.Columns)
catByIssue := indexIssueCategories(issues, issueCols, catByStatus)
depIdx := indexDeps(deps, catByIssue)
labelsByIssue := indexLabels(labels)
var cards []Card
for _, r := range rowsOf(issues) {
id := cell(issueCols, r, "id")
cat := catByIssue[id]
blocked := truthy(cell(issueCols, r, "is_blocked")) || depIdx.blockedOpen[id]
if !readyRow(cat, blocked, r, issueCols) {
continue
}
cards = append(cards, Card{
ID: id,
Title: cell(issueCols, r, "title"),
Type: cell(issueCols, r, "issue_type"),
Priority: cell(issueCols, r, "priority"),
Assignee: cell(issueCols, r, "assignee"),
Labels: labelsByIssue[id],
BlockedBy: depIdx.blockedByCount[id],
Blocks: depIdx.blocksCount[id],
Ready: true,
Category: cat,
})
}
sortReadyCards(cards)
return cards, read, nil
}
// sortReadyCards orders a group by priority (0 = highest first), then id. The
// board sorts by created_at between the two; here the id is the tie-break,
// because across databases the created_at of one tracker says nothing about the
// order of another's.
func sortReadyCards(cards []Card) {
sort.SliceStable(cards, func(i, j int) bool {
pi, pj := priorityRank(cards[i].Priority), priorityRank(cards[j].Priority)
if pi != pj {
return pi < pj
}
return cards[i].ID < cards[j].ID
})
}