~bigbes/sr-ht-dolt

ref: b6a015582614b2a0ec7d07984db1ce7e8e9f8523 sr-ht-dolt/beads/ready.go -rw-r--r-- 16.5 KiB
b6a01558 — Eugene Blikh web: drop the author from a memory's revision line 5 days ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
package beads

import (
	"context"
	"net/url"
	"sort"
	"strings"
	"sync"
	"time"

	"sourcecraft.dev/bigbes/sr-ht-dolt/browse"
)

// --- the cross-database ready set ---------------------------------------------
//
// "What is ready to work" is answerable in each beads database on the instance
// and, until this, nowhere across them — which is the question the split into a
// global tracker plus per-project trackers was supposed to make askable.
//
// This is one function with two consumers: the /ready page renders it, and the
// MCP surface's ready_work with no database named calls it. Two implementations
// would answer differently the first time the ready rule moved.
//
// Authorization is not here. By the time a caller reaches this function it has
// already decided which databases this caller may browse (see the package
// comment: this package renders nothing and authorizes nothing); handing it a
// database is the statement that the caller may read it.

const (
	// ReadyMaxDatabases bounds how many databases one call opens. Opening N
	// stores per request is exactly what the per-request browse discipline does
	// not scale to, and a page that hit this ceiling says so: a silent cap reads
	// as "that is everything".
	ReadyMaxDatabases = 64

	// ReadyCacheTTL is how long a cached projection stands regardless of the head
	// hash. The head-hash gate is what makes the cache correct; the TTL is what
	// makes it impossible for the cache to be the reason a reader sees yesterday's
	// answer.
	ReadyCacheTTL = 60 * time.Second

	// readyCacheMaxEntries bounds the cache by entry count. A cache is not a
	// store: over the ceiling it drops what has expired and, failing that, starts
	// again, rather than growing with the instance.
	readyCacheMaxEntries = 256
)

// ReadyDatabase names one database to consider. ID is the identity the cache is
// keyed on — the repository row id, which no two databases share and which
// survives a rename.
//
// The field names are OwnerName and Name deliberately: the freshness partial the
// beads views render is handed a database here rather than a repository, and a
// partial that reads .OwnerName must find it on both.
type ReadyDatabase struct {
	ID        int
	OwnerName string
	Name      string
}

// Slug is the database's "owner/name" address — what ?db= names and what the
// group headers show.
func (d ReadyDatabase) Slug() string { return d.OwnerName + "/" + d.Name }

// ReadySession is the read-only surface one database is read through. It is
// BrowseSession (the rows) plus the three things this aggregation needs that a
// projection of a single, already-opened database does not: the branch list (for
// the head hash the cache gates on), the table list (for the fingerprint), the
// log (for the group's own freshness line) — and Close, because here the
// aggregation owns the session's lifetime.
//
// web's BrowseSession and *browse.DB satisfy it structurally.
type ReadySession interface {
	BrowseSession
	Branches(ctx context.Context) ([]browse.Branch, error)
	Tables(ctx context.Context, refStr string) ([]browse.TableInfo, error)
	Log(ctx context.Context, refStr, fromHash string, limit int) ([]browse.CommitInfo, string, error)
	Close() error
}

// ReadyOpener opens one database. The returned session is closed by this
// package — a caller that kept it would be hoarding a file handle and a memory
// mapping, which is what per-request opening exists to avoid.
type ReadyOpener func(ctx context.Context, db ReadyDatabase) (ReadySession, error)

// ReadyGroup is one database's ready work.
type ReadyGroup struct {
	Database ReadyDatabase
	Ref      string             // the branch the ready set was read from
	Head     *browse.CommitInfo // that branch's head, or nil when it cannot be read
	Cards    []Card             // ready issues, priority then id
}

// ReadyFailure is one database that could not be read. It carries the error for
// the caller's log and nothing for the reader: an error string on a page is how
// a store path and a dolt internal end up in a browser (sr-ht-dolt-7ta).
type ReadyFailure struct {
	Database ReadyDatabase
	Err      error
}

// ReadyView is the whole answer: the groups, what was considered, and the two
// facts a reader needs in order not to over-read it (the ceiling, and that some
// databases could not be read).
type ReadyView struct {
	Groups     []ReadyGroup
	Total      int  // ready cards across every group, after filtering
	Considered int  // databases actually opened (or served from cache)
	Capped     bool // there were more candidates than Max
	Max        int  // ReadyMaxDatabases, so the page can name the number it hit
	Failed     []ReadyFailure
	Filter     ReadyFilter
	Options    ReadyOptions
	// Query is the request's query as parsed, carried so a link can rebuild this
	// exact URL with one key replaced (web's withQuery) instead of re-listing the
	// parameters it happens to know about.
	Query url.Values
}

// ReadyOptions lists the distinct values present across the ready set, so the
// filter dropdowns offer only real choices. Collected before the card filters
// narrow anything, so the options do not shrink as a filter is applied.
type ReadyOptions struct {
	Assignees  []string
	Priorities []string
}

// ReadyFilter is the page's filter state: ?q=, ?assignee=, ?priority= over the
// cards, and a repeatable ?db=<owner>/<name> over the databases.
type ReadyFilter struct {
	Query     string
	Assignee  string
	Priority  string
	Databases []string // empty means every database the caller was given
}

// ParseReadyFilter reads the filter out of a request query.
func ParseReadyFilter(q url.Values) ReadyFilter {
	f := ReadyFilter{
		Query:    strings.TrimSpace(q.Get("q")),
		Assignee: strings.TrimSpace(q.Get("assignee")),
		Priority: strings.TrimSpace(q.Get("priority")),
	}
	for _, d := range q["db"] {
		if d = strings.TrimSpace(d); d != "" {
			f.Databases = append(f.Databases, d)
		}
	}
	return f
}

// Active reports whether any filter is set (drives the "Clear" link and the
// empty-page wording).
func (f ReadyFilter) Active() bool {
	return f.Query != "" || f.Assignee != "" || f.Priority != "" || len(f.Databases) > 0
}

// selects reports whether a database is one of the ones asked for. No ?db= at
// all means every database the caller was handed.
func (f ReadyFilter) selects(slug string) bool {
	if len(f.Databases) == 0 {
		return true
	}
	for _, d := range f.Databases {
		if d == slug {
			return true
		}
	}
	return false
}

// matches reports whether one ready card passes every set card filter.
func (f ReadyFilter) matches(c Card) bool {
	if f.Assignee != "" && c.Assignee != f.Assignee {
		return false
	}
	if f.Priority != "" && c.Priority != f.Priority {
		return false
	}
	if f.Query != "" {
		hay := strings.ToLower(c.ID + " " + c.Title)
		if !strings.Contains(hay, strings.ToLower(f.Query)) {
			return false
		}
	}
	return true
}

// ReadyCache holds one projection per database, keyed by the repository id and
// gated on the head hash. It is deliberately small: a mutex, a map, and the two
// bounds above.
//
// What it holds is the projection — the ready cards — and never an open
// session. An open store is a file handle and a memory mapping; caching those is
// the thing per-request opening exists to prevent.
//
// The zero value is not usable; call NewReadyCache. A nil *ReadyCache is a
// programming error and panics on first use rather than quietly disabling the
// bound it exists to enforce.
type ReadyCache struct {
	mu      sync.Mutex
	entries map[int]readyEntry
}

// readyEntry is one database's cached projection.
type readyEntry struct {
	head  string    // the branch head the projection was read at; the gate
	ref   string    // the branch it was read from
	at    time.Time // when it was stored; the TTL
	beads bool      // the fingerprint held — a false entry is a database to skip
	// commit is the head commit for the group's freshness line. It cannot go
	// stale under an unmoved head, so it is cached with the cards and a cache hit
	// reads no log either.
	commit *browse.CommitInfo
	cards  []Card
}

// NewReadyCache returns an empty cache.
func NewReadyCache() *ReadyCache {
	return &ReadyCache{entries: map[int]readyEntry{}}
}

// lookup returns the cached projection for a database when it was read at the
// same head and has not expired. Both conditions, not either: the head hash is
// what makes it correct, the TTL is what makes it bounded.
func (c *ReadyCache) lookup(id int, head string, now time.Time) (readyEntry, bool) {
	if head == "" {
		// A database whose head cannot be named cannot be gated on one.
		return readyEntry{}, false
	}
	c.mu.Lock()
	defer c.mu.Unlock()
	e, ok := c.entries[id]
	if !ok || e.head != head || now.Sub(e.at) >= ReadyCacheTTL {
		return readyEntry{}, false
	}
	return e, true
}

// store records a projection, dropping expired entries — and, if that was not
// enough, everything — when the map is at its ceiling.
func (c *ReadyCache) store(id int, e readyEntry) {
	if e.head == "" {
		return
	}
	c.mu.Lock()
	defer c.mu.Unlock()
	if len(c.entries) >= readyCacheMaxEntries {
		for k, old := range c.entries {
			if e.at.Sub(old.at) >= ReadyCacheTTL {
				delete(c.entries, k)
			}
		}
		if len(c.entries) >= readyCacheMaxEntries {
			c.entries = make(map[int]readyEntry, readyCacheMaxEntries)
		}
	}
	c.entries[id] = e
}

// ReadyAcross collects the ready set of every database it is handed, grouped by
// database: groups ordered by ready count desc then name, cards inside a group
// by priority then id, and a database with no ready work absent rather than
// shown empty.
//
// now is the clock the TTL is measured against, passed in rather than read here
// for the reason BuildMemories takes one: this package reads no hidden clock,
// and a caller that pins its own gets a deterministic answer.
//
// It returns no error. One database that cannot be opened or read costs itself
// only — it lands in Failed for the caller's log — because a page that 500s
// because the seventeenth store is corrupt answers nothing about the other
// sixteen.
func ReadyAcross(
	ctx context.Context,
	dbs []ReadyDatabase,
	open ReadyOpener,
	cache *ReadyCache,
	filter ReadyFilter,
	now time.Time,
) *ReadyView {
	view := &ReadyView{Filter: filter, Max: ReadyMaxDatabases}

	// ?db= narrows the candidates before the ceiling applies: a database the
	// caller named is the one thing the cap must not be able to drop.
	candidates := make([]ReadyDatabase, 0, len(dbs))
	for _, d := range dbs {
		if filter.selects(d.Slug()) {
			candidates = append(candidates, d)
		}
	}
	if len(candidates) > ReadyMaxDatabases {
		// The first Max in the order the caller listed them, which is the caller's
		// own ordering (newest first, as the listing produces it) and stable
		// across requests.
		candidates = candidates[:ReadyMaxDatabases]
		view.Capped = true
	}
	view.Considered = len(candidates)

	assignees, priorities := map[string]bool{}, map[string]bool{}
	for _, d := range candidates {
		entry, err := readyProjection(ctx, d, open, cache, now)
		if err != nil {
			view.Failed = append(view.Failed, ReadyFailure{Database: d, Err: err})
			continue
		}
		if !entry.beads {
			// Not a beads database (or a store with no branches at all): skipped
			// silently, exactly as the view tabs skip it.
			continue
		}
		cards := make([]Card, 0, len(entry.cards))
		for _, c := range entry.cards {
			if c.Assignee != "" {
				assignees[c.Assignee] = true
			}
			if c.Priority != "" {
				priorities[c.Priority] = true
			}
			if filter.matches(c) {
				cards = append(cards, c)
			}
		}
		if len(cards) == 0 {
			// Nothing ready here: absent from the page rather than an empty group.
			continue
		}
		view.Groups = append(view.Groups, ReadyGroup{
			Database: d,
			Ref:      entry.ref,
			Head:     entry.commit,
			Cards:    cards,
		})
		view.Total += len(cards)
	}

	sort.SliceStable(view.Groups, func(i, j int) bool {
		ni, nj := len(view.Groups[i].Cards), len(view.Groups[j].Cards)
		if ni != nj {
			return ni > nj
		}
		return view.Groups[i].Database.Slug() < view.Groups[j].Database.Slug()
	})
	view.Options = ReadyOptions{
		Assignees:  sortedKeys(assignees),
		Priorities: sortedKeys(priorities), // single digits sort numerically as strings
	}
	return view
}

// readyProjection returns one database's ready cards, from the cache when the
// head has not moved and by reading the store otherwise.
//
// The order is the whole point of the gate: open, list branches, and only then
// consult the cache. Opening a session and reading the branch list is cheap;
// reading and projecting issues + dependencies is not, and on a hit neither the
// tables, nor the rows, nor the log are touched.
func readyProjection(
	ctx context.Context,
	d ReadyDatabase,
	open ReadyOpener,
	cache *ReadyCache,
	now time.Time,
) (readyEntry, error) {
	sess, err := open(ctx, d)
	if err != nil {
		return readyEntry{}, err
	}
	defer sess.Close()

	branches, err := sess.Branches(ctx)
	if err != nil {
		return readyEntry{}, err
	}
	ref := browse.DefaultBranch(branches)
	if ref == "" {
		// A store with no branches carries no tables either: nothing to skip past
		// and nothing to cache.
		return readyEntry{}, nil
	}
	head := headHashOf(branches, ref)
	if e, ok := cache.lookup(d.ID, head, now); ok {
		return e, nil
	}

	tables, err := sess.Tables(ctx, ref)
	if err != nil {
		return readyEntry{}, err
	}
	entry := readyEntry{head: head, ref: ref, at: now, beads: Applies(tables)}
	if !entry.beads {
		// Cached too: a database that is not a tracker is not one on the next
		// request either, and the fingerprint is worth exactly one table listing.
		cache.store(d.ID, entry)
		return entry, nil
	}

	cards, err := readyCards(ctx, sess, ref)
	if err != nil {
		return readyEntry{}, err
	}
	entry.cards = cards
	entry.commit = readyHead(ctx, sess, ref)
	cache.store(d.ID, entry)
	return entry, nil
}

// headHashOf returns the head hash of the named branch, or "" when the branch
// list does not carry it. An empty hash disables the cache for that database
// rather than letting an entry stand un-gated.
func headHashOf(branches []browse.Branch, ref string) string {
	for _, b := range branches {
		if b.Name == ref {
			return b.Head
		}
	}
	return ""
}

// readyHead reads the head commit of ref for the group's freshness line, and
// returns nil rather than an error on both failure arms. "Ready" read from a
// store that stopped receiving pushes is exactly the claim this page must not
// make silently — but a log that cannot be read may not be the reason the ready
// set is withheld either, so the line is simply absent (the partial renders
// nothing for nil).
func readyHead(ctx context.Context, sess ReadySession, ref string) *browse.CommitInfo {
	commits, _, err := sess.Log(ctx, ref, "", 1)
	if err != nil || len(commits) == 0 {
		return nil
	}
	return &commits[0]
}

// readyCards projects one database's ready set: the same tables the board reads,
// through the same ready rule (readyRow), sorted priority then id.
func readyCards(ctx context.Context, sess BrowseSession, ref string) ([]Card, error) {
	issues, _, err := readRows(ctx, sess, ref, "issues")
	if err != nil {
		return nil, err
	}
	deps, _, err := readRows(ctx, sess, ref, "dependencies")
	if err != nil {
		return nil, err
	}
	labels, _, _ := readRowsOptional(ctx, sess, ref, "labels")
	statuses, _, _ := readRowsOptional(ctx, sess, ref, "custom_statuses")

	catByStatus := indexStatusCategories(statuses)
	issueCols := indexCols(issues.Columns)
	catByIssue := indexIssueCategories(issues, issueCols, catByStatus)
	depIdx := indexDeps(deps, catByIssue)
	labelsByIssue := indexLabels(labels)

	var cards []Card
	for _, r := range issues.Rows {
		id := cell(issueCols, r, "id")
		cat := catByIssue[id]
		blocked := truthy(cell(issueCols, r, "is_blocked")) || depIdx.blockedOpen[id]
		if !readyRow(cat, blocked, r, issueCols) {
			continue
		}
		cards = append(cards, Card{
			ID:        id,
			Title:     cell(issueCols, r, "title"),
			Type:      cell(issueCols, r, "issue_type"),
			Priority:  cell(issueCols, r, "priority"),
			Assignee:  cell(issueCols, r, "assignee"),
			Labels:    labelsByIssue[id],
			BlockedBy: depIdx.blockedByCount[id],
			Blocks:    depIdx.blocksCount[id],
			Ready:     true,
			Category:  cat,
		})
	}
	sortReadyCards(cards)
	return cards, nil
}

// sortReadyCards orders a group by priority (0 = highest first), then id. The
// board sorts by created_at between the two; here the id is the tie-break,
// because across databases the created_at of one tracker says nothing about the
// order of another's.
func sortReadyCards(cards []Card) {
	sort.SliceStable(cards, func(i, j int) bool {
		pi, pj := priorityRank(cards[i].Priority), priorityRank(cards[j].Priority)
		if pi != pj {
			return pi < pj
		}
		return cards[i].ID < cards[j].ID
	})
}