A => +124 -0
@@ 0,0 1,124 @@
package search
import (
"fmt"
"path"
"strings"
"sync"
"sourcecraft.dev/bigbes/sr-ht-spec/doc"
"sourcecraft.dev/bigbes/sr-ht-spec/gitx"
)
// renderer is shared: doc.Renderer is documented as reusable and
// concurrency-safe, and building one per extraction would rebuild the whole
// goldmark pipeline for every space.
var renderer = sync.OnceValue(doc.NewRenderer)
// Bodies keys a revision's documents by tree path, which is the shape Extract
// wants them in. It pairs with doc.FromDocuments: the same []gitx.Document
// builds the Archive and supplies the text.
func Bodies(docs []gitx.Document) map[string][]byte {
m := make(map[string][]byte, len(docs))
for _, d := range docs {
m[d.Path] = d.Data
}
return m
}
// Extract projects one space at one revision into the documents the index
// stores. bodies holds each page's raw markdown — frontmatter included, exactly
// gitx.Document.Data — keyed by doc.Page.Path.
//
// A page in the archive with no body in bodies is an error, not a page indexed
// with an empty body. The two are indistinguishable once indexed, and the
// second is how a document silently stops being findable.
//
// Three document shapes come out, matching what doc/ models:
//
// - an ordinary document, indexed as its frontmatter projected to "key:
// value" lines followed by its rendered plain text. The frontmatter is in
// there because tags, owners and summaries render as chips rather than
// prose, and a search for one of them should still find the document.
// - a catalog (`type: catalog`, or index.md), indexed by title and section
// only. A catalog is a page of one-line descriptions of other documents;
// indexed whole, a query lands on the description instead of on the
// document that owns it.
// - an activity log (`type: log`, or log.md), which contributes its own
// title-only document plus one document per dated entry. A hit anywhere in
// a log otherwise resolves to the whole log; split, each entry is the size
// of the thing it describes and carries an anchor into it.
func Extract(arc *doc.Archive, bodies map[string][]byte) ([]Document, error) {
if arc == nil {
return nil, fmt.Errorf("search: Extract needs an archive")
}
pages := arc.All()
out := make([]Document, 0, len(pages))
r := renderer()
for _, p := range pages {
src, ok := bodies[p.Path]
if !ok {
return nil, fmt.Errorf("search: no body supplied for %s in %s", p.Path, arc.Space)
}
front, body := doc.ParseFront(src)
dir := path.Dir(p.Path)
if dir == "." {
dir = ""
}
res := r.Render(body, dir, arc)
d := Document{
Space: arc.Space,
ID: p.ID,
Rev: arc.Rev,
Path: p.Path,
Section: p.Section,
Title: p.Title,
Text: front.SearchText() + res.PlainText,
}
switch p.Kind {
case doc.KindCatalog:
d.Text = ""
case doc.KindLog:
d.Section = doc.LogSection
d.Text = ""
out = append(out, d)
out = append(out, logEntries(arc, p, body)...)
continue
}
out = append(out, d)
}
return out, nil
}
// logEntries splits an activity log into one indexable document per dated
// entry.
//
// The entry ids doc.SplitLog produces are "log#<date>-<n>", named after
// warren's single vault-wide log. In a space that is not unique: a second
// document marked `type: log` — or simply a second file named log.md in another
// directory — produces the same ids, and in one index the same ids are the same
// documents, so one log would silently overwrite the other. The owning page's
// id is therefore substituted for the "log" prefix, which is a no-op for a log
// whose page id is in fact "log" and disambiguates every other case. The result
// also resolves better: "notes/dev-log#2026-05-31-1" names the document the
// entry is in.
func logEntries(arc *doc.Archive, p *doc.Page, body []byte) []Document {
entries := doc.SplitLog(body)
out := make([]Document, 0, len(entries))
for _, e := range entries {
suffix := strings.TrimPrefix(e.ID, "log#")
out = append(out, Document{
Space: arc.Space,
ID: p.ID + "#" + suffix,
Rev: arc.Rev,
Path: p.Path,
Anchor: e.Anchor,
Section: doc.LogSection,
Title: e.Date + " " + e.Title,
Text: e.SearchText(),
})
}
return out
}
A => +151 -0
@@ 0,0 1,151 @@
package search
import (
"testing"
"github.com/stretchr/testify/require"
"sourcecraft.dev/bigbes/sr-ht-spec/doc"
"sourcecraft.dev/bigbes/sr-ht-spec/gitx"
)
func byID(docs []Document) map[string]Document {
m := make(map[string]Document, len(docs))
for _, d := range docs {
m[d.ID] = d
}
return m
}
func TestExtractCarriesSpaceRevisionAndPath(t *testing.T) {
c := newCorpus(t, "~bigbes/specs", "8f14e45fceea167a5a36dedd4bea2543").
add("specs/storage.md", "---\nid: SPEC-0001\ntitle: Storage model\n---\n\nBodies live in git.\n")
docs := c.extract(t)
require.Len(t, docs, 1)
require.Equal(t, c.Space, docs[0].Space)
require.Equal(t, "8f14e45fceea167a5a36dedd4bea2543", docs[0].Rev)
require.Equal(t, "specs/storage.md", docs[0].Path)
require.Equal(t, "specs", docs[0].Section)
require.Equal(t, "SPEC-0001", docs[0].ID)
require.Empty(t, docs[0].Anchor)
}
// Frontmatter is prepended to the searchable text, so a tag or an owner finds
// the document even though neither is prose. Carried over from warren.
func TestExtractIndexesFrontmatterAsText(t *testing.T) {
c := newCorpus(t, "~bigbes/specs", "rev1").
add("specs/storage.md", "---\nid: SPEC-0001\ntitle: Storage model\n"+
"tags: [storage, review]\nowners: ['~bigbes']\nsummary: One tier, git objects only\n---\n\n"+
"Bodies live in git.\n")
docs := c.extract(t)
require.Len(t, docs, 1)
require.Contains(t, docs[0].Text, "storage")
require.Contains(t, docs[0].Text, "review")
require.Contains(t, docs[0].Text, "One tier, git objects only")
require.Contains(t, docs[0].Text, "Bodies live in git.")
}
// A catalog is a page of one-line descriptions of other documents. Indexed
// whole, a query lands on the description instead of the document that owns it.
func TestExtractIndexesACatalogByTitleOnly(t *testing.T) {
c := newCorpus(t, "~bigbes/specs", "rev1").
add("specs/index.md", "# Specifications\n\n- [[SPEC-0001]] — the storage model\n- [[SPEC-0002]] — review\n").
add("specs/storage.md", "---\nid: SPEC-0001\ntitle: Storage model\n---\n\nBodies live in git.\n")
docs := byID(c.extract(t))
catalog, ok := docs["specs/index"]
require.True(t, ok, "the catalog is still indexed: %v", docs)
require.Equal(t, doc.KindCatalog, kindOf(t, c, "specs/index.md"))
require.Empty(t, catalog.Text, "a catalog contributes no body")
require.Equal(t, "Specifications", catalog.Title)
}
func kindOf(t *testing.T, c *corpus, path string) doc.PageKind {
t.Helper()
arc := doc.FromDocuments(c.Space, c.Rev, c.docs)
p, ok := arc.ByPath(path)
require.True(t, ok)
return p.Kind
}
// An activity log contributes its own title-only document plus one per dated
// entry, each anchored into the log it came from. Carried over from warren,
// where a hit anywhere in a 248 KB log resolved to the whole file.
func TestExtractSplitsAnActivityLogIntoEntries(t *testing.T) {
c := newCorpus(t, "~bigbes/specs", "rev1").
add("notes/log.md", "# Work log\n\n"+
"## [2026-05-31] ingest | Imported the RFC set\nPulled 40 documents in from the old wiki.\n\n"+
"## [2026-05-30] lint | Fixed frontmatter\nEvery document now carries a status.\n")
docs := byID(c.extract(t))
require.Len(t, docs, 3)
page, ok := docs["notes/log"]
require.True(t, ok)
require.Equal(t, doc.LogSection, page.Section, "the log page moves to the log section")
require.Empty(t, page.Text)
first, ok := docs["notes/log#2026-05-31-1"]
require.True(t, ok, "entries are keyed under their own document: %v", docs)
require.Equal(t, doc.LogSection, first.Section)
require.Equal(t, "notes/log.md", first.Path, "an entry points back at the log it lives in")
require.Equal(t, "e-2026-05-31-1", first.Anchor)
require.Equal(t, "2026-05-31 Imported the RFC set", first.Title)
require.Contains(t, first.Text, "Pulled 40 documents in from the old wiki.")
require.NotContains(t, first.Text, "Fixed frontmatter", "entries do not bleed into each other")
}
// doc.SplitLog names entries "log#<date>-<n>", after warren's single vault-wide
// log. Two logs in one space would then produce colliding ids, and in one index
// colliding ids are the same document — one log would silently overwrite the
// other. Entry ids are namespaced by their own document to prevent it.
func TestExtractNamespacesLogEntriesPerDocument(t *testing.T) {
entry := "\n## [2026-05-31] ingest | Same day, two logs\nBody.\n"
c := newCorpus(t, "~bigbes/specs", "rev1").
add("log.md", "# Space log\n"+entry).
add("notes/log.md", "# Notes log\n"+entry)
docs := byID(c.extract(t))
require.Contains(t, docs, "log#2026-05-31-1", "a log whose page id is \"log\" keeps warren's ids")
require.Contains(t, docs, "notes/log#2026-05-31-1")
require.Len(t, docs, 4, "two log pages and two entries, none of them merged")
}
// A page in the archive with no body is a caller bug. Indexing it with an empty
// body would leave a document that exists and is unfindable, which is the
// failure mode that surfaces months later.
func TestExtractRefusesAPageWithNoBody(t *testing.T) {
c := newCorpus(t, "~bigbes/specs", "rev1").
add("specs/storage.md", "---\nid: SPEC-0001\ntitle: Storage model\n---\n\nBodies live in git.\n")
arc := doc.FromDocuments(c.Space, c.Rev, c.docs)
_, err := Extract(arc, map[string][]byte{})
require.ErrorContains(t, err, "no body supplied for specs/storage.md")
}
// A document whose frontmatter core rejects still renders and still indexes:
// --push-option=skip-validation means a broken header can reach the approved
// branch, and refusing to index it would turn a typo into a silent hole.
func TestExtractIndexesADocumentWithABrokenHeader(t *testing.T) {
c := newCorpus(t, "~bigbes/specs", "rev1").
add("specs/broken.md", "---\nid: [not, a, string\n---\n\n# Broken but readable\n\nThe body is still prose.\n")
docs := c.extract(t)
require.Len(t, docs, 1)
require.Equal(t, "specs/broken", docs[0].ID, "it falls back to its path")
require.Contains(t, docs[0].Text, "The body is still prose.")
}
func TestBodiesKeysByPath(t *testing.T) {
got := Bodies([]gitx.Document{
{Path: "a.md", Data: []byte("one")},
{Path: "b/c.md", Data: []byte("two")},
})
require.Equal(t, map[string][]byte{"a.md": []byte("one"), "b/c.md": []byte("two")}, got)
}
func TestExtractNeedsAnArchive(t *testing.T) {
_, err := Extract(nil, nil)
require.ErrorContains(t, err, "needs an archive")
}
A search/fixture_test.go => search/fixture_test.go +82 -0
@@ 0,0 1,82 @@
+package search
+
+import (
+ "context"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "sourcecraft.dev/bigbes/sr-ht-spec/core"
+ "sourcecraft.dev/bigbes/sr-ht-spec/doc"
+ "sourcecraft.dev/bigbes/sr-ht-spec/gitx"
+)
+
+// space is a shorthand for a valid SpaceRef in tests.
+func space(t *testing.T, s string) core.SpaceRef {
+ t.Helper()
+ sp, err := core.ParseSpaceRef(s)
+ require.NoError(t, err)
+ return sp
+}
+
+// corpus is a space's worth of documents, built in memory. No repository, no
+// checkout: doc.FromDocuments is the seam that takes a document set directly,
+// and gitx.Document is exactly what a tree walk yields.
+type corpus struct {
+ Space core.SpaceRef
+ Rev string
+ docs []gitx.Document
+}
+
+func newCorpus(t *testing.T, ref, rev string) *corpus {
+ t.Helper()
+ return &corpus{Space: space(t, ref), Rev: rev}
+}
+
+func (c *corpus) add(path, body string) *corpus {
+ c.docs = append(c.docs, gitx.Document{Path: path, Data: []byte(body)})
+ return c
+}
+
+// extract builds the archive and projects it into indexable documents, which is
+// the whole of what service/ will do between a tree walk and a rebuild.
+func (c *corpus) extract(t *testing.T) []Document {
+ t.Helper()
+ arc := doc.FromDocuments(c.Space, c.Rev, c.docs)
+ docs, err := Extract(arc, Bodies(c.docs))
+ require.NoError(t, err)
+ return docs
+}
+
+// openIndex opens an empty index in a temp dir, closed at the end of the test.
+func openIndex(t *testing.T) *Index {
+ t.Helper()
+ idx, err := Open(filepath.Join(t.TempDir(), "spec.bleve"))
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, idx.Close()) })
+ return idx
+}
+
+// indexCorpus opens an index and rebuilds every corpus into it.
+func indexCorpus(t *testing.T, corpora ...*corpus) *Index {
+ t.Helper()
+ idx := openIndex(t)
+ for _, c := range corpora {
+ _, err := idx.RebuildSpace(context.Background(), c.Space, c.extract(t))
+ require.NoError(t, err)
+ }
+ return idx
+}
+
+// hitIDs is the ordered list of document ids a search returned.
+func hitIDs(t *testing.T, idx *Index, q Query) []string {
+ t.Helper()
+ res, err := idx.Search(context.Background(), q)
+ require.NoError(t, err)
+ ids := make([]string, 0, len(res.Hits))
+ for _, h := range res.Hits {
+ ids = append(ids, h.ID)
+ }
+ return ids
+}
A search/index.go => search/index.go +379 -0
@@ 0,0 1,379 @@
+package search
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "sync"
+ "time"
+
+ "github.com/blevesearch/bleve/v2"
+
+ "sourcecraft.dev/bigbes/sr-ht-spec/core"
+)
+
+// Document is one unit of the index: a whole document, or one dated entry of an
+// activity log. Extract produces these from a doc.Archive; the index stores
+// them and hands back Hits shaped the same way.
+type Document struct {
+ // Space is what makes the one global index filterable per project. Every
+ // document carries it, and a project query is a term filter over the set.
+ Space core.SpaceRef
+ // ID is the document's id within its space: doc.Page.ID, or
+ // "<page id>#<date>-<n>" for one entry of an activity log.
+ ID string
+ // Rev is the revision the document was read at, carried through so a hit
+ // can be turned into a pinned `?rev=` URL rather than a link to whatever
+ // the branch says now.
+ Rev string
+ // Path is the document's path in the git tree. For a log entry it is the
+ // path of the log document the entry came out of.
+ Path string
+ // Anchor is the heading anchor within Path a hit should land on. Empty for
+ // an ordinary document, set for a log entry.
+ Anchor string
+ // Section is the top-level directory the document lives under, or
+ // doc.LogSection for an activity log and its entries.
+ Section string
+ Title string
+ // Text is everything searchable: the frontmatter projected to "key: value"
+ // lines followed by the rendered plain text. It is split by language and
+ // stored in the analyzed fields; it is not stored verbatim under its own
+ // name.
+ Text string
+}
+
+// Index is the one global bleve index, shared by every space and every project.
+//
+// One index, not one per project and not one per space. Per-project indexes
+// were specified and then retracted in the design for a concrete reason: with N
+// projects every merge fans out to N rebuilds and adding a space to a project
+// forces one, while the "everything" project is a second full copy of the
+// corpus. Here a project is a filter — see Query.Spaces — so a merge touches
+// one index and every project containing the space sees the change for free.
+//
+// bleve is single-writer. One process holds the index open, which is why the
+// push hooks are RPC shims into the daemon rather than separate processes.
+// Within that process an Index is safe for concurrent use: searches share a
+// read lock, and a rebuild takes the write lock.
+type Index struct {
+ path string
+
+ mu sync.RWMutex
+ idx bleve.Index
+}
+
+// batchSize is how many documents are buffered before a batch is flushed
+// during a full rebuild. Carried over from warren.
+const batchSize = 200
+
+// Stats reports what a rebuild did and how long it took.
+//
+// The design absorbs warren's batch full rebuild deliberately — at tens of
+// documents a day, incremental indexing is machinery bought against a cost
+// nobody has measured — and asks for the duration to be instrumented so the
+// decision to revisit is triggered by a number rather than a hunch. That is
+// what Took is for: it is the trigger, and callers are expected to log it.
+type Stats struct {
+ // Spaces is how many distinct spaces the rebuild wrote.
+ Spaces int
+ // Indexed is how many documents were written.
+ Indexed int
+ // Deleted is how many stale documents were removed: documents that were in
+ // the index for a rebuilt space and are not in the new document set.
+ Deleted int
+ Took time.Duration
+}
+
+func (s Stats) String() string {
+ return fmt.Sprintf("spaces=%d indexed=%d deleted=%d took=%s",
+ s.Spaces, s.Indexed, s.Deleted, s.Took.Round(time.Millisecond))
+}
+
+// Open opens the global index at path, creating an empty one if it is not
+// there. The index is a pure cache: deleting the directory and letting Open
+// recreate it, followed by RebuildAll, is always a valid repair.
+func Open(path string) (*Index, error) {
+ if path == "" {
+ return nil, errors.New("search: index path is required")
+ }
+ idx, err := bleve.Open(path)
+ switch {
+ case errors.Is(err, bleve.ErrorIndexPathDoesNotExist):
+ idx, err = bleve.New(path, buildMapping())
+ if err != nil {
+ return nil, fmt.Errorf("search: create index at %s: %w", path, err)
+ }
+ case err != nil:
+ return nil, fmt.Errorf("search: open index at %s: %w", path, err)
+ }
+ return &Index{path: path, idx: idx}, nil
+}
+
+// Path is where the index lives on disk.
+func (x *Index) Path() string { return x.path }
+
+// Close releases the index.
+func (x *Index) Close() error {
+ x.mu.Lock()
+ defer x.mu.Unlock()
+ if x.idx == nil {
+ return nil
+ }
+ err := x.idx.Close()
+ x.idx = nil
+ if err != nil {
+ return fmt.Errorf("search: close index: %w", err)
+ }
+ return nil
+}
+
+// Count is how many documents the index holds, across every space.
+func (x *Index) Count() (uint64, error) {
+ x.mu.RLock()
+ defer x.mu.RUnlock()
+ if x.idx == nil {
+ return 0, errors.New("search: index is closed")
+ }
+ n, err := x.idx.DocCount()
+ if err != nil {
+ return 0, fmt.Errorf("search: count documents: %w", err)
+ }
+ return n, nil
+}
+
+// RebuildSpace replaces everything the index holds for one space.
+//
+// This is a rebuild, not an incremental update: whatever was indexed for sp is
+// removed and docs are written in its place, with no diffing of individual
+// documents and no per-document staleness bookkeeping. The unit of freshness is
+// a space at a revision, which is exactly what the index_stamp row in Postgres
+// records — this package is handed a revision's worth of documents and does not
+// know or care which of them changed.
+//
+// docs may be empty, which empties the space. Every document must belong to sp;
+// a document from another space is a caller bug and is refused rather than
+// written somewhere surprising.
+func (x *Index) RebuildSpace(ctx context.Context, sp core.SpaceRef, docs []Document) (Stats, error) {
+ start := time.Now()
+ if sp.Owner == "" || sp.Name == "" {
+ return Stats{}, errors.New("search: RebuildSpace needs a space")
+ }
+ fields := make([]map[string]any, len(docs))
+ for i, d := range docs {
+ if d.Space != sp {
+ return Stats{}, fmt.Errorf("search: document %q belongs to space %s, not %s", d.ID, d.Space, sp)
+ }
+ f, err := bleveDoc(d)
+ if err != nil {
+ return Stats{}, err
+ }
+ fields[i] = f
+ }
+
+ x.mu.Lock()
+ defer x.mu.Unlock()
+ if x.idx == nil {
+ return Stats{}, errors.New("search: index is closed")
+ }
+
+ stale, err := x.keysOf(sp)
+ if err != nil {
+ return Stats{}, err
+ }
+ if err := ctx.Err(); err != nil {
+ return Stats{}, err
+ }
+
+ // One batch for the whole space, so a space is never half-replaced. Deletes
+ // go in first: a batch is keyed by document id and the last operation on a
+ // key wins, so a document that survives the rebuild is re-indexed rather
+ // than dropped.
+ batch := x.idx.NewBatch()
+ kept := 0
+ for key := range stale {
+ batch.Delete(key)
+ }
+ for i, d := range docs {
+ key := Key(d.Space, d.ID)
+ if _, ok := stale[key]; ok {
+ kept++
+ }
+ if err := batch.Index(key, fields[i]); err != nil {
+ return Stats{}, fmt.Errorf("search: stage %s: %w", key, err)
+ }
+ }
+ if err := x.idx.Batch(batch); err != nil {
+ return Stats{}, fmt.Errorf("search: rebuild space %s: %w", sp, err)
+ }
+
+ st := Stats{Spaces: 1, Indexed: len(docs), Deleted: len(stale) - kept, Took: time.Since(start)}
+ if len(docs) == 0 {
+ st.Spaces = 0
+ }
+ return st, nil
+}
+
+// DeleteSpace removes every document of a space from the index. It is what a
+// deleted space calls; RebuildSpace with no documents does the same thing.
+func (x *Index) DeleteSpace(ctx context.Context, sp core.SpaceRef) (Stats, error) {
+ return x.RebuildSpace(ctx, sp, nil)
+}
+
+// RebuildAll replaces the entire index with docs, which may span any number of
+// spaces.
+//
+// The rebuild runs into a fresh index beside the live one and the two are
+// swapped at the end, so a failure part-way through leaves the old index intact
+// and serving. That is the one thing warren's `bleve.New` over the live path
+// did not give, and it matters here because the index is open in a daemon that
+// is answering queries while the rebuild runs.
+func (x *Index) RebuildAll(ctx context.Context, docs []Document) (Stats, error) {
+ start := time.Now()
+ spaces := make(map[core.SpaceRef]struct{})
+ fields := make([]map[string]any, len(docs))
+ for i, d := range docs {
+ f, err := bleveDoc(d)
+ if err != nil {
+ return Stats{}, err
+ }
+ fields[i] = f
+ spaces[d.Space] = struct{}{}
+ }
+
+ tmp := x.path + ".rebuilding"
+ if err := os.RemoveAll(tmp); err != nil {
+ return Stats{}, fmt.Errorf("search: clear %s: %w", tmp, err)
+ }
+ fresh, err := bleve.New(tmp, buildMapping())
+ if err != nil {
+ return Stats{}, fmt.Errorf("search: create index at %s: %w", tmp, err)
+ }
+ if err := indexAll(ctx, fresh, docs, fields); err != nil {
+ _ = fresh.Close()
+ _ = os.RemoveAll(tmp)
+ return Stats{}, err
+ }
+ if err := fresh.Close(); err != nil {
+ _ = os.RemoveAll(tmp)
+ return Stats{}, fmt.Errorf("search: close rebuilt index: %w", err)
+ }
+
+ x.mu.Lock()
+ defer x.mu.Unlock()
+ if x.idx == nil {
+ _ = os.RemoveAll(tmp)
+ return Stats{}, errors.New("search: index is closed")
+ }
+ if err := x.swap(tmp); err != nil {
+ return Stats{}, err
+ }
+ return Stats{
+ Spaces: len(spaces),
+ Indexed: len(docs),
+ Took: time.Since(start),
+ }, nil
+}
+
+// swap puts the freshly built index at tmp in place of the live one. The caller
+// holds the write lock.
+//
+// The old directory is renamed aside rather than deleted first, so the window in
+// which neither exists is a rename rather than a recursive delete. If reopening
+// the new index fails the Index is left closed and the error is returned: the
+// index is a cache, and a caller that cannot open it must rebuild it, not
+// silently serve an empty one.
+func (x *Index) swap(tmp string) error {
+ if err := x.idx.Close(); err != nil {
+ x.idx = nil
+ return fmt.Errorf("search: close live index: %w", err)
+ }
+ x.idx = nil
+
+ old := x.path + ".old"
+ if err := os.RemoveAll(old); err != nil {
+ return fmt.Errorf("search: clear %s: %w", old, err)
+ }
+ if err := os.Rename(x.path, old); err != nil && !errors.Is(err, os.ErrNotExist) {
+ return fmt.Errorf("search: move live index aside: %w", err)
+ }
+ if err := os.Rename(tmp, x.path); err != nil {
+ return fmt.Errorf("search: move rebuilt index into place: %w", err)
+ }
+ idx, err := bleve.Open(x.path)
+ if err != nil {
+ return fmt.Errorf("search: reopen index at %s: %w", x.path, err)
+ }
+ x.idx = idx
+ if err := os.RemoveAll(old); err != nil {
+ return fmt.Errorf("search: remove %s: %w", old, err)
+ }
+ return nil
+}
+
+func indexAll(ctx context.Context, idx bleve.Index, docs []Document, fields []map[string]any) error {
+ batch := idx.NewBatch()
+ for i, d := range docs {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ key := Key(d.Space, d.ID)
+ if err := batch.Index(key, fields[i]); err != nil {
+ return fmt.Errorf("search: stage %s: %w", key, err)
+ }
+ if batch.Size() >= batchSize {
+ if err := idx.Batch(batch); err != nil {
+ return fmt.Errorf("search: flush batch: %w", err)
+ }
+ batch = idx.NewBatch()
+ }
+ }
+ if batch.Size() > 0 {
+ if err := idx.Batch(batch); err != nil {
+ return fmt.Errorf("search: flush final batch: %w", err)
+ }
+ }
+ return nil
+}
+
+// keysOf lists the index keys currently held for a space. The caller holds a
+// lock.
+func (x *Index) keysOf(sp core.SpaceRef) (map[string]struct{}, error) {
+ q := bleve.NewTermQuery(sp.String())
+ q.SetField(fieldSpace)
+
+ const page = 1000
+ keys := make(map[string]struct{})
+ for from := 0; ; from += page {
+ req := bleve.NewSearchRequestOptions(q, page, from, false)
+ res, err := x.idx.Search(req)
+ if err != nil {
+ return nil, fmt.Errorf("search: list documents of %s: %w", sp, err)
+ }
+ for _, h := range res.Hits {
+ keys[h.ID] = struct{}{}
+ }
+ if len(res.Hits) < page {
+ return keys, nil
+ }
+ }
+}
+
+// tmpPaths are the working directories a rebuild uses, exposed only so a caller
+// cleaning up after a crash knows what to look for.
+func tmpPaths(path string) []string {
+ return []string{path + ".rebuilding", path + ".old"}
+}
+
+// CleanStale removes the working directories a crashed rebuild may have left
+// behind next to the index. Safe to call at startup, before Open.
+func CleanStale(path string) error {
+ for _, p := range tmpPaths(path) {
+ if err := os.RemoveAll(p); err != nil {
+ return fmt.Errorf("search: remove %s: %w", filepath.Base(p), err)
+ }
+ }
+ return nil
+}
A search/index_test.go => search/index_test.go +476 -0
@@ 0,0 1,476 @@
+package search
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "sourcecraft.dev/bigbes/sr-ht-spec/core"
+ "sourcecraft.dev/bigbes/sr-ht-spec/doc"
+)
+
+// Ported from warren: an empty or whitespace-only query returns nothing rather
+// than everything.
+func TestSearchIgnoresWhitespaceOnlyQuery(t *testing.T) {
+ idx := indexCorpus(t, newCorpus(t, "~bigbes/specs", "rev1").
+ add("specs/storage.md", "---\ntitle: Needle handbook\n---\n\nreference material\n"))
+
+ for _, q := range []string{"", " \t\n "} {
+ res, err := idx.Search(context.Background(), Query{Text: q})
+ require.NoError(t, err)
+ require.Empty(t, res.Hits)
+ require.Zero(t, res.Total)
+ }
+}
+
+// Ported from warren's keyword fixture: a title match outranks a body match, so
+// a query that names a document returns the document.
+func TestTitleMatchOutranksBodyMatch(t *testing.T) {
+ idx := indexCorpus(t, newCorpus(t, "~bigbes/specs", "rev1").
+ add("specs/handbook.md", "---\nid: SPEC-0001\ntitle: Needle handbook\n---\n\nreference material\n").
+ add("specs/other.md", "---\nid: SPEC-0002\ntitle: Other document\n---\n\na needle appears in the body\n"))
+
+ require.Equal(t, []string{"SPEC-0001", "SPEC-0002"}, hitIDs(t, idx, Query{Text: "needle"}))
+}
+
+// A project is a saved filter over the one global index. This is the whole of
+// it: the same index, queried with a space set.
+func TestProjectIsASpaceFilterOverOneIndex(t *testing.T) {
+ rfcs := newCorpus(t, "~bigbes/rfcs", "rev1").
+ add("specs/storage.md", "---\nid: SPEC-0001\ntitle: Storage model\n---\n\nA shared vocabulary term.\n")
+ ops := newCorpus(t, "~bigbes/home-ops", "rev1").
+ add("notes/hosts.md", "---\nid: NOTE-0001\ntitle: Hosts\n---\n\nAnother shared vocabulary term.\n")
+ other := newCorpus(t, "~someone/private", "rev1").
+ add("specs/x.md", "---\nid: SPEC-0009\ntitle: Elsewhere\n---\n\nA third shared vocabulary term.\n")
+ idx := indexCorpus(t, rfcs, ops, other)
+
+ // The meta-project: a filter that excludes nothing.
+ all := hitIDs(t, idx, Query{Text: "vocabulary"})
+ require.ElementsMatch(t, []string{"SPEC-0001", "NOTE-0001", "SPEC-0009"}, all)
+
+ // A project over two of the three spaces.
+ project := hitIDs(t, idx, Query{Text: "vocabulary", Spaces: []core.SpaceRef{rfcs.Space, ops.Space}})
+ require.ElementsMatch(t, []string{"SPEC-0001", "NOTE-0001"}, project)
+
+ // One space.
+ require.Equal(t, []string{"SPEC-0009"},
+ hitIDs(t, idx, Query{Text: "vocabulary", Spaces: []core.SpaceRef{other.Space}}))
+}
+
+// Space names are matched whole. Filtering through an analyzed field — which is
+// what warren did for sections — would tokenize "~bigbes/home-ops" and let a
+// query for one space return another.
+func TestSpaceFilterMatchesWholeNamesOnly(t *testing.T) {
+ ops := newCorpus(t, "~bigbes/home-ops", "rev1").
+ add("notes/a.md", "---\nid: NOTE-0001\ntitle: A\n---\n\nshared vocabulary\n")
+ home := newCorpus(t, "~bigbes/home", "rev1").
+ add("notes/b.md", "---\nid: NOTE-0002\ntitle: B\n---\n\nshared vocabulary\n")
+ idx := indexCorpus(t, ops, home)
+
+ require.Equal(t, []string{"NOTE-0002"},
+ hitIDs(t, idx, Query{Text: "vocabulary", Spaces: []core.SpaceRef{home.Space}}))
+}
+
+func TestSearchRejectsAnEmptySpaceFilter(t *testing.T) {
+ idx := indexCorpus(t, newCorpus(t, "~bigbes/specs", "rev1").
+ add("a.md", "---\ntitle: A\n---\n\nbody text here\n"))
+
+ _, err := idx.Search(context.Background(), Query{Text: "body", Spaces: []core.SpaceRef{{}}})
+ require.ErrorContains(t, err, "empty space")
+}
+
+// Ported from warren: log entries summarise other documents, so they are out of
+// an unrestricted search and reachable by naming the section.
+func TestLogEntriesAreExcludedUntilAskedFor(t *testing.T) {
+ idx := indexCorpus(t, newCorpus(t, "~bigbes/specs", "rev1").
+ add("specs/storage.md", "---\nid: SPEC-0001\ntitle: Storage model\n---\n\nThe write path resolves a tree.\n").
+ add("log.md", "# Log\n\n## [2026-05-31] update | Storage model\nRewrote the write path section.\n"))
+
+ require.Equal(t, []string{"SPEC-0001"}, hitIDs(t, idx, Query{Text: "write path"}))
+ // Naming the section is the way back in. The log document itself carries no
+ // body, so only its entry matches the text.
+ require.Equal(t, []string{"log#2026-05-31-1"},
+ hitIDs(t, idx, Query{Text: "write path", Sections: []string{doc.LogSection}}))
+ require.Equal(t, []string{"log"},
+ hitIDs(t, idx, Query{Text: "Log", Sections: []string{doc.LogSection}}),
+ "the log document stays findable by name")
+}
+
+func TestSectionFilterRestrictsResults(t *testing.T) {
+ idx := indexCorpus(t, newCorpus(t, "~bigbes/specs", "rev1").
+ add("specs/storage.md", "---\nid: SPEC-0001\ntitle: Storage\n---\n\nshared vocabulary term\n").
+ add("notes/scratch.md", "---\nid: NOTE-0001\ntitle: Scratch\n---\n\nshared vocabulary term\n"))
+
+ require.Equal(t, []string{"SPEC-0001"}, hitIDs(t, idx, Query{Text: "vocabulary", Sections: []string{"specs"}}))
+ require.Equal(t, []string{"NOTE-0001"}, hitIDs(t, idx, Query{Text: "vocabulary", Sections: []string{"notes"}}))
+ require.Len(t, hitIDs(t, idx, Query{Text: "vocabulary", Sections: []string{"specs", "notes"}}), 2)
+}
+
+func TestSearchRejectsAnEmptySection(t *testing.T) {
+ idx := indexCorpus(t, newCorpus(t, "~bigbes/specs", "rev1").
+ add("a.md", "---\ntitle: A\n---\n\nbody text here\n"))
+
+ _, err := idx.Search(context.Background(), Query{Text: "body", Sections: []string{""}})
+ require.ErrorContains(t, err, "empty section")
+}
+
+func TestHitCarriesAPinnedAddress(t *testing.T) {
+ idx := indexCorpus(t, newCorpus(t, "~bigbes/specs", "8f14e45fceea167a").
+ add("specs/storage.md", "---\nid: SPEC-0001\ntitle: Storage model\n---\n\nThe write path resolves a tree.\n"))
+
+ res, err := idx.Search(context.Background(), Query{Text: "resolves"})
+ require.NoError(t, err)
+ require.Len(t, res.Hits, 1)
+ h := res.Hits[0]
+ require.Equal(t, space(t, "~bigbes/specs"), h.Space)
+ require.Equal(t, "SPEC-0001", h.ID)
+ require.Equal(t, "8f14e45fceea167a", h.Rev)
+ require.Equal(t, "specs/storage.md", h.Path)
+ require.Equal(t, "specs", h.Section)
+ require.Equal(t, "Storage model", h.Title)
+ require.Equal(t, LangEN, h.Lang)
+ require.Greater(t, h.Score, 0.0)
+ require.Contains(t, h.Snippet, "<mark>resolves</mark>")
+}
+
+// Snippets are rendered as HTML by the review UI, so the text around the marks
+// must be escaped. bleve's html formatter does it; this pins the behaviour.
+func TestSnippetIsHTMLEscaped(t *testing.T) {
+ idx := indexCorpus(t, newCorpus(t, "~bigbes/specs", "rev1").
+ add("specs/x.md", "---\nid: SPEC-0001\ntitle: Escaping\n---\n\n"+
+ "A needle inside `<script>alert(1)</script>` and more prose after it.\n"))
+
+ res, err := idx.Search(context.Background(), Query{Text: "needle"})
+ require.NoError(t, err)
+ require.Len(t, res.Hits, 1)
+ require.Contains(t, res.Hits[0].Snippet, "<script>")
+ require.NotContains(t, res.Hits[0].Snippet, "<script>")
+}
+
+// A rebuild is per space at a revision: documents that are gone at the new
+// revision leave the index, and the other spaces are untouched.
+func TestRebuildSpaceReplacesTheSpaceAndOnlyTheSpace(t *testing.T) {
+ rfcs := newCorpus(t, "~bigbes/rfcs", "rev1").
+ add("specs/keep.md", "---\nid: SPEC-0001\ntitle: Kept\n---\n\nshared vocabulary term\n").
+ add("specs/gone.md", "---\nid: SPEC-0002\ntitle: Removed\n---\n\nshared vocabulary term\n")
+ ops := newCorpus(t, "~bigbes/home-ops", "rev1").
+ add("notes/hosts.md", "---\nid: NOTE-0001\ntitle: Hosts\n---\n\nshared vocabulary term\n")
+ idx := indexCorpus(t, rfcs, ops)
+ require.ElementsMatch(t, []string{"SPEC-0001", "SPEC-0002", "NOTE-0001"},
+ hitIDs(t, idx, Query{Text: "vocabulary"}))
+
+ next := newCorpus(t, "~bigbes/rfcs", "rev2").
+ add("specs/keep.md", "---\nid: SPEC-0001\ntitle: Kept\n---\n\nshared vocabulary term, reworded\n").
+ add("specs/new.md", "---\nid: SPEC-0003\ntitle: Added\n---\n\nshared vocabulary term\n")
+ st, err := idx.RebuildSpace(context.Background(), next.Space, next.extract(t))
+ require.NoError(t, err)
+ require.Equal(t, 2, st.Indexed)
+ require.Equal(t, 1, st.Deleted, "SPEC-0002 is gone at rev2")
+ require.Positive(t, st.Took)
+
+ require.ElementsMatch(t, []string{"SPEC-0001", "SPEC-0003", "NOTE-0001"},
+ hitIDs(t, idx, Query{Text: "vocabulary"}))
+
+ res, err := idx.Search(context.Background(), Query{Text: "reworded"})
+ require.NoError(t, err)
+ require.Len(t, res.Hits, 1)
+ require.Equal(t, "rev2", res.Hits[0].Rev, "a surviving document is re-indexed at the new revision")
+}
+
+func TestRebuildSpaceRejectsADocumentFromAnotherSpace(t *testing.T) {
+ idx := openIndex(t)
+ other := newCorpus(t, "~bigbes/rfcs", "rev1").
+ add("a.md", "---\ntitle: A\n---\n\nbody text here\n")
+
+ _, err := idx.RebuildSpace(context.Background(), space(t, "~bigbes/home-ops"), other.extract(t))
+ require.ErrorContains(t, err, "belongs to space ~bigbes/rfcs, not ~bigbes/home-ops")
+}
+
+func TestRebuildSpaceNeedsASpace(t *testing.T) {
+ _, err := openIndex(t).RebuildSpace(context.Background(), core.SpaceRef{}, nil)
+ require.ErrorContains(t, err, "needs a space")
+}
+
+func TestDeleteSpaceEmptiesIt(t *testing.T) {
+ rfcs := newCorpus(t, "~bigbes/rfcs", "rev1").
+ add("specs/a.md", "---\nid: SPEC-0001\ntitle: A\n---\n\nshared vocabulary term\n")
+ ops := newCorpus(t, "~bigbes/home-ops", "rev1").
+ add("notes/b.md", "---\nid: NOTE-0001\ntitle: B\n---\n\nshared vocabulary term\n")
+ idx := indexCorpus(t, rfcs, ops)
+
+ st, err := idx.DeleteSpace(context.Background(), rfcs.Space)
+ require.NoError(t, err)
+ require.Equal(t, 1, st.Deleted)
+ require.Equal(t, []string{"NOTE-0001"}, hitIDs(t, idx, Query{Text: "vocabulary"}))
+
+ n, err := idx.Count()
+ require.NoError(t, err)
+ require.Equal(t, uint64(1), n)
+}
+
+// RebuildAll builds beside the live index and swaps, so the old index stays
+// open and serving until the new one is complete.
+func TestRebuildAllReplacesEverything(t *testing.T) {
+ stale := newCorpus(t, "~bigbes/rfcs", "rev1").
+ add("specs/a.md", "---\nid: SPEC-0001\ntitle: A\n---\n\nshared vocabulary term\n")
+ idx := indexCorpus(t, stale)
+
+ rfcs := newCorpus(t, "~bigbes/rfcs", "rev2").
+ add("specs/b.md", "---\nid: SPEC-0002\ntitle: B\n---\n\nshared vocabulary term\n")
+ ops := newCorpus(t, "~bigbes/home-ops", "rev1").
+ add("notes/c.md", "---\nid: NOTE-0001\ntitle: C\n---\n\nshared vocabulary term\n")
+ st, err := idx.RebuildAll(context.Background(), append(rfcs.extract(t), ops.extract(t)...))
+ require.NoError(t, err)
+ require.Equal(t, 2, st.Spaces)
+ require.Equal(t, 2, st.Indexed)
+ require.Positive(t, st.Took)
+
+ require.ElementsMatch(t, []string{"SPEC-0002", "NOTE-0001"}, hitIDs(t, idx, Query{Text: "vocabulary"}))
+
+ // Neither working directory survives a successful rebuild.
+ for _, p := range tmpPaths(idx.Path()) {
+ _, err := os.Stat(p)
+ require.True(t, os.IsNotExist(err), "%s should be gone", filepath.Base(p))
+ }
+}
+
+func TestCleanStaleRemovesRebuildLeftovers(t *testing.T) {
+ base := filepath.Join(t.TempDir(), "spec.bleve")
+ for _, p := range tmpPaths(base) {
+ require.NoError(t, os.MkdirAll(p, 0o755))
+ }
+ require.NoError(t, CleanStale(base))
+ for _, p := range tmpPaths(base) {
+ _, err := os.Stat(p)
+ require.True(t, os.IsNotExist(err))
+ }
+}
+
+// The index survives a restart: it is a cache, but not one that has to be
+// rebuilt on every boot.
+func TestOpenReusesAnExistingIndex(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "spec.bleve")
+ c := newCorpus(t, "~bigbes/rfcs", "rev1").
+ add("specs/a.md", "---\nid: SPEC-0001\ntitle: A\n---\n\nshared vocabulary term\n")
+
+ idx, err := Open(path)
+ require.NoError(t, err)
+ _, err = idx.RebuildSpace(context.Background(), c.Space, c.extract(t))
+ require.NoError(t, err)
+ require.NoError(t, idx.Close())
+
+ reopened, err := Open(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, reopened.Close()) })
+ require.Equal(t, []string{"SPEC-0001"}, hitIDs(t, reopened, Query{Text: "vocabulary"}))
+}
+
+func TestOpenNeedsAPath(t *testing.T) {
+ _, err := Open("")
+ require.ErrorContains(t, err, "path is required")
+}
+
+func TestClosedIndexRefusesWork(t *testing.T) {
+ idx, err := Open(filepath.Join(t.TempDir(), "spec.bleve"))
+ require.NoError(t, err)
+ require.NoError(t, idx.Close())
+ require.NoError(t, idx.Close(), "closing twice is not an error")
+
+ _, err = idx.Search(context.Background(), Query{Text: "anything"})
+ require.ErrorContains(t, err, "index is closed")
+ _, err = idx.Count()
+ require.ErrorContains(t, err, "index is closed")
+ _, err = idx.RebuildSpace(context.Background(), space(t, "~bigbes/rfcs"), nil)
+ require.ErrorContains(t, err, "index is closed")
+}
+
+func TestSearchPaginates(t *testing.T) {
+ c := newCorpus(t, "~bigbes/rfcs", "rev1")
+ for i := range 5 {
+ c.add(fmt.Sprintf("specs/%d.md", i),
+ fmt.Sprintf("---\nid: SPEC-000%d\ntitle: Document %d\n---\n\nshared vocabulary term\n", i, i))
+ }
+ idx := indexCorpus(t, c)
+
+ res, err := idx.Search(context.Background(), Query{Text: "vocabulary", Limit: 2})
+ require.NoError(t, err)
+ require.Len(t, res.Hits, 2)
+ require.Equal(t, uint64(5), res.Total, "Total counts matches, not returned hits")
+
+ page2, err := idx.Search(context.Background(), Query{Text: "vocabulary", Limit: 2, Offset: 2})
+ require.NoError(t, err)
+ require.Len(t, page2.Hits, 2)
+ require.NotEqual(t, res.Hits[0].ID, page2.Hits[0].ID)
+
+ _, err = idx.Search(context.Background(), Query{Text: "vocabulary", Offset: -1})
+ require.ErrorContains(t, err, "negative offset")
+}
+
+func TestRebuildHonoursCancellation(t *testing.T) {
+ c := newCorpus(t, "~bigbes/rfcs", "rev1").
+ add("specs/a.md", "---\nid: SPEC-0001\ntitle: A\n---\n\nbody text here\n")
+ idx := openIndex(t)
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ _, err := idx.RebuildAll(ctx, c.extract(t))
+ require.ErrorIs(t, err, context.Canceled)
+ _, err = idx.RebuildSpace(ctx, c.Space, c.extract(t))
+ require.ErrorIs(t, err, context.Canceled)
+}
+
+func TestStatsString(t *testing.T) {
+ require.Equal(t, "spaces=2 indexed=10 deleted=1 took=0s", Stats{Spaces: 2, Indexed: 10, Deleted: 1}.String())
+}
+
+func TestKeyIsScopedToItsSpace(t *testing.T) {
+ a := space(t, "~bigbes/rfcs")
+ b := space(t, "~bigbes/home-ops")
+ require.Equal(t, "~bigbes/rfcs:specs/storage", Key(a, "specs/storage"))
+ require.NotEqual(t, Key(a, "specs/storage"), Key(b, "specs/storage"))
+}
+
+// Two spaces whose documents fall back to the same path-derived id are two
+// documents in the global index, not one overwriting the other.
+func TestSameIDInTwoSpacesStaysTwoDocuments(t *testing.T) {
+ a := newCorpus(t, "~bigbes/rfcs", "rev1").
+ add("specs/storage.md", "# Storage\n\nshared vocabulary term\n")
+ b := newCorpus(t, "~bigbes/home-ops", "rev1").
+ add("specs/storage.md", "# Storage\n\nshared vocabulary term\n")
+ idx := indexCorpus(t, a, b)
+
+ res, err := idx.Search(context.Background(), Query{Text: "vocabulary"})
+ require.NoError(t, err)
+ require.Len(t, res.Hits, 2)
+ require.ElementsMatch(t,
+ []core.SpaceRef{a.Space, b.Space},
+ []core.SpaceRef{res.Hits[0].Space, res.Hits[1].Space})
+ for _, h := range res.Hits {
+ require.Equal(t, "specs/storage", h.ID)
+ }
+}
+
+// Rebuild timing on a corpus far larger than the confirmed volume. The design
+// absorbs the batch rebuild on the argument that at tens of documents a day it
+// is cheap; this is the measurement that argument is owed. It asserts almost
+// nothing — the number is the point, and it is logged.
+func TestRebuildAllTimingOnARealisticCorpus(t *testing.T) {
+ if testing.Short() {
+ t.Skip("timing run")
+ }
+ const spaces, perSpace = 5, 200
+ var all []Document
+ for s := range spaces {
+ c := newCorpus(t, fmt.Sprintf("~bigbes/space-%d", s), "rev1")
+ for i := range perSpace {
+ c.add(fmt.Sprintf("specs/doc-%03d.md", i), syntheticDocument(s, i))
+ }
+ all = append(all, c.extract(t)...)
+ }
+
+ idx := openIndex(t)
+ st, err := idx.RebuildAll(context.Background(), all)
+ require.NoError(t, err)
+ require.Equal(t, spaces*perSpace, st.Indexed)
+ t.Logf("RebuildAll over %d documents in %d spaces: %s", st.Indexed, st.Spaces, st)
+
+ one := all[:perSpace]
+ spaceStats, err := idx.RebuildSpace(context.Background(), one[0].Space, one)
+ require.NoError(t, err)
+ t.Logf("RebuildSpace over %d documents: %s", spaceStats.Indexed, spaceStats)
+
+ for _, q := range []string{"ревизия", "proposal", "предложение"} {
+ res, err := idx.Search(context.Background(), Query{Text: q, Limit: 10})
+ require.NoError(t, err)
+ t.Logf("query %q over %d documents: %d matches in %s", q, st.Indexed, res.Total, res.Took)
+ require.Positive(t, res.Total)
+ }
+}
+
+// syntheticDocument is roughly the size and shape of a real spec: frontmatter,
+// English prose, Russian prose, and a fenced block.
+func syntheticDocument(space, n int) string {
+ var b strings.Builder
+ fmt.Fprintf(&b, "---\nid: SPEC-%d%04d\ntitle: Storage model revision %d\n"+
+ "status: draft\ntags: [storage, review, index]\nowners: ['~bigbes']\n"+
+ "summary: One tier, git objects only, revision %d\n---\n\n", space, n, n, n)
+ for i := range 6 {
+ fmt.Fprintf(&b, "## Section %d\n\n%s\n\n%s\n\n", i, enSpecBody, ruSpecBody)
+ fmt.Fprintf(&b, "```go\nfunc Rebuild%d(ctx context.Context) error { return nil }\n```\n\n", i)
+ }
+ return b.String()
+}
+
+// A failed rebuild must leave the live index serving. RebuildAll builds beside
+// it and swaps only on success, so a cancellation mid-rebuild is a no-op.
+func TestFailedRebuildAllLeavesTheLiveIndexIntact(t *testing.T) {
+ live := newCorpus(t, "~bigbes/rfcs", "rev1").
+ add("specs/a.md", "---\nid: SPEC-0001\ntitle: A\n---\n\nshared vocabulary term\n")
+ idx := indexCorpus(t, live)
+
+ next := newCorpus(t, "~bigbes/rfcs", "rev2").
+ add("specs/b.md", "---\nid: SPEC-0002\ntitle: B\n---\n\nshared vocabulary term\n")
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ _, err := idx.RebuildAll(ctx, next.extract(t))
+ require.ErrorIs(t, err, context.Canceled)
+
+ require.Equal(t, []string{"SPEC-0001"}, hitIDs(t, idx, Query{Text: "vocabulary"}),
+ "the old index is still open and still answering")
+ for _, p := range tmpPaths(idx.Path()) {
+ _, err := os.Stat(p)
+ require.True(t, os.IsNotExist(err), "%s should have been cleaned up", filepath.Base(p))
+ }
+}
+
+// The daemon answers queries while a rebuild runs. Searches take a read lock
+// and a rebuild takes the write lock; this is the check that they compose.
+func TestSearchesRunConcurrentlyWithARebuild(t *testing.T) {
+ c := newCorpus(t, "~bigbes/rfcs", "rev1")
+ for i := range 50 {
+ c.add(fmt.Sprintf("specs/%d.md", i),
+ fmt.Sprintf("---\nid: SPEC-00%02d\ntitle: Document %d\n---\n\nshared vocabulary term\n", i, i))
+ }
+ idx := indexCorpus(t, c)
+ docs := c.extract(t)
+
+ var wg sync.WaitGroup
+ for range 8 {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for range 20 {
+ _, err := idx.Search(context.Background(), Query{Text: "vocabulary"})
+ assert.NoError(t, err)
+ }
+ }()
+ }
+ for range 3 {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ _, err := idx.RebuildSpace(context.Background(), c.Space, docs)
+ assert.NoError(t, err)
+ }()
+ }
+ wg.Wait()
+
+ require.Len(t, hitIDs(t, idx, Query{Text: "vocabulary", Limit: 100}), 50)
+}
+
+// A document at the space root has no section. Excluding the log section must
+// not also exclude it: "not in the log" is not "has a section".
+func TestRootLevelDocumentsSurviveTheDefaultLogExclusion(t *testing.T) {
+ idx := indexCorpus(t, newCorpus(t, "~bigbes/rfcs", "rev1").
+ add("README.md", "---\nid: SPEC-0001\ntitle: Read me\n---\n\nshared vocabulary term\n").
+ add("specs/a.md", "---\nid: SPEC-0002\ntitle: A\n---\n\nshared vocabulary term\n"))
+
+ require.ElementsMatch(t, []string{"SPEC-0001", "SPEC-0002"}, hitIDs(t, idx, Query{Text: "vocabulary"}))
+}
A search/lang.go => search/lang.go +119 -0
@@ 0,0 1,119 @@
+package search
+
+import (
+ "strings"
+ "unicode"
+)
+
+// Lang is a language the index has a stemming analyzer for. Every indexed
+// document is labelled with exactly one — its dominant language — and its text
+// is routed block by block into the matching analyzed field.
+type Lang string
+
+const (
+ LangEN Lang = "en"
+ LangRU Lang = "ru"
+)
+
+// DefaultLang is the label a document with too little text to classify gets,
+// and the language a block falls back to when it is too short to classify on
+// its own. English rather than Russian because the machine-generated half of
+// this corpus — frontmatter keys, paths, identifiers, fenced code — is English
+// whatever language the prose around it is written in.
+const DefaultLang = LangEN
+
+// ruLetterRatio is the share of a text's letters that must be Cyrillic for it
+// to count as Russian.
+//
+// The threshold is deliberately far below one half. The two error directions
+// are not symmetric in likelihood: an English block essentially never contains
+// Cyrillic at all, while a Russian block in this corpus routinely carries a
+// third or more Latin letters — identifiers, product names, and untranslated
+// technical terms are written in Latin inside Russian prose. A 0.5 threshold
+// would therefore misfile real Russian paragraphs as English, and misfile
+// almost no English ones as Russian.
+const ruLetterRatio = 0.35
+
+// minDetectLetters is the least number of letters a text needs before its
+// script mix is treated as evidence. Below it, a single Latin acronym in a
+// Russian heading (or one Russian word in an English one) would decide the
+// whole block, so the caller's fallback is used instead.
+const minDetectLetters = 12
+
+// Detect classifies a text by script. ok is false when the text carries too
+// few letters to classify, in which case the caller supplies the fallback —
+// Detect never guesses.
+func Detect(text string) (lang Lang, ok bool) {
+ var cyrillic, letters int
+ for _, r := range text {
+ if !unicode.IsLetter(r) {
+ continue
+ }
+ letters++
+ if unicode.Is(unicode.Cyrillic, r) {
+ cyrillic++
+ }
+ }
+ if letters < minDetectLetters {
+ return "", false
+ }
+ if float64(cyrillic)/float64(letters) >= ruLetterRatio {
+ return LangRU, true
+ }
+ return LangEN, true
+}
+
+// DetectIn classifies a text, falling back to a language when it is too short
+// to classify on its own.
+func DetectIn(text string, fallback Lang) Lang {
+ if l, ok := Detect(text); ok {
+ return l
+ }
+ return fallback
+}
+
+// route splits a text into its Russian and its English part, block by block,
+// so that each half is stemmed by the analyzer that understands it.
+//
+// This is the part the design left open, and per-*document* routing — the
+// obvious reading of "detect the language and write the matching field" — is
+// not sufficient. The two analyzers pass each other's script through
+// untouched: bleve's `ru` analyzer leaves "indexes" as "indexes" and its `en`
+// analyzer leaves "индексы" as "индексы". Foreign-script terms therefore still
+// match literally (which is why a single-analyzer index is not catastrophic),
+// but they match *unstemmed*, so "index" does not find "indexes" and
+// "документы" does not find "документ". A specification whose prose is Russian
+// and whose examples, headings and quoted requirements are English is one
+// document, and one label for it necessarily mangles one of its two halves.
+//
+// Routing per block costs nothing extra — the document is being walked anyway —
+// and removes the failure entirely: each block lands in the field whose
+// analyzer stems it, and a query is run against both fields. Duplicating the
+// whole text into both fields would also fix the stemming, but it doubles the
+// index and double-counts every document that matches in both fields, which
+// biases ranking toward mixed documents for no reason related to relevance.
+//
+// The unit is a line, because that is the unit the text arrives in: doc's
+// plain-text projection emits a newline after every block-level node and at
+// every soft line break, and frontmatter search text is one "key: value" line
+// per key. So a line here is a paragraph, a wrapped fragment of one, a heading,
+// a table row, a list item or a line of code. A line too short to classify
+// takes fallback, which is the document's dominant language — the language of
+// the prose it sits inside.
+func route(text string, fallback Lang) (ru, en string) {
+ var ruB, enB strings.Builder
+ for _, line := range strings.Split(text, "\n") {
+ if strings.TrimSpace(line) == "" {
+ continue
+ }
+ b := &enB
+ if DetectIn(line, fallback) == LangRU {
+ b = &ruB
+ }
+ if b.Len() > 0 {
+ b.WriteByte('\n')
+ }
+ b.WriteString(line)
+ }
+ return ruB.String(), enB.String()
+}
A search/lang_test.go => search/lang_test.go +70 -0
@@ 0,0 1,70 @@
+package search
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestDetectClassifiesByScript(t *testing.T) {
+ cases := []struct {
+ name string
+ text string
+ want Lang
+ ok bool
+ }{
+ {"english prose", "The approved revision is the one a bot reads.", LangEN, true},
+ {"russian prose", "Одобренная ревизия — это та, которую читает бот.", LangRU, true},
+ {
+ "russian prose with latin identifiers",
+ "Ревизия хранится в ветке approved, а черновик — в proposals/<id>.",
+ LangRU, true,
+ },
+ {
+ "english prose with one russian word",
+ "The reviewer approves the merge, sometimes annotated проверено, before it lands.",
+ LangEN, true,
+ },
+ {"too short to classify", "## API", "", false},
+ {"digits and punctuation only", "1234-5678 | --- | 9.0", "", false},
+ {"empty", "", "", false},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ got, ok := Detect(c.text)
+ require.Equal(t, c.ok, ok)
+ require.Equal(t, c.want, got)
+ })
+ }
+}
+
+func TestDetectInFallsBackWhenUndecidable(t *testing.T) {
+ require.Equal(t, LangRU, DetectIn("## API", LangRU))
+ require.Equal(t, LangEN, DetectIn("## API", LangEN))
+ // A decidable text ignores the fallback entirely.
+ require.Equal(t, LangRU, DetectIn("Ревизия хранится в ветке approved.", LangEN))
+}
+
+func TestRouteSplitsAMixedDocumentByLine(t *testing.T) {
+ text := "Одобренная ревизия — это та, которую читает бот.\n" +
+ "## API\n" +
+ "The approved revision is the one a bot reads and pins.\n" +
+ "Черновики живут в отдельной ветке предложений.\n"
+
+ ru, en := route(text, LangRU)
+ require.Contains(t, ru, "Одобренная ревизия")
+ require.Contains(t, ru, "Черновики живут")
+ require.NotContains(t, ru, "approved revision is the one")
+ require.Contains(t, en, "The approved revision is the one a bot reads")
+ require.NotContains(t, en, "Одобренная")
+ // The undecidable heading follows the document's language.
+ require.Contains(t, ru, "## API")
+ require.NotContains(t, en, "## API")
+}
+
+func TestRouteLeavesASingleLanguageDocumentWhole(t *testing.T) {
+ text := "The approved revision is the one a bot reads.\nProposals live on their own branch.\n"
+ ru, en := route(text, LangEN)
+ require.Empty(t, ru)
+ require.Equal(t, "The approved revision is the one a bot reads.\nProposals live on their own branch.", en)
+}
A search/mapping.go => search/mapping.go +166 -0
@@ 0,0 1,166 @@
+package search
+
+import (
+ "fmt"
+
+ "github.com/blevesearch/bleve/v2"
+ "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
+ "github.com/blevesearch/bleve/v2/analysis/lang/en"
+ "github.com/blevesearch/bleve/v2/analysis/lang/ru"
+ "github.com/blevesearch/bleve/v2/mapping"
+
+ "sourcecraft.dev/bigbes/sr-ht-spec/core"
+)
+
+// Field names in the bleve index. Three groups, and the difference between them
+// is what the whole mapping is:
+//
+// - filters (space, section, lang) are keyword-analyzed, so they are matched
+// as whole strings by a term query and never tokenized. Filtering by a
+// space through an analyzed field — which is what warren did for sections —
+// means "~bigbes/b-tree" matches the space "~someone/tree", and a project
+// is precisely a filter over spaces, so it has to be exact.
+// - analyzed text (title_en/title_ru, body_en/body_ru) carries the
+// searchable words, one field pair per language. See lang.go.
+// - metadata (rev, path, anchor, title) is stored and not indexed: it is what
+// turns a hit into a URL, not something to search on.
+const (
+ fieldSpace = "space"
+ fieldSection = "section"
+ fieldLang = "lang"
+ fieldRev = "rev"
+ fieldPath = "path"
+ fieldAnchor = "anchor"
+ fieldTitle = "title"
+ fieldTitleEN = "title_en"
+ fieldTitleRU = "title_ru"
+ fieldBodyEN = "body_en"
+ fieldBodyRU = "body_ru"
+)
+
+// titleBoost is how much more a title match is worth than a body match. Carried
+// over from warren unchanged: a query that names a document should return that
+// document, not the twenty documents that mention it.
+const titleBoost = 3.0
+
+// buildMapping is the index mapping of the one global index. There is exactly
+// one, and it is not parameterized by language: the design's earlier
+// per-index-language choice is what this replaces.
+func buildMapping() mapping.IndexMapping {
+ text := func(analyzer string, termVectors bool) *mapping.FieldMapping {
+ f := bleve.NewTextFieldMapping()
+ f.Analyzer = analyzer
+ f.Store = true
+ f.IncludeTermVectors = termVectors
+ f.IncludeInAll = false
+ return f
+ }
+ exact := func() *mapping.FieldMapping {
+ f := bleve.NewTextFieldMapping()
+ f.Analyzer = keyword.Name
+ f.Store = true
+ f.IncludeInAll = false
+ return f
+ }
+ meta := func() *mapping.FieldMapping {
+ f := bleve.NewTextFieldMapping()
+ f.Index = false
+ f.Store = true
+ f.IncludeInAll = false
+ return f
+ }
+
+ d := bleve.NewDocumentMapping()
+ // Nothing is indexed that this file does not name. A dynamic mapping would
+ // silently index whatever a future field happens to be called, with the
+ // default analyzer, which is how an index acquires fields nobody meant.
+ d.Dynamic = false
+ d.AddFieldMappingsAt(fieldSpace, exact())
+ d.AddFieldMappingsAt(fieldSection, exact())
+ d.AddFieldMappingsAt(fieldLang, exact())
+ d.AddFieldMappingsAt(fieldRev, meta())
+ d.AddFieldMappingsAt(fieldPath, meta())
+ d.AddFieldMappingsAt(fieldAnchor, meta())
+ d.AddFieldMappingsAt(fieldTitle, meta())
+ d.AddFieldMappingsAt(fieldTitleEN, text(en.AnalyzerName, false))
+ d.AddFieldMappingsAt(fieldTitleRU, text(ru.AnalyzerName, false))
+ // Body fields carry term vectors because they are the highlighted ones: a
+ // snippet is reconstructed from stored text plus term locations.
+ d.AddFieldMappingsAt(fieldBodyEN, text(en.AnalyzerName, true))
+ d.AddFieldMappingsAt(fieldBodyRU, text(ru.AnalyzerName, true))
+
+ m := bleve.NewIndexMapping()
+ m.DefaultAnalyzer = en.AnalyzerName
+ m.DefaultMapping = d
+ return m
+}
+
+// Key is the id a document is stored under in the global index. The space is
+// part of it because the index is global: two spaces may each hold a document
+// whose id fell back to the path "specs/storage", and in a single index those
+// are two documents, not one overwriting the other.
+//
+// It is deliberately not parsed back. Space, path and the rest are stored
+// fields; the key is an opaque identity.
+func Key(sp core.SpaceRef, id string) string {
+ return sp.String() + ":" + id
+}
+
+// bleveDoc projects a Document into the field map bleve indexes, doing the
+// language detection and routing on the way. A map rather than a struct so
+// that a field a document has nothing for is absent from the index instead of
+// present and empty.
+func bleveDoc(d Document) (map[string]any, error) {
+ if err := d.validate(); err != nil {
+ return nil, err
+ }
+ // Dominant language of the document as a whole, used to label it and as the
+ // fallback for lines too short to classify on their own. Title first: it is
+ // the most reliably prose-like text a document has.
+ lang := DetectIn(d.Title+"\n"+d.Text, DefaultLang)
+
+ m := map[string]any{
+ fieldSpace: d.Space.String(),
+ fieldLang: string(lang),
+ }
+ put := func(field, value string) {
+ if value != "" {
+ m[field] = value
+ }
+ }
+ put(fieldSection, d.Section)
+ put(fieldRev, d.Rev)
+ put(fieldPath, d.Path)
+ put(fieldAnchor, d.Anchor)
+ put(fieldTitle, d.Title)
+
+ // A title is one line and takes the document's language; splitting it would
+ // only ever misfile the shorter half of a name.
+ if d.Title != "" {
+ if lang == LangRU {
+ put(fieldTitleRU, d.Title)
+ } else {
+ put(fieldTitleEN, d.Title)
+ }
+ }
+ bodyRU, bodyEN := route(d.Text, lang)
+ put(fieldBodyRU, bodyRU)
+ put(fieldBodyEN, bodyEN)
+ return m, nil
+}
+
+func (d Document) validate() error {
+ if d.Space.Owner == "" || d.Space.Name == "" {
+ return fmt.Errorf("search: document %q has no space", d.ID)
+ }
+ if err := core.ValidateOwner(d.Space.Owner); err != nil {
+ return fmt.Errorf("search: document %q space owner: %w", d.ID, err)
+ }
+ if err := core.ValidateSpaceName(d.Space.Name); err != nil {
+ return fmt.Errorf("search: document %q space name: %w", d.ID, err)
+ }
+ if d.ID == "" {
+ return fmt.Errorf("search: document in space %s has no id", d.Space)
+ }
+ return nil
+}
A search/mixed_test.go => search/mixed_test.go +165 -0
@@ 0,0 1,165 @@
+package search
+
+import (
+ "context"
+ "path/filepath"
+ "testing"
+
+ "github.com/blevesearch/bleve/v2"
+ "github.com/blevesearch/bleve/v2/analysis/lang/en"
+ "github.com/blevesearch/bleve/v2/analysis/lang/ru"
+ "github.com/blevesearch/bleve/v2/registry"
+ "github.com/stretchr/testify/require"
+)
+
+// The design leaves "mixed Russian/English search" open, with warren's
+// per-index analyzer choice as the starting point. These tests are the evidence
+// the decision was made on. They run in three steps:
+//
+// 1. what one analyzer actually does to the other language's words,
+// 2. what that costs a genuinely mixed document under per-document routing,
+// 3. that per-line routing removes the cost.
+
+const (
+ enSpecBody = "The approved revision is the one a bot reads. Every read resolves a git tree\n" +
+ "and reads blobs from it. Proposals live on their own branch and are merged\n" +
+ "only after a human approves them.\n"
+
+ ruSpecBody = "Одобренная ревизия — это та, которую читают агенты. Каждое чтение разрешает\n" +
+ "дерево гита и читает из него блобы. Предложения живут в отдельной ветке и\n" +
+ "объединяются только после проверки человеком.\n"
+
+ // A real shape for this corpus: Russian prose around English requirements
+ // quoted verbatim from the upstream document they came from.
+ mixedSpecBody = "Этот документ описывает требования к вложениям и двоичным файлам в спецификациях.\n" +
+ "The requirement is quoted verbatim from upstream: attachments larger than the\n" +
+ "configured cap are rejected by the daemon before the push is accepted.\n" +
+ "Ограничение размера вложений обсуждается отдельно и пока не выбрано.\n"
+)
+
+func analyze(t *testing.T, analyzer, text string) []string {
+ t.Helper()
+ a, err := registry.NewCache().AnalyzerNamed(analyzer)
+ require.NoError(t, err)
+ var out []string
+ for _, tok := range a.Analyze([]byte(text)) {
+ out = append(out, string(tok.Term))
+ }
+ return out
+}
+
+// TestOneAnalyzerCannotStemBothLanguages records the premise: bleve's per-index
+// analyzer choice does not break the other language, it merely stops stemming
+// it. Foreign-script terms survive as literals — which is why warren's
+// single-analyzer index was usable — but they survive unstemmed, so singular
+// and plural stop being the same word.
+func TestOneAnalyzerCannotStemBothLanguages(t *testing.T) {
+ // Its own language: stemmed, stop words dropped.
+ require.Equal(t, []string{"index", "rebuild", "document"},
+ analyze(t, en.AnalyzerName, "indexes rebuild all the documents"))
+ require.Equal(t, []string{"индекс", "перестраива", "документ"},
+ analyze(t, ru.AnalyzerName, "индексы перестраивает все документы"))
+
+ // The other language: passed through whole, stop words and all.
+ require.Equal(t, []string{"indexes", "rebuild", "all", "the", "documents"},
+ analyze(t, ru.AnalyzerName, "indexes rebuild all the documents"))
+ require.Equal(t, []string{"индексы", "перестраивает", "все", "документы"},
+ analyze(t, en.AnalyzerName, "индексы перестраивает все документы"))
+}
+
+// TestPerDocumentRoutingLosesTheMinorityLanguage is the negative result that
+// rules out the obvious reading of the design's open item — "detect the
+// dominant language of the document and write the matching field".
+//
+// The document below is dominantly Russian, so per-document routing files all
+// of it, English requirements included, under the Russian analyzer. The Russian
+// half then searches correctly and the English half is reachable only by exact
+// word form: "attachments" is found, "attachment" is not.
+func TestPerDocumentRoutingLosesTheMinorityLanguage(t *testing.T) {
+ idx, err := bleve.New(filepath.Join(t.TempDir(), "per-document.bleve"), buildMapping())
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, idx.Close()) })
+
+ // Exactly what per-document routing produces: one field, chosen by the
+ // document's dominant language.
+ lang := DetectIn(mixedSpecBody, DefaultLang)
+ require.Equal(t, LangRU, lang, "the fixture is dominantly Russian")
+ require.NoError(t, idx.Index("mixed", map[string]any{
+ fieldSpace: "~bigbes/specs",
+ fieldTitle: "Вложения",
+ fieldBodyRU: mixedSpecBody,
+ }))
+
+ found := func(term, field string) bool {
+ q := bleve.NewMatchQuery(term)
+ q.SetField(field)
+ res, err := idx.Search(bleve.NewSearchRequest(q))
+ require.NoError(t, err)
+ return res.Total > 0
+ }
+
+ // The dominant half is fine.
+ require.True(t, found("вложение", fieldBodyRU), "singular Russian finds the plural in the text")
+ // The minority half is not: it is in the index, but only as a literal.
+ require.True(t, found("attachments", fieldBodyRU), "the exact English word form is still there")
+ require.False(t, found("attachment", fieldBodyRU),
+ "per-document routing leaves the English half unstemmed: singular misses the plural")
+}
+
+// TestPerLineRoutingFindsBothHalvesOfAMixedDocument is the same document
+// through the real path. Each line lands in the field whose analyzer stems it,
+// and both halves answer stemmed queries.
+func TestPerLineRoutingFindsBothHalvesOfAMixedDocument(t *testing.T) {
+ c := newCorpus(t, "~bigbes/specs", "rev1").
+ add("specs/storage.md", "---\nid: SPEC-0001\ntitle: Storage model\nstatus: draft\n---\n\n"+enSpecBody).
+ add("specs/hranenie.md", "---\nid: SPEC-0002\ntitle: Модель хранения\nstatus: draft\n---\n\n"+ruSpecBody).
+ add("specs/attachments.md", "---\nid: SPEC-0003\ntitle: Вложения и двоичные файлы\nstatus: draft\n---\n\n"+mixedSpecBody)
+ idx := indexCorpus(t, c)
+
+ // Singular English query, plural in the text: only the stemmer bridges it.
+ require.Equal(t, []string{"SPEC-0003"}, hitIDs(t, idx, Query{Text: "attachment"}),
+ "the English half of the mixed document is stemmed")
+ // Singular Russian query, plural in the text.
+ require.Equal(t, []string{"SPEC-0003"}, hitIDs(t, idx, Query{Text: "вложение"}),
+ "the Russian half of the mixed document is stemmed")
+
+ // The single-language documents are unaffected.
+ require.Equal(t, []string{"SPEC-0001"}, hitIDs(t, idx, Query{Text: "proposal"}))
+ require.Equal(t, []string{"SPEC-0002"}, hitIDs(t, idx, Query{Text: "предложение"}))
+}
+
+// TestMixedDocumentIsLabelledByItsDominantLanguage checks the label a hit
+// carries. The document is routed per line, but it is still one document and
+// reports one language — which is what picks the snippet's half.
+func TestMixedDocumentIsLabelledByItsDominantLanguage(t *testing.T) {
+ c := newCorpus(t, "~bigbes/specs", "rev1").
+ add("specs/storage.md", "---\nid: SPEC-0001\ntitle: Storage model\n---\n\n"+enSpecBody).
+ add("specs/attachments.md", "---\nid: SPEC-0003\ntitle: Вложения и двоичные файлы\n---\n\n"+mixedSpecBody)
+ idx := indexCorpus(t, c)
+
+ res, err := idx.Search(context.Background(), Query{Text: "attachment"})
+ require.NoError(t, err)
+ require.Len(t, res.Hits, 1)
+ require.Equal(t, LangRU, res.Hits[0].Lang)
+ require.Contains(t, res.Hits[0].Snippet, "<mark>attachments</mark>",
+ "the snippet comes from the half that matched, even though it is not the document's language")
+
+ res, err = idx.Search(context.Background(), Query{Text: "proposal"})
+ require.NoError(t, err)
+ require.Len(t, res.Hits, 1)
+ require.Equal(t, LangEN, res.Hits[0].Lang)
+}
+
+// TestCyrillicTitleIsSearchableAndBoosted checks that titles are routed too: a
+// Russian title lands in the Russian title field, so a query naming a document
+// returns it above documents that merely mention it.
+func TestCyrillicTitleIsSearchableAndBoosted(t *testing.T) {
+ c := newCorpus(t, "~bigbes/specs", "rev1").
+ add("specs/hranenie.md", "---\nid: SPEC-0002\ntitle: Модель хранения данных\n---\n\n"+ruSpecBody).
+ add("specs/mention.md", "---\nid: SPEC-0004\ntitle: Прочее\n---\n\n"+
+ "Здесь упоминается модель хранения данных, но документ совсем о другом предмете.\n")
+ idx := indexCorpus(t, c)
+
+ require.Equal(t, []string{"SPEC-0002", "SPEC-0004"}, hitIDs(t, idx, Query{Text: "модель хранения"}),
+ "the document named by the query outranks the one that mentions it")
+}
A search/search.go => search/search.go +265 -0
@@ 0,0 1,265 @@
+// Package search is spec.sr.ht's keyword search: one global bleve index over
+// every document of every space, queried through a filter.
+//
+// It is warren's index/ + search/ packages absorbed, with three structural
+// changes the design calls for.
+//
+// # One index, filtered at query time
+//
+// warren indexed one vault, so the question never came up. Here it is the
+// central decision: there is exactly one bleve index, every document in it
+// carries its space, and a project — a named set of spaces — is a term filter
+// over that field, not an index of its own. Per-project indexes were specified
+// in an earlier draft and retracted: with them, every merge fans out to N
+// rebuilds, adding a space to a project forces one, and the "everything"
+// project is a second full copy of the corpus. As a filter, the meta-project is
+// genuinely degenerate — a filter that excludes nothing — and a merge touches
+// one index.
+//
+// # Rebuilds, not incremental updates
+//
+// At tens of documents a day, a batch rebuild is cheap and a per-document
+// upsert/delete path is machinery bought against a cost nobody has measured.
+// The unit of a rebuild is therefore a space at a revision (RebuildSpace) or
+// the whole corpus (RebuildAll), never a document. Both report their duration
+// in Stats so the decision to revisit is made against a measurement.
+//
+// # Keyword only
+//
+// warren also had a sqlite-vec semantic index and fused the two rankings with
+// reciprocal rank fusion. Vector search is Phase 5 here, so none of it is
+// ported — not even as unreachable code. What is left in its place is a seam,
+// not a stub: Search returns ranked Hits, and a later hybrid ranker fuses two
+// such lists. Nothing in this package assumes it is the only ranker.
+//
+// The package depends on core/, doc/ and gitx/ document types and on nothing
+// else of this service. In particular it does not touch db/: index staleness
+// stamps live in Postgres and service/ writes them, while this package is
+// handed a revision's worth of documents and returns hits.
+package search
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+ "time"
+
+ "github.com/blevesearch/bleve/v2"
+ bsearch "github.com/blevesearch/bleve/v2/search"
+ "github.com/blevesearch/bleve/v2/search/query"
+
+ "sourcecraft.dev/bigbes/sr-ht-spec/core"
+ "sourcecraft.dev/bigbes/sr-ht-spec/doc"
+)
+
+// DefaultLimit is how many hits a Query with no Limit returns.
+const DefaultLimit = 20
+
+// Query is one search over the global index.
+//
+// The zero value searches nothing: an empty Text returns no hits rather than
+// every document, because "search for nothing" is a caller that has not
+// collected its input yet, not a request to list the corpus.
+type Query struct {
+ // Text is the user's query, analyzed with the same analyzers the documents
+ // were indexed with, per field.
+ Text string
+ // Spaces restricts results to a set of spaces. This is what a project is:
+ // the design's "a project is a saved filter over one global index, not a
+ // container" is this field and nothing else. Empty means every space, which
+ // is the meta-project.
+ Spaces []core.SpaceRef
+ // Sections restricts results to top-level sections ("specs", "notes",
+ // "reports"). Empty means every section except the activity log — see
+ // doc.LogSection: log entries summarise other documents, so left in they
+ // compete with the documents they describe for the same queries. Naming
+ // "log" here is the way back in.
+ Sections []string
+ Limit int
+ Offset int
+}
+
+// Hit is one ranked document. Space, Path, Rev and Anchor together are a
+// pinned, immutable URL for the result: the read plane serves
+// `/~owner/space/path?rev=<sha>#<anchor>`.
+type Hit struct {
+ Space core.SpaceRef `json:"space"`
+ ID string `json:"id"`
+ Rev string `json:"rev,omitempty"`
+ Path string `json:"path,omitempty"`
+ // Anchor is the heading anchor within Path, set for an activity-log entry.
+ Anchor string `json:"anchor,omitempty"`
+ Title string `json:"title,omitempty"`
+ Section string `json:"section,omitempty"`
+ Lang Lang `json:"lang,omitempty"`
+ Score float64 `json:"score"`
+ // Snippet is a highlighted fragment of the matching text, with the matched
+ // terms wrapped in <mark>. Everything around them is HTML-escaped by bleve's
+ // formatter, so the fragment is safe to render as HTML and must be, or the
+ // marks show up as literal text.
+ Snippet string `json:"snippet,omitempty"`
+}
+
+// Results is one page of ranked hits.
+type Results struct {
+ Hits []Hit `json:"hits"`
+ // Total is how many documents matched, not how many were returned.
+ Total uint64 `json:"total"`
+ Took time.Duration `json:"took"`
+}
+
+// Search runs a query against the global index.
+func (x *Index) Search(ctx context.Context, q Query) (Results, error) {
+ text := strings.TrimSpace(q.Text)
+ if text == "" {
+ return Results{}, nil
+ }
+ if q.Limit <= 0 {
+ q.Limit = DefaultLimit
+ }
+ if q.Offset < 0 {
+ return Results{}, fmt.Errorf("search: negative offset %d", q.Offset)
+ }
+ bq, err := buildQuery(text, q)
+ if err != nil {
+ return Results{}, err
+ }
+
+ req := bleve.NewSearchRequestOptions(bq, q.Limit, q.Offset, false)
+ req.Fields = []string{fieldSpace, fieldRev, fieldPath, fieldAnchor, fieldTitle, fieldSection, fieldLang}
+ req.Highlight = bleve.NewHighlight()
+ req.Highlight.AddField(fieldBodyEN)
+ req.Highlight.AddField(fieldBodyRU)
+
+ x.mu.RLock()
+ defer x.mu.RUnlock()
+ if x.idx == nil {
+ return Results{}, errors.New("search: index is closed")
+ }
+ res, err := x.idx.SearchInContext(ctx, req)
+ if err != nil {
+ return Results{}, fmt.Errorf("search: query %q: %w", text, err)
+ }
+
+ out := Results{Total: res.Total, Took: res.Took, Hits: make([]Hit, 0, len(res.Hits))}
+ for _, h := range res.Hits {
+ hit, err := toHit(h)
+ if err != nil {
+ return Results{}, err
+ }
+ out.Hits = append(out.Hits, hit)
+ }
+ return out, nil
+}
+
+// buildQuery assembles the bleve query: the text across both languages' fields,
+// conjoined with the space and section filters.
+func buildQuery(text string, q Query) (query.Query, error) {
+ // The query text is run against all four analyzed fields. Both languages
+ // every time, not the detected language of the query: a two-word query is
+ // far too short to classify, and an English term inside a Russian document
+ // lives in that document's English field.
+ match := func(field string, boost float64) query.Query {
+ m := bleve.NewMatchQuery(text)
+ m.SetField(field)
+ m.SetBoost(boost)
+ return m
+ }
+ b := bleve.NewBooleanQuery()
+ b.AddMust(bleve.NewDisjunctionQuery(
+ match(fieldTitleEN, titleBoost),
+ match(fieldTitleRU, titleBoost),
+ match(fieldBodyEN, 1),
+ match(fieldBodyRU, 1),
+ ))
+
+ if len(q.Spaces) > 0 {
+ want := make([]query.Query, 0, len(q.Spaces))
+ for _, sp := range q.Spaces {
+ if sp.Owner == "" || sp.Name == "" {
+ return nil, errors.New("search: query carries an empty space")
+ }
+ t := bleve.NewTermQuery(sp.String())
+ t.SetField(fieldSpace)
+ want = append(want, t)
+ }
+ b.AddMust(bleve.NewDisjunctionQuery(want...))
+ }
+
+ if len(q.Sections) > 0 {
+ want := make([]query.Query, 0, len(q.Sections))
+ for _, s := range q.Sections {
+ if s == "" {
+ return nil, errors.New("search: query carries an empty section")
+ }
+ t := bleve.NewTermQuery(s)
+ t.SetField(fieldSection)
+ want = append(want, t)
+ }
+ b.AddMust(bleve.NewDisjunctionQuery(want...))
+ } else {
+ t := bleve.NewTermQuery(doc.LogSection)
+ t.SetField(fieldSection)
+ b.AddMustNot(t)
+ }
+ return b, nil
+}
+
+func toHit(h *bsearch.DocumentMatch) (Hit, error) {
+ str := func(field string) string {
+ s, _ := h.Fields[field].(string)
+ return s
+ }
+ raw := str(fieldSpace)
+ if raw == "" {
+ return Hit{}, fmt.Errorf("search: indexed document %q carries no space", h.ID)
+ }
+ sp, err := core.ParseSpaceRef(raw)
+ if err != nil {
+ return Hit{}, fmt.Errorf("search: indexed document %q carries space %q: %w", h.ID, raw, err)
+ }
+ hit := Hit{
+ Space: sp,
+ ID: strings.TrimPrefix(h.ID, raw+":"),
+ Rev: str(fieldRev),
+ Path: str(fieldPath),
+ Anchor: str(fieldAnchor),
+ Title: str(fieldTitle),
+ Section: str(fieldSection),
+ Lang: Lang(str(fieldLang)),
+ Score: h.Score,
+ }
+ hit.Snippet = snippet(h, hit.Lang)
+ return hit, nil
+}
+
+// snippet picks the highlighted fragment to show. The two body fields are the
+// two halves of one document, and bleve highlights every requested field
+// whether or not it matched — a field with no term locations yields its opening
+// text, unmarked. Preferring the document's own language would therefore show
+// the Russian opening of a document that matched on its English half. The
+// matched field is the one that appears in Locations; the language preference
+// only breaks a tie between two halves that both matched.
+func snippet(h *bsearch.DocumentMatch, lang Lang) string {
+ order := []string{fieldBodyEN, fieldBodyRU}
+ if lang == LangRU {
+ order = []string{fieldBodyRU, fieldBodyEN}
+ }
+ for _, field := range order {
+ if len(h.Locations[field]) == 0 {
+ continue
+ }
+ if frags := h.Fragments[field]; len(frags) > 0 {
+ return frags[0]
+ }
+ }
+ // Matched on a title or on nothing highlightable: fall back to whichever
+ // half has text, so a hit is never returned with no context at all.
+ for _, field := range order {
+ if frags := h.Fragments[field]; len(frags) > 0 {
+ return frags[0]
+ }
+ }
+ return ""
+}