From 36b9e1830fa09548dc188dcf7cfed7750021ad2e Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Wed, 22 Jul 2026 14:21:24 +0300 Subject: [PATCH] =?UTF-8?q?feat:=20search=20=E2=80=94=20one=20global=20ble?= =?UTF-8?q?ve=20index=20with=20per-line=20ru/en=20routing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Absorbs warren's index/ and search/ packages, keyword half only, with the three structural changes the design calls for: - One global index, not one per project. Every document carries its space; a project is a term filter over that field (Query.Spaces), so a merge touches one index and the meta-project is a filter that excludes nothing. - Rebuilds, not incremental updates. RebuildSpace replaces one space at a revision, RebuildAll replaces the corpus by building beside the live index and swapping. Both report duration in Stats. - Keyword only. warren's vector store and RRF fusion are not ported, not even as dead code; Search returns ranked hits a later ranker can fuse. Resolves the design's open mixed Russian/English question. Per-document routing is not sufficient: the ru analyzer passes English through unstemmed and vice versa, so a Russian spec quoting English requirements loses singular/plural matching on whichever half is the minority. Text is routed per line into ru- and en-analyzed field pairs and queried across both. Not ported: chunking (an embedding concern, and the vector path is Phase 5), the graph/pages/meta JSON artifacts, and the vault-commit shell-out. --- search/extract.go | 124 +++++++++++ search/extract_test.go | 151 +++++++++++++ search/fixture_test.go | 82 +++++++ search/index.go | 379 ++++++++++++++++++++++++++++++++ search/index_test.go | 476 +++++++++++++++++++++++++++++++++++++++++ search/lang.go | 119 +++++++++++ search/lang_test.go | 70 ++++++ search/mapping.go | 166 ++++++++++++++ search/mixed_test.go | 165 ++++++++++++++ search/search.go | 265 +++++++++++++++++++++++ 10 files changed, 1997 insertions(+) create mode 100644 search/extract.go create mode 100644 search/extract_test.go create mode 100644 search/fixture_test.go create mode 100644 search/index.go create mode 100644 search/index_test.go create mode 100644 search/lang.go create mode 100644 search/lang_test.go create mode 100644 search/mapping.go create mode 100644 search/mixed_test.go create mode 100644 search/search.go diff --git a/search/extract.go b/search/extract.go new file mode 100644 index 0000000000000000000000000000000000000000..a220949c4335e797b6fa9453efc6414ff3581706 --- /dev/null +++ b/search/extract.go @@ -0,0 +1,124 @@ +package search + +import ( + "fmt" + "path" + "strings" + "sync" + + "sourcecraft.dev/bigbes/sr-ht-spec/doc" + "sourcecraft.dev/bigbes/sr-ht-spec/gitx" +) + +// renderer is shared: doc.Renderer is documented as reusable and +// concurrency-safe, and building one per extraction would rebuild the whole +// goldmark pipeline for every space. +var renderer = sync.OnceValue(doc.NewRenderer) + +// Bodies keys a revision's documents by tree path, which is the shape Extract +// wants them in. It pairs with doc.FromDocuments: the same []gitx.Document +// builds the Archive and supplies the text. +func Bodies(docs []gitx.Document) map[string][]byte { + m := make(map[string][]byte, len(docs)) + for _, d := range docs { + m[d.Path] = d.Data + } + return m +} + +// Extract projects one space at one revision into the documents the index +// stores. bodies holds each page's raw markdown — frontmatter included, exactly +// gitx.Document.Data — keyed by doc.Page.Path. +// +// A page in the archive with no body in bodies is an error, not a page indexed +// with an empty body. The two are indistinguishable once indexed, and the +// second is how a document silently stops being findable. +// +// Three document shapes come out, matching what doc/ models: +// +// - an ordinary document, indexed as its frontmatter projected to "key: +// value" lines followed by its rendered plain text. The frontmatter is in +// there because tags, owners and summaries render as chips rather than +// prose, and a search for one of them should still find the document. +// - a catalog (`type: catalog`, or index.md), indexed by title and section +// only. A catalog is a page of one-line descriptions of other documents; +// indexed whole, a query lands on the description instead of on the +// document that owns it. +// - an activity log (`type: log`, or log.md), which contributes its own +// title-only document plus one document per dated entry. A hit anywhere in +// a log otherwise resolves to the whole log; split, each entry is the size +// of the thing it describes and carries an anchor into it. +func Extract(arc *doc.Archive, bodies map[string][]byte) ([]Document, error) { + if arc == nil { + return nil, fmt.Errorf("search: Extract needs an archive") + } + pages := arc.All() + out := make([]Document, 0, len(pages)) + r := renderer() + + for _, p := range pages { + src, ok := bodies[p.Path] + if !ok { + return nil, fmt.Errorf("search: no body supplied for %s in %s", p.Path, arc.Space) + } + front, body := doc.ParseFront(src) + dir := path.Dir(p.Path) + if dir == "." { + dir = "" + } + res := r.Render(body, dir, arc) + + d := Document{ + Space: arc.Space, + ID: p.ID, + Rev: arc.Rev, + Path: p.Path, + Section: p.Section, + Title: p.Title, + Text: front.SearchText() + res.PlainText, + } + switch p.Kind { + case doc.KindCatalog: + d.Text = "" + case doc.KindLog: + d.Section = doc.LogSection + d.Text = "" + out = append(out, d) + out = append(out, logEntries(arc, p, body)...) + continue + } + out = append(out, d) + } + return out, nil +} + +// logEntries splits an activity log into one indexable document per dated +// entry. +// +// The entry ids doc.SplitLog produces are "log#-", named after +// warren's single vault-wide log. In a space that is not unique: a second +// document marked `type: log` — or simply a second file named log.md in another +// directory — produces the same ids, and in one index the same ids are the same +// documents, so one log would silently overwrite the other. The owning page's +// id is therefore substituted for the "log" prefix, which is a no-op for a log +// whose page id is in fact "log" and disambiguates every other case. The result +// also resolves better: "notes/dev-log#2026-05-31-1" names the document the +// entry is in. +func logEntries(arc *doc.Archive, p *doc.Page, body []byte) []Document { + entries := doc.SplitLog(body) + out := make([]Document, 0, len(entries)) + for _, e := range entries { + suffix := strings.TrimPrefix(e.ID, "log#") + out = append(out, Document{ + Space: arc.Space, + ID: p.ID + "#" + suffix, + Rev: arc.Rev, + Path: p.Path, + Anchor: e.Anchor, + Section: doc.LogSection, + Title: e.Date + " " + e.Title, + Text: e.SearchText(), + }) + } + return out +} diff --git a/search/extract_test.go b/search/extract_test.go new file mode 100644 index 0000000000000000000000000000000000000000..25c824095be3066e280b6247365e288a24517301 --- /dev/null +++ b/search/extract_test.go @@ -0,0 +1,151 @@ +package search + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "sourcecraft.dev/bigbes/sr-ht-spec/doc" + "sourcecraft.dev/bigbes/sr-ht-spec/gitx" +) + +func byID(docs []Document) map[string]Document { + m := make(map[string]Document, len(docs)) + for _, d := range docs { + m[d.ID] = d + } + return m +} + +func TestExtractCarriesSpaceRevisionAndPath(t *testing.T) { + c := newCorpus(t, "~bigbes/specs", "8f14e45fceea167a5a36dedd4bea2543"). + add("specs/storage.md", "---\nid: SPEC-0001\ntitle: Storage model\n---\n\nBodies live in git.\n") + docs := c.extract(t) + + require.Len(t, docs, 1) + require.Equal(t, c.Space, docs[0].Space) + require.Equal(t, "8f14e45fceea167a5a36dedd4bea2543", docs[0].Rev) + require.Equal(t, "specs/storage.md", docs[0].Path) + require.Equal(t, "specs", docs[0].Section) + require.Equal(t, "SPEC-0001", docs[0].ID) + require.Empty(t, docs[0].Anchor) +} + +// Frontmatter is prepended to the searchable text, so a tag or an owner finds +// the document even though neither is prose. Carried over from warren. +func TestExtractIndexesFrontmatterAsText(t *testing.T) { + c := newCorpus(t, "~bigbes/specs", "rev1"). + add("specs/storage.md", "---\nid: SPEC-0001\ntitle: Storage model\n"+ + "tags: [storage, review]\nowners: ['~bigbes']\nsummary: One tier, git objects only\n---\n\n"+ + "Bodies live in git.\n") + docs := c.extract(t) + + require.Len(t, docs, 1) + require.Contains(t, docs[0].Text, "storage") + require.Contains(t, docs[0].Text, "review") + require.Contains(t, docs[0].Text, "One tier, git objects only") + require.Contains(t, docs[0].Text, "Bodies live in git.") +} + +// A catalog is a page of one-line descriptions of other documents. Indexed +// whole, a query lands on the description instead of the document that owns it. +func TestExtractIndexesACatalogByTitleOnly(t *testing.T) { + c := newCorpus(t, "~bigbes/specs", "rev1"). + add("specs/index.md", "# Specifications\n\n- [[SPEC-0001]] — the storage model\n- [[SPEC-0002]] — review\n"). + add("specs/storage.md", "---\nid: SPEC-0001\ntitle: Storage model\n---\n\nBodies live in git.\n") + docs := byID(c.extract(t)) + + catalog, ok := docs["specs/index"] + require.True(t, ok, "the catalog is still indexed: %v", docs) + require.Equal(t, doc.KindCatalog, kindOf(t, c, "specs/index.md")) + require.Empty(t, catalog.Text, "a catalog contributes no body") + require.Equal(t, "Specifications", catalog.Title) +} + +func kindOf(t *testing.T, c *corpus, path string) doc.PageKind { + t.Helper() + arc := doc.FromDocuments(c.Space, c.Rev, c.docs) + p, ok := arc.ByPath(path) + require.True(t, ok) + return p.Kind +} + +// An activity log contributes its own title-only document plus one per dated +// entry, each anchored into the log it came from. Carried over from warren, +// where a hit anywhere in a 248 KB log resolved to the whole file. +func TestExtractSplitsAnActivityLogIntoEntries(t *testing.T) { + c := newCorpus(t, "~bigbes/specs", "rev1"). + add("notes/log.md", "# Work log\n\n"+ + "## [2026-05-31] ingest | Imported the RFC set\nPulled 40 documents in from the old wiki.\n\n"+ + "## [2026-05-30] lint | Fixed frontmatter\nEvery document now carries a status.\n") + docs := byID(c.extract(t)) + require.Len(t, docs, 3) + + page, ok := docs["notes/log"] + require.True(t, ok) + require.Equal(t, doc.LogSection, page.Section, "the log page moves to the log section") + require.Empty(t, page.Text) + + first, ok := docs["notes/log#2026-05-31-1"] + require.True(t, ok, "entries are keyed under their own document: %v", docs) + require.Equal(t, doc.LogSection, first.Section) + require.Equal(t, "notes/log.md", first.Path, "an entry points back at the log it lives in") + require.Equal(t, "e-2026-05-31-1", first.Anchor) + require.Equal(t, "2026-05-31 Imported the RFC set", first.Title) + require.Contains(t, first.Text, "Pulled 40 documents in from the old wiki.") + require.NotContains(t, first.Text, "Fixed frontmatter", "entries do not bleed into each other") +} + +// doc.SplitLog names entries "log#-", after warren's single vault-wide +// log. Two logs in one space would then produce colliding ids, and in one index +// colliding ids are the same document — one log would silently overwrite the +// other. Entry ids are namespaced by their own document to prevent it. +func TestExtractNamespacesLogEntriesPerDocument(t *testing.T) { + entry := "\n## [2026-05-31] ingest | Same day, two logs\nBody.\n" + c := newCorpus(t, "~bigbes/specs", "rev1"). + add("log.md", "# Space log\n"+entry). + add("notes/log.md", "# Notes log\n"+entry) + docs := byID(c.extract(t)) + + require.Contains(t, docs, "log#2026-05-31-1", "a log whose page id is \"log\" keeps warren's ids") + require.Contains(t, docs, "notes/log#2026-05-31-1") + require.Len(t, docs, 4, "two log pages and two entries, none of them merged") +} + +// A page in the archive with no body is a caller bug. Indexing it with an empty +// body would leave a document that exists and is unfindable, which is the +// failure mode that surfaces months later. +func TestExtractRefusesAPageWithNoBody(t *testing.T) { + c := newCorpus(t, "~bigbes/specs", "rev1"). + add("specs/storage.md", "---\nid: SPEC-0001\ntitle: Storage model\n---\n\nBodies live in git.\n") + arc := doc.FromDocuments(c.Space, c.Rev, c.docs) + + _, err := Extract(arc, map[string][]byte{}) + require.ErrorContains(t, err, "no body supplied for specs/storage.md") +} + +// A document whose frontmatter core rejects still renders and still indexes: +// --push-option=skip-validation means a broken header can reach the approved +// branch, and refusing to index it would turn a typo into a silent hole. +func TestExtractIndexesADocumentWithABrokenHeader(t *testing.T) { + c := newCorpus(t, "~bigbes/specs", "rev1"). + add("specs/broken.md", "---\nid: [not, a, string\n---\n\n# Broken but readable\n\nThe body is still prose.\n") + docs := c.extract(t) + + require.Len(t, docs, 1) + require.Equal(t, "specs/broken", docs[0].ID, "it falls back to its path") + require.Contains(t, docs[0].Text, "The body is still prose.") +} + +func TestBodiesKeysByPath(t *testing.T) { + got := Bodies([]gitx.Document{ + {Path: "a.md", Data: []byte("one")}, + {Path: "b/c.md", Data: []byte("two")}, + }) + require.Equal(t, map[string][]byte{"a.md": []byte("one"), "b/c.md": []byte("two")}, got) +} + +func TestExtractNeedsAnArchive(t *testing.T) { + _, err := Extract(nil, nil) + require.ErrorContains(t, err, "needs an archive") +} diff --git a/search/fixture_test.go b/search/fixture_test.go new file mode 100644 index 0000000000000000000000000000000000000000..e9cb0a0d08d922490af1fc20cf746e278c73f996 --- /dev/null +++ b/search/fixture_test.go @@ -0,0 +1,82 @@ +package search + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "sourcecraft.dev/bigbes/sr-ht-spec/core" + "sourcecraft.dev/bigbes/sr-ht-spec/doc" + "sourcecraft.dev/bigbes/sr-ht-spec/gitx" +) + +// space is a shorthand for a valid SpaceRef in tests. +func space(t *testing.T, s string) core.SpaceRef { + t.Helper() + sp, err := core.ParseSpaceRef(s) + require.NoError(t, err) + return sp +} + +// corpus is a space's worth of documents, built in memory. No repository, no +// checkout: doc.FromDocuments is the seam that takes a document set directly, +// and gitx.Document is exactly what a tree walk yields. +type corpus struct { + Space core.SpaceRef + Rev string + docs []gitx.Document +} + +func newCorpus(t *testing.T, ref, rev string) *corpus { + t.Helper() + return &corpus{Space: space(t, ref), Rev: rev} +} + +func (c *corpus) add(path, body string) *corpus { + c.docs = append(c.docs, gitx.Document{Path: path, Data: []byte(body)}) + return c +} + +// extract builds the archive and projects it into indexable documents, which is +// the whole of what service/ will do between a tree walk and a rebuild. +func (c *corpus) extract(t *testing.T) []Document { + t.Helper() + arc := doc.FromDocuments(c.Space, c.Rev, c.docs) + docs, err := Extract(arc, Bodies(c.docs)) + require.NoError(t, err) + return docs +} + +// openIndex opens an empty index in a temp dir, closed at the end of the test. +func openIndex(t *testing.T) *Index { + t.Helper() + idx, err := Open(filepath.Join(t.TempDir(), "spec.bleve")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, idx.Close()) }) + return idx +} + +// indexCorpus opens an index and rebuilds every corpus into it. +func indexCorpus(t *testing.T, corpora ...*corpus) *Index { + t.Helper() + idx := openIndex(t) + for _, c := range corpora { + _, err := idx.RebuildSpace(context.Background(), c.Space, c.extract(t)) + require.NoError(t, err) + } + return idx +} + +// hitIDs is the ordered list of document ids a search returned. +func hitIDs(t *testing.T, idx *Index, q Query) []string { + t.Helper() + res, err := idx.Search(context.Background(), q) + require.NoError(t, err) + ids := make([]string, 0, len(res.Hits)) + for _, h := range res.Hits { + ids = append(ids, h.ID) + } + return ids +} diff --git a/search/index.go b/search/index.go new file mode 100644 index 0000000000000000000000000000000000000000..bd226898dcb91de2ea9848472ff0f9bc238d7f0e --- /dev/null +++ b/search/index.go @@ -0,0 +1,379 @@ +package search + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/blevesearch/bleve/v2" + + "sourcecraft.dev/bigbes/sr-ht-spec/core" +) + +// Document is one unit of the index: a whole document, or one dated entry of an +// activity log. Extract produces these from a doc.Archive; the index stores +// them and hands back Hits shaped the same way. +type Document struct { + // Space is what makes the one global index filterable per project. Every + // document carries it, and a project query is a term filter over the set. + Space core.SpaceRef + // ID is the document's id within its space: doc.Page.ID, or + // "#-" for one entry of an activity log. + ID string + // Rev is the revision the document was read at, carried through so a hit + // can be turned into a pinned `?rev=` URL rather than a link to whatever + // the branch says now. + Rev string + // Path is the document's path in the git tree. For a log entry it is the + // path of the log document the entry came out of. + Path string + // Anchor is the heading anchor within Path a hit should land on. Empty for + // an ordinary document, set for a log entry. + Anchor string + // Section is the top-level directory the document lives under, or + // doc.LogSection for an activity log and its entries. + Section string + Title string + // Text is everything searchable: the frontmatter projected to "key: value" + // lines followed by the rendered plain text. It is split by language and + // stored in the analyzed fields; it is not stored verbatim under its own + // name. + Text string +} + +// Index is the one global bleve index, shared by every space and every project. +// +// One index, not one per project and not one per space. Per-project indexes +// were specified and then retracted in the design for a concrete reason: with N +// projects every merge fans out to N rebuilds and adding a space to a project +// forces one, while the "everything" project is a second full copy of the +// corpus. Here a project is a filter — see Query.Spaces — so a merge touches +// one index and every project containing the space sees the change for free. +// +// bleve is single-writer. One process holds the index open, which is why the +// push hooks are RPC shims into the daemon rather than separate processes. +// Within that process an Index is safe for concurrent use: searches share a +// read lock, and a rebuild takes the write lock. +type Index struct { + path string + + mu sync.RWMutex + idx bleve.Index +} + +// batchSize is how many documents are buffered before a batch is flushed +// during a full rebuild. Carried over from warren. +const batchSize = 200 + +// Stats reports what a rebuild did and how long it took. +// +// The design absorbs warren's batch full rebuild deliberately — at tens of +// documents a day, incremental indexing is machinery bought against a cost +// nobody has measured — and asks for the duration to be instrumented so the +// decision to revisit is triggered by a number rather than a hunch. That is +// what Took is for: it is the trigger, and callers are expected to log it. +type Stats struct { + // Spaces is how many distinct spaces the rebuild wrote. + Spaces int + // Indexed is how many documents were written. + Indexed int + // Deleted is how many stale documents were removed: documents that were in + // the index for a rebuilt space and are not in the new document set. + Deleted int + Took time.Duration +} + +func (s Stats) String() string { + return fmt.Sprintf("spaces=%d indexed=%d deleted=%d took=%s", + s.Spaces, s.Indexed, s.Deleted, s.Took.Round(time.Millisecond)) +} + +// Open opens the global index at path, creating an empty one if it is not +// there. The index is a pure cache: deleting the directory and letting Open +// recreate it, followed by RebuildAll, is always a valid repair. +func Open(path string) (*Index, error) { + if path == "" { + return nil, errors.New("search: index path is required") + } + idx, err := bleve.Open(path) + switch { + case errors.Is(err, bleve.ErrorIndexPathDoesNotExist): + idx, err = bleve.New(path, buildMapping()) + if err != nil { + return nil, fmt.Errorf("search: create index at %s: %w", path, err) + } + case err != nil: + return nil, fmt.Errorf("search: open index at %s: %w", path, err) + } + return &Index{path: path, idx: idx}, nil +} + +// Path is where the index lives on disk. +func (x *Index) Path() string { return x.path } + +// Close releases the index. +func (x *Index) Close() error { + x.mu.Lock() + defer x.mu.Unlock() + if x.idx == nil { + return nil + } + err := x.idx.Close() + x.idx = nil + if err != nil { + return fmt.Errorf("search: close index: %w", err) + } + return nil +} + +// Count is how many documents the index holds, across every space. +func (x *Index) Count() (uint64, error) { + x.mu.RLock() + defer x.mu.RUnlock() + if x.idx == nil { + return 0, errors.New("search: index is closed") + } + n, err := x.idx.DocCount() + if err != nil { + return 0, fmt.Errorf("search: count documents: %w", err) + } + return n, nil +} + +// RebuildSpace replaces everything the index holds for one space. +// +// This is a rebuild, not an incremental update: whatever was indexed for sp is +// removed and docs are written in its place, with no diffing of individual +// documents and no per-document staleness bookkeeping. The unit of freshness is +// a space at a revision, which is exactly what the index_stamp row in Postgres +// records — this package is handed a revision's worth of documents and does not +// know or care which of them changed. +// +// docs may be empty, which empties the space. Every document must belong to sp; +// a document from another space is a caller bug and is refused rather than +// written somewhere surprising. +func (x *Index) RebuildSpace(ctx context.Context, sp core.SpaceRef, docs []Document) (Stats, error) { + start := time.Now() + if sp.Owner == "" || sp.Name == "" { + return Stats{}, errors.New("search: RebuildSpace needs a space") + } + fields := make([]map[string]any, len(docs)) + for i, d := range docs { + if d.Space != sp { + return Stats{}, fmt.Errorf("search: document %q belongs to space %s, not %s", d.ID, d.Space, sp) + } + f, err := bleveDoc(d) + if err != nil { + return Stats{}, err + } + fields[i] = f + } + + x.mu.Lock() + defer x.mu.Unlock() + if x.idx == nil { + return Stats{}, errors.New("search: index is closed") + } + + stale, err := x.keysOf(sp) + if err != nil { + return Stats{}, err + } + if err := ctx.Err(); err != nil { + return Stats{}, err + } + + // One batch for the whole space, so a space is never half-replaced. Deletes + // go in first: a batch is keyed by document id and the last operation on a + // key wins, so a document that survives the rebuild is re-indexed rather + // than dropped. + batch := x.idx.NewBatch() + kept := 0 + for key := range stale { + batch.Delete(key) + } + for i, d := range docs { + key := Key(d.Space, d.ID) + if _, ok := stale[key]; ok { + kept++ + } + if err := batch.Index(key, fields[i]); err != nil { + return Stats{}, fmt.Errorf("search: stage %s: %w", key, err) + } + } + if err := x.idx.Batch(batch); err != nil { + return Stats{}, fmt.Errorf("search: rebuild space %s: %w", sp, err) + } + + st := Stats{Spaces: 1, Indexed: len(docs), Deleted: len(stale) - kept, Took: time.Since(start)} + if len(docs) == 0 { + st.Spaces = 0 + } + return st, nil +} + +// DeleteSpace removes every document of a space from the index. It is what a +// deleted space calls; RebuildSpace with no documents does the same thing. +func (x *Index) DeleteSpace(ctx context.Context, sp core.SpaceRef) (Stats, error) { + return x.RebuildSpace(ctx, sp, nil) +} + +// RebuildAll replaces the entire index with docs, which may span any number of +// spaces. +// +// The rebuild runs into a fresh index beside the live one and the two are +// swapped at the end, so a failure part-way through leaves the old index intact +// and serving. That is the one thing warren's `bleve.New` over the live path +// did not give, and it matters here because the index is open in a daemon that +// is answering queries while the rebuild runs. +func (x *Index) RebuildAll(ctx context.Context, docs []Document) (Stats, error) { + start := time.Now() + spaces := make(map[core.SpaceRef]struct{}) + fields := make([]map[string]any, len(docs)) + for i, d := range docs { + f, err := bleveDoc(d) + if err != nil { + return Stats{}, err + } + fields[i] = f + spaces[d.Space] = struct{}{} + } + + tmp := x.path + ".rebuilding" + if err := os.RemoveAll(tmp); err != nil { + return Stats{}, fmt.Errorf("search: clear %s: %w", tmp, err) + } + fresh, err := bleve.New(tmp, buildMapping()) + if err != nil { + return Stats{}, fmt.Errorf("search: create index at %s: %w", tmp, err) + } + if err := indexAll(ctx, fresh, docs, fields); err != nil { + _ = fresh.Close() + _ = os.RemoveAll(tmp) + return Stats{}, err + } + if err := fresh.Close(); err != nil { + _ = os.RemoveAll(tmp) + return Stats{}, fmt.Errorf("search: close rebuilt index: %w", err) + } + + x.mu.Lock() + defer x.mu.Unlock() + if x.idx == nil { + _ = os.RemoveAll(tmp) + return Stats{}, errors.New("search: index is closed") + } + if err := x.swap(tmp); err != nil { + return Stats{}, err + } + return Stats{ + Spaces: len(spaces), + Indexed: len(docs), + Took: time.Since(start), + }, nil +} + +// swap puts the freshly built index at tmp in place of the live one. The caller +// holds the write lock. +// +// The old directory is renamed aside rather than deleted first, so the window in +// which neither exists is a rename rather than a recursive delete. If reopening +// the new index fails the Index is left closed and the error is returned: the +// index is a cache, and a caller that cannot open it must rebuild it, not +// silently serve an empty one. +func (x *Index) swap(tmp string) error { + if err := x.idx.Close(); err != nil { + x.idx = nil + return fmt.Errorf("search: close live index: %w", err) + } + x.idx = nil + + old := x.path + ".old" + if err := os.RemoveAll(old); err != nil { + return fmt.Errorf("search: clear %s: %w", old, err) + } + if err := os.Rename(x.path, old); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("search: move live index aside: %w", err) + } + if err := os.Rename(tmp, x.path); err != nil { + return fmt.Errorf("search: move rebuilt index into place: %w", err) + } + idx, err := bleve.Open(x.path) + if err != nil { + return fmt.Errorf("search: reopen index at %s: %w", x.path, err) + } + x.idx = idx + if err := os.RemoveAll(old); err != nil { + return fmt.Errorf("search: remove %s: %w", old, err) + } + return nil +} + +func indexAll(ctx context.Context, idx bleve.Index, docs []Document, fields []map[string]any) error { + batch := idx.NewBatch() + for i, d := range docs { + if err := ctx.Err(); err != nil { + return err + } + key := Key(d.Space, d.ID) + if err := batch.Index(key, fields[i]); err != nil { + return fmt.Errorf("search: stage %s: %w", key, err) + } + if batch.Size() >= batchSize { + if err := idx.Batch(batch); err != nil { + return fmt.Errorf("search: flush batch: %w", err) + } + batch = idx.NewBatch() + } + } + if batch.Size() > 0 { + if err := idx.Batch(batch); err != nil { + return fmt.Errorf("search: flush final batch: %w", err) + } + } + return nil +} + +// keysOf lists the index keys currently held for a space. The caller holds a +// lock. +func (x *Index) keysOf(sp core.SpaceRef) (map[string]struct{}, error) { + q := bleve.NewTermQuery(sp.String()) + q.SetField(fieldSpace) + + const page = 1000 + keys := make(map[string]struct{}) + for from := 0; ; from += page { + req := bleve.NewSearchRequestOptions(q, page, from, false) + res, err := x.idx.Search(req) + if err != nil { + return nil, fmt.Errorf("search: list documents of %s: %w", sp, err) + } + for _, h := range res.Hits { + keys[h.ID] = struct{}{} + } + if len(res.Hits) < page { + return keys, nil + } + } +} + +// tmpPaths are the working directories a rebuild uses, exposed only so a caller +// cleaning up after a crash knows what to look for. +func tmpPaths(path string) []string { + return []string{path + ".rebuilding", path + ".old"} +} + +// CleanStale removes the working directories a crashed rebuild may have left +// behind next to the index. Safe to call at startup, before Open. +func CleanStale(path string) error { + for _, p := range tmpPaths(path) { + if err := os.RemoveAll(p); err != nil { + return fmt.Errorf("search: remove %s: %w", filepath.Base(p), err) + } + } + return nil +} diff --git a/search/index_test.go b/search/index_test.go new file mode 100644 index 0000000000000000000000000000000000000000..6001acd92df11b49435ea34be41854a13fa191f9 --- /dev/null +++ b/search/index_test.go @@ -0,0 +1,476 @@ +package search + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "sourcecraft.dev/bigbes/sr-ht-spec/core" + "sourcecraft.dev/bigbes/sr-ht-spec/doc" +) + +// Ported from warren: an empty or whitespace-only query returns nothing rather +// than everything. +func TestSearchIgnoresWhitespaceOnlyQuery(t *testing.T) { + idx := indexCorpus(t, newCorpus(t, "~bigbes/specs", "rev1"). + add("specs/storage.md", "---\ntitle: Needle handbook\n---\n\nreference material\n")) + + for _, q := range []string{"", " \t\n "} { + res, err := idx.Search(context.Background(), Query{Text: q}) + require.NoError(t, err) + require.Empty(t, res.Hits) + require.Zero(t, res.Total) + } +} + +// Ported from warren's keyword fixture: a title match outranks a body match, so +// a query that names a document returns the document. +func TestTitleMatchOutranksBodyMatch(t *testing.T) { + idx := indexCorpus(t, newCorpus(t, "~bigbes/specs", "rev1"). + add("specs/handbook.md", "---\nid: SPEC-0001\ntitle: Needle handbook\n---\n\nreference material\n"). + add("specs/other.md", "---\nid: SPEC-0002\ntitle: Other document\n---\n\na needle appears in the body\n")) + + require.Equal(t, []string{"SPEC-0001", "SPEC-0002"}, hitIDs(t, idx, Query{Text: "needle"})) +} + +// A project is a saved filter over the one global index. This is the whole of +// it: the same index, queried with a space set. +func TestProjectIsASpaceFilterOverOneIndex(t *testing.T) { + rfcs := newCorpus(t, "~bigbes/rfcs", "rev1"). + add("specs/storage.md", "---\nid: SPEC-0001\ntitle: Storage model\n---\n\nA shared vocabulary term.\n") + ops := newCorpus(t, "~bigbes/home-ops", "rev1"). + add("notes/hosts.md", "---\nid: NOTE-0001\ntitle: Hosts\n---\n\nAnother shared vocabulary term.\n") + other := newCorpus(t, "~someone/private", "rev1"). + add("specs/x.md", "---\nid: SPEC-0009\ntitle: Elsewhere\n---\n\nA third shared vocabulary term.\n") + idx := indexCorpus(t, rfcs, ops, other) + + // The meta-project: a filter that excludes nothing. + all := hitIDs(t, idx, Query{Text: "vocabulary"}) + require.ElementsMatch(t, []string{"SPEC-0001", "NOTE-0001", "SPEC-0009"}, all) + + // A project over two of the three spaces. + project := hitIDs(t, idx, Query{Text: "vocabulary", Spaces: []core.SpaceRef{rfcs.Space, ops.Space}}) + require.ElementsMatch(t, []string{"SPEC-0001", "NOTE-0001"}, project) + + // One space. + require.Equal(t, []string{"SPEC-0009"}, + hitIDs(t, idx, Query{Text: "vocabulary", Spaces: []core.SpaceRef{other.Space}})) +} + +// Space names are matched whole. Filtering through an analyzed field — which is +// what warren did for sections — would tokenize "~bigbes/home-ops" and let a +// query for one space return another. +func TestSpaceFilterMatchesWholeNamesOnly(t *testing.T) { + ops := newCorpus(t, "~bigbes/home-ops", "rev1"). + add("notes/a.md", "---\nid: NOTE-0001\ntitle: A\n---\n\nshared vocabulary\n") + home := newCorpus(t, "~bigbes/home", "rev1"). + add("notes/b.md", "---\nid: NOTE-0002\ntitle: B\n---\n\nshared vocabulary\n") + idx := indexCorpus(t, ops, home) + + require.Equal(t, []string{"NOTE-0002"}, + hitIDs(t, idx, Query{Text: "vocabulary", Spaces: []core.SpaceRef{home.Space}})) +} + +func TestSearchRejectsAnEmptySpaceFilter(t *testing.T) { + idx := indexCorpus(t, newCorpus(t, "~bigbes/specs", "rev1"). + add("a.md", "---\ntitle: A\n---\n\nbody text here\n")) + + _, err := idx.Search(context.Background(), Query{Text: "body", Spaces: []core.SpaceRef{{}}}) + require.ErrorContains(t, err, "empty space") +} + +// Ported from warren: log entries summarise other documents, so they are out of +// an unrestricted search and reachable by naming the section. +func TestLogEntriesAreExcludedUntilAskedFor(t *testing.T) { + idx := indexCorpus(t, newCorpus(t, "~bigbes/specs", "rev1"). + add("specs/storage.md", "---\nid: SPEC-0001\ntitle: Storage model\n---\n\nThe write path resolves a tree.\n"). + add("log.md", "# Log\n\n## [2026-05-31] update | Storage model\nRewrote the write path section.\n")) + + require.Equal(t, []string{"SPEC-0001"}, hitIDs(t, idx, Query{Text: "write path"})) + // Naming the section is the way back in. The log document itself carries no + // body, so only its entry matches the text. + require.Equal(t, []string{"log#2026-05-31-1"}, + hitIDs(t, idx, Query{Text: "write path", Sections: []string{doc.LogSection}})) + require.Equal(t, []string{"log"}, + hitIDs(t, idx, Query{Text: "Log", Sections: []string{doc.LogSection}}), + "the log document stays findable by name") +} + +func TestSectionFilterRestrictsResults(t *testing.T) { + idx := indexCorpus(t, newCorpus(t, "~bigbes/specs", "rev1"). + add("specs/storage.md", "---\nid: SPEC-0001\ntitle: Storage\n---\n\nshared vocabulary term\n"). + add("notes/scratch.md", "---\nid: NOTE-0001\ntitle: Scratch\n---\n\nshared vocabulary term\n")) + + require.Equal(t, []string{"SPEC-0001"}, hitIDs(t, idx, Query{Text: "vocabulary", Sections: []string{"specs"}})) + require.Equal(t, []string{"NOTE-0001"}, hitIDs(t, idx, Query{Text: "vocabulary", Sections: []string{"notes"}})) + require.Len(t, hitIDs(t, idx, Query{Text: "vocabulary", Sections: []string{"specs", "notes"}}), 2) +} + +func TestSearchRejectsAnEmptySection(t *testing.T) { + idx := indexCorpus(t, newCorpus(t, "~bigbes/specs", "rev1"). + add("a.md", "---\ntitle: A\n---\n\nbody text here\n")) + + _, err := idx.Search(context.Background(), Query{Text: "body", Sections: []string{""}}) + require.ErrorContains(t, err, "empty section") +} + +func TestHitCarriesAPinnedAddress(t *testing.T) { + idx := indexCorpus(t, newCorpus(t, "~bigbes/specs", "8f14e45fceea167a"). + add("specs/storage.md", "---\nid: SPEC-0001\ntitle: Storage model\n---\n\nThe write path resolves a tree.\n")) + + res, err := idx.Search(context.Background(), Query{Text: "resolves"}) + require.NoError(t, err) + require.Len(t, res.Hits, 1) + h := res.Hits[0] + require.Equal(t, space(t, "~bigbes/specs"), h.Space) + require.Equal(t, "SPEC-0001", h.ID) + require.Equal(t, "8f14e45fceea167a", h.Rev) + require.Equal(t, "specs/storage.md", h.Path) + require.Equal(t, "specs", h.Section) + require.Equal(t, "Storage model", h.Title) + require.Equal(t, LangEN, h.Lang) + require.Greater(t, h.Score, 0.0) + require.Contains(t, h.Snippet, "resolves") +} + +// Snippets are rendered as HTML by the review UI, so the text around the marks +// must be escaped. bleve's html formatter does it; this pins the behaviour. +func TestSnippetIsHTMLEscaped(t *testing.T) { + idx := indexCorpus(t, newCorpus(t, "~bigbes/specs", "rev1"). + add("specs/x.md", "---\nid: SPEC-0001\ntitle: Escaping\n---\n\n"+ + "A needle inside `` and more prose after it.\n")) + + res, err := idx.Search(context.Background(), Query{Text: "needle"}) + require.NoError(t, err) + require.Len(t, res.Hits, 1) + require.Contains(t, res.Hits[0].Snippet, "<script>") + require.NotContains(t, res.Hits[0].Snippet, "