package search import ( "fmt" "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" "github.com/blevesearch/bleve/v2/analysis/lang/en" "github.com/blevesearch/bleve/v2/analysis/lang/ru" "github.com/blevesearch/bleve/v2/mapping" "sourcecraft.dev/bigbes/sr-ht-spec/core" ) // Field names in the bleve index. Three groups, and the difference between them // is what the whole mapping is: // // - filters (space, section, lang) are keyword-analyzed, so they are matched // as whole strings by a term query and never tokenized. Filtering by a // space through an analyzed field — which is what warren did for sections — // means "~bigbes/b-tree" matches the space "~someone/tree", and a project // is precisely a filter over spaces, so it has to be exact. // - analyzed text (title_en/title_ru, body_en/body_ru) carries the // searchable words, one field pair per language. See lang.go. // - metadata (rev, path, anchor, title) is stored and not indexed: it is what // turns a hit into a URL, not something to search on. const ( fieldSpace = "space" fieldSection = "section" fieldLang = "lang" fieldRev = "rev" fieldPath = "path" fieldAnchor = "anchor" fieldTitle = "title" fieldTitleEN = "title_en" fieldTitleRU = "title_ru" fieldBodyEN = "body_en" fieldBodyRU = "body_ru" ) // titleBoost is how much more a title match is worth than a body match. Carried // over from warren unchanged: a query that names a document should return that // document, not the twenty documents that mention it. const titleBoost = 3.0 // buildMapping is the index mapping of the one global index. There is exactly // one, and it is not parameterized by language: the design's earlier // per-index-language choice is what this replaces. func buildMapping() mapping.IndexMapping { text := func(analyzer string, termVectors bool) *mapping.FieldMapping { f := bleve.NewTextFieldMapping() f.Analyzer = analyzer f.Store = true f.IncludeTermVectors = termVectors f.IncludeInAll = false return f } exact := func() *mapping.FieldMapping { f := bleve.NewTextFieldMapping() f.Analyzer = keyword.Name f.Store = true f.IncludeInAll = false return f } meta := func() *mapping.FieldMapping { f := bleve.NewTextFieldMapping() f.Index = false f.Store = true f.IncludeInAll = false return f } d := bleve.NewDocumentMapping() // Nothing is indexed that this file does not name. A dynamic mapping would // silently index whatever a future field happens to be called, with the // default analyzer, which is how an index acquires fields nobody meant. d.Dynamic = false d.AddFieldMappingsAt(fieldSpace, exact()) d.AddFieldMappingsAt(fieldSection, exact()) d.AddFieldMappingsAt(fieldLang, exact()) d.AddFieldMappingsAt(fieldRev, meta()) d.AddFieldMappingsAt(fieldPath, meta()) d.AddFieldMappingsAt(fieldAnchor, meta()) d.AddFieldMappingsAt(fieldTitle, meta()) d.AddFieldMappingsAt(fieldTitleEN, text(en.AnalyzerName, false)) d.AddFieldMappingsAt(fieldTitleRU, text(ru.AnalyzerName, false)) // Body fields carry term vectors because they are the highlighted ones: a // snippet is reconstructed from stored text plus term locations. d.AddFieldMappingsAt(fieldBodyEN, text(en.AnalyzerName, true)) d.AddFieldMappingsAt(fieldBodyRU, text(ru.AnalyzerName, true)) m := bleve.NewIndexMapping() m.DefaultAnalyzer = en.AnalyzerName m.DefaultMapping = d return m } // Key is the id a document is stored under in the global index. The space is // part of it because the index is global: two spaces may each hold a document // whose id fell back to the path "specs/storage", and in a single index those // are two documents, not one overwriting the other. // // It is deliberately not parsed back. Space, path and the rest are stored // fields; the key is an opaque identity. func Key(sp core.SpaceRef, id string) string { return sp.String() + ":" + id } // bleveDoc projects a Document into the field map bleve indexes, doing the // language detection and routing on the way. A map rather than a struct so // that a field a document has nothing for is absent from the index instead of // present and empty. func bleveDoc(d Document) (map[string]any, error) { if err := d.validate(); err != nil { return nil, err } // Dominant language of the document as a whole, used to label it and as the // fallback for lines too short to classify on their own. Title first: it is // the most reliably prose-like text a document has. lang := DetectIn(d.Title+"\n"+d.Text, DefaultLang) m := map[string]any{ fieldSpace: d.Space.String(), fieldLang: string(lang), } put := func(field, value string) { if value != "" { m[field] = value } } put(fieldSection, d.Section) put(fieldRev, d.Rev) put(fieldPath, d.Path) put(fieldAnchor, d.Anchor) put(fieldTitle, d.Title) // A title is one line and takes the document's language; splitting it would // only ever misfile the shorter half of a name. if d.Title != "" { if lang == LangRU { put(fieldTitleRU, d.Title) } else { put(fieldTitleEN, d.Title) } } bodyRU, bodyEN := route(d.Text, lang) put(fieldBodyRU, bodyRU) put(fieldBodyEN, bodyEN) return m, nil } func (d Document) validate() error { if d.Space.Owner == "" || d.Space.Name == "" { return fmt.Errorf("search: document %q has no space", d.ID) } if err := core.ValidateOwner(d.Space.Owner); err != nil { return fmt.Errorf("search: document %q space owner: %w", d.ID, err) } if err := core.ValidateSpaceName(d.Space.Name); err != nil { return fmt.Errorf("search: document %q space name: %w", d.ID, err) } if d.ID == "" { return fmt.Errorf("search: document in space %s has no id", d.Space) } return nil }