package search
import (
"fmt"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
"github.com/blevesearch/bleve/v2/analysis/lang/en"
"github.com/blevesearch/bleve/v2/analysis/lang/ru"
"github.com/blevesearch/bleve/v2/mapping"
"sourcecraft.dev/bigbes/sr-ht-spec/core"
)
// Field names in the bleve index. Three groups, and the difference between them
// is what the whole mapping is:
//
// - filters (space, section, lang) are keyword-analyzed, so they are matched
// as whole strings by a term query and never tokenized. Filtering by a
// space through an analyzed field — which is what warren did for sections —
// means "~bigbes/b-tree" matches the space "~someone/tree", and a project
// is precisely a filter over spaces, so it has to be exact.
// - analyzed text (title_en/title_ru, body_en/body_ru) carries the
// searchable words, one field pair per language. See lang.go.
// - metadata (rev, path, anchor, title) is stored and not indexed: it is what
// turns a hit into a URL, not something to search on.
const (
fieldSpace = "space"
fieldSection = "section"
fieldLang = "lang"
fieldRev = "rev"
fieldPath = "path"
fieldAnchor = "anchor"
fieldTitle = "title"
fieldTitleEN = "title_en"
fieldTitleRU = "title_ru"
fieldBodyEN = "body_en"
fieldBodyRU = "body_ru"
)
// titleBoost is how much more a title match is worth than a body match. Carried
// over from warren unchanged: a query that names a document should return that
// document, not the twenty documents that mention it.
const titleBoost = 3.0
// buildMapping is the index mapping of the one global index. There is exactly
// one, and it is not parameterized by language: the design's earlier
// per-index-language choice is what this replaces.
func buildMapping() mapping.IndexMapping {
text := func(analyzer string, termVectors bool) *mapping.FieldMapping {
f := bleve.NewTextFieldMapping()
f.Analyzer = analyzer
f.Store = true
f.IncludeTermVectors = termVectors
f.IncludeInAll = false
return f
}
exact := func() *mapping.FieldMapping {
f := bleve.NewTextFieldMapping()
f.Analyzer = keyword.Name
f.Store = true
f.IncludeInAll = false
return f
}
meta := func() *mapping.FieldMapping {
f := bleve.NewTextFieldMapping()
f.Index = false
f.Store = true
f.IncludeInAll = false
return f
}
d := bleve.NewDocumentMapping()
// Nothing is indexed that this file does not name. A dynamic mapping would
// silently index whatever a future field happens to be called, with the
// default analyzer, which is how an index acquires fields nobody meant.
d.Dynamic = false
d.AddFieldMappingsAt(fieldSpace, exact())
d.AddFieldMappingsAt(fieldSection, exact())
d.AddFieldMappingsAt(fieldLang, exact())
d.AddFieldMappingsAt(fieldRev, meta())
d.AddFieldMappingsAt(fieldPath, meta())
d.AddFieldMappingsAt(fieldAnchor, meta())
d.AddFieldMappingsAt(fieldTitle, meta())
d.AddFieldMappingsAt(fieldTitleEN, text(en.AnalyzerName, false))
d.AddFieldMappingsAt(fieldTitleRU, text(ru.AnalyzerName, false))
// Body fields carry term vectors because they are the highlighted ones: a
// snippet is reconstructed from stored text plus term locations.
d.AddFieldMappingsAt(fieldBodyEN, text(en.AnalyzerName, true))
d.AddFieldMappingsAt(fieldBodyRU, text(ru.AnalyzerName, true))
m := bleve.NewIndexMapping()
m.DefaultAnalyzer = en.AnalyzerName
m.DefaultMapping = d
return m
}
// Key is the id a document is stored under in the global index. The space is
// part of it because the index is global: two spaces may each hold a document
// whose id fell back to the path "specs/storage", and in a single index those
// are two documents, not one overwriting the other.
//
// It is deliberately not parsed back. Space, path and the rest are stored
// fields; the key is an opaque identity.
func Key(sp core.SpaceRef, id string) string {
return sp.String() + ":" + id
}
// bleveDoc projects a Document into the field map bleve indexes, doing the
// language detection and routing on the way. A map rather than a struct so
// that a field a document has nothing for is absent from the index instead of
// present and empty.
func bleveDoc(d Document) (map[string]any, error) {
if err := d.validate(); err != nil {
return nil, err
}
// Dominant language of the document as a whole, used to label it and as the
// fallback for lines too short to classify on their own. Title first: it is
// the most reliably prose-like text a document has.
lang := DetectIn(d.Title+"\n"+d.Text, DefaultLang)
m := map[string]any{
fieldSpace: d.Space.String(),
fieldLang: string(lang),
}
put := func(field, value string) {
if value != "" {
m[field] = value
}
}
put(fieldSection, d.Section)
put(fieldRev, d.Rev)
put(fieldPath, d.Path)
put(fieldAnchor, d.Anchor)
put(fieldTitle, d.Title)
// A title is one line and takes the document's language; splitting it would
// only ever misfile the shorter half of a name.
if d.Title != "" {
if lang == LangRU {
put(fieldTitleRU, d.Title)
} else {
put(fieldTitleEN, d.Title)
}
}
bodyRU, bodyEN := route(d.Text, lang)
put(fieldBodyRU, bodyRU)
put(fieldBodyEN, bodyEN)
return m, nil
}
func (d Document) validate() error {
if d.Space.Owner == "" || d.Space.Name == "" {
return fmt.Errorf("search: document %q has no space", d.ID)
}
if err := core.ValidateOwner(d.Space.Owner); err != nil {
return fmt.Errorf("search: document %q space owner: %w", d.ID, err)
}
if err := core.ValidateSpaceName(d.Space.Name); err != nil {
return fmt.Errorf("search: document %q space name: %w", d.ID, err)
}
if d.ID == "" {
return fmt.Errorf("search: document in space %s has no id", d.Space)
}
return nil
}