~bigbes/sr-ht-spec

ref: e97532ccd54a6aacc597e9db1648e580e2936132 sr-ht-spec/search/mapping.go -rw-r--r-- 5.6 KiB
e97532cc — Eugene Blikh test(db): guard OpenProposal branch SQL against core.ProposalBranch drift (spec-wcr #1) 25 days ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
package search

import (
	"fmt"

	"github.com/blevesearch/bleve/v2"
	"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
	"github.com/blevesearch/bleve/v2/analysis/lang/en"
	"github.com/blevesearch/bleve/v2/analysis/lang/ru"
	"github.com/blevesearch/bleve/v2/mapping"

	"sourcecraft.dev/bigbes/sr-ht-spec/core"
)

// Field names in the bleve index. Three groups, and the difference between them
// is what the whole mapping is:
//
//   - filters (space, section, lang) are keyword-analyzed, so they are matched
//     as whole strings by a term query and never tokenized. Filtering by a
//     space through an analyzed field — which is what warren did for sections —
//     means "~bigbes/b-tree" matches the space "~someone/tree", and a project
//     is precisely a filter over spaces, so it has to be exact.
//   - analyzed text (title_en/title_ru, body_en/body_ru) carries the
//     searchable words, one field pair per language. See lang.go.
//   - metadata (rev, path, anchor, title) is stored and not indexed: it is what
//     turns a hit into a URL, not something to search on.
const (
	fieldSpace   = "space"
	fieldSection = "section"
	fieldLang    = "lang"
	fieldRev     = "rev"
	fieldPath    = "path"
	fieldAnchor  = "anchor"
	fieldTitle   = "title"
	fieldTitleEN = "title_en"
	fieldTitleRU = "title_ru"
	fieldBodyEN  = "body_en"
	fieldBodyRU  = "body_ru"
)

// titleBoost is how much more a title match is worth than a body match. Carried
// over from warren unchanged: a query that names a document should return that
// document, not the twenty documents that mention it.
const titleBoost = 3.0

// buildMapping is the index mapping of the one global index. There is exactly
// one, and it is not parameterized by language: the design's earlier
// per-index-language choice is what this replaces.
func buildMapping() mapping.IndexMapping {
	text := func(analyzer string, termVectors bool) *mapping.FieldMapping {
		f := bleve.NewTextFieldMapping()
		f.Analyzer = analyzer
		f.Store = true
		f.IncludeTermVectors = termVectors
		f.IncludeInAll = false
		return f
	}
	exact := func() *mapping.FieldMapping {
		f := bleve.NewTextFieldMapping()
		f.Analyzer = keyword.Name
		f.Store = true
		f.IncludeInAll = false
		return f
	}
	meta := func() *mapping.FieldMapping {
		f := bleve.NewTextFieldMapping()
		f.Index = false
		f.Store = true
		f.IncludeInAll = false
		return f
	}

	d := bleve.NewDocumentMapping()
	// Nothing is indexed that this file does not name. A dynamic mapping would
	// silently index whatever a future field happens to be called, with the
	// default analyzer, which is how an index acquires fields nobody meant.
	d.Dynamic = false
	d.AddFieldMappingsAt(fieldSpace, exact())
	d.AddFieldMappingsAt(fieldSection, exact())
	d.AddFieldMappingsAt(fieldLang, exact())
	d.AddFieldMappingsAt(fieldRev, meta())
	d.AddFieldMappingsAt(fieldPath, meta())
	d.AddFieldMappingsAt(fieldAnchor, meta())
	d.AddFieldMappingsAt(fieldTitle, meta())
	d.AddFieldMappingsAt(fieldTitleEN, text(en.AnalyzerName, false))
	d.AddFieldMappingsAt(fieldTitleRU, text(ru.AnalyzerName, false))
	// Body fields carry term vectors because they are the highlighted ones: a
	// snippet is reconstructed from stored text plus term locations.
	d.AddFieldMappingsAt(fieldBodyEN, text(en.AnalyzerName, true))
	d.AddFieldMappingsAt(fieldBodyRU, text(ru.AnalyzerName, true))

	m := bleve.NewIndexMapping()
	m.DefaultAnalyzer = en.AnalyzerName
	m.DefaultMapping = d
	return m
}

// Key is the id a document is stored under in the global index. The space is
// part of it because the index is global: two spaces may each hold a document
// whose id fell back to the path "specs/storage", and in a single index those
// are two documents, not one overwriting the other.
//
// It is deliberately not parsed back. Space, path and the rest are stored
// fields; the key is an opaque identity.
func Key(sp core.SpaceRef, id string) string {
	return sp.String() + ":" + id
}

// bleveDoc projects a Document into the field map bleve indexes, doing the
// language detection and routing on the way. A map rather than a struct so
// that a field a document has nothing for is absent from the index instead of
// present and empty.
func bleveDoc(d Document) (map[string]any, error) {
	if err := d.validate(); err != nil {
		return nil, err
	}
	// Dominant language of the document as a whole, used to label it and as the
	// fallback for lines too short to classify on their own. Title first: it is
	// the most reliably prose-like text a document has.
	lang := DetectIn(d.Title+"\n"+d.Text, DefaultLang)

	m := map[string]any{
		fieldSpace: d.Space.String(),
		fieldLang:  string(lang),
	}
	put := func(field, value string) {
		if value != "" {
			m[field] = value
		}
	}
	put(fieldSection, d.Section)
	put(fieldRev, d.Rev)
	put(fieldPath, d.Path)
	put(fieldAnchor, d.Anchor)
	put(fieldTitle, d.Title)

	// A title is one line and takes the document's language; splitting it would
	// only ever misfile the shorter half of a name.
	if d.Title != "" {
		if lang == LangRU {
			put(fieldTitleRU, d.Title)
		} else {
			put(fieldTitleEN, d.Title)
		}
	}
	bodyRU, bodyEN := route(d.Text, lang)
	put(fieldBodyRU, bodyRU)
	put(fieldBodyEN, bodyEN)
	return m, nil
}

func (d Document) validate() error {
	if d.Space.Owner == "" || d.Space.Name == "" {
		return fmt.Errorf("search: document %q has no space", d.ID)
	}
	if err := core.ValidateOwner(d.Space.Owner); err != nil {
		return fmt.Errorf("search: document %q space owner: %w", d.ID, err)
	}
	if err := core.ValidateSpaceName(d.Space.Name); err != nil {
		return fmt.Errorf("search: document %q space name: %w", d.ID, err)
	}
	if d.ID == "" {
		return fmt.Errorf("search: document in space %s has no id", d.Space)
	}
	return nil
}