~bigbes/sr-ht-dolt

ref: ede3b0bb2671ffde35a66527b920c40eaccc096f sr-ht-dolt/storage/init.go -rw-r--r-- 9.6 KiB
ede3b0bb — Eugene Blikh go.mod: take the shared libraries' current heads 2 days ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
// Package storage owns dolt.sr.ht's on-disk NBS chunk stores and the
// remotesapi DBCache that serves them.
//
// A hosted database is a bare NBS chunk-store directory (no ".dolt/", no
// working set) laid out at "<root>/~<owner>/<name>". This is exactly what
// remotesrv serves and what "file://" dolt remotes consume, so InitStore can
// create one with the low-level doltdb primitives and remotesrv can read and
// write it directly.
//
// # remotesrv FS working directory (load-bearing)
//
// remotesrv seals its chunk-download URLs relative to the working directory of
// the filesys it is given. The server MUST be constructed with
// filesys.LocalFilesysWithWorkingDir(root) pointing at the repos root — NOT a
// plain filesys.LocalFS. With a working-dir-rooted FS the sealed URLs carry
// clean relative prefixes (e.g. "~owner/db"); with a bare LocalFS they carry
// "../../.." escapes that the sealed-URL file handler rejects, and every clone
// or push breaks at the chunk-transfer stage. This was proven end-to-end in the
// Phase-0 spike (storage/spike_test.go). Cache in this package keys stores by
// absolute disk path, so it works with either FS, but the server assembly in
// remoteapi/ must still honor this rule.
package storage

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
	"github.com/dolthub/dolt/go/libraries/utils/earl"
	"github.com/dolthub/dolt/go/libraries/utils/filesys"
	"github.com/dolthub/dolt/go/store/nbs"
	"github.com/dolthub/dolt/go/store/types"
)

// RepoDiskPath returns the absolute on-disk store directory for a database,
// laid out as "<root>/~<owner>/<name>". Callers are responsible for validating
// owner and name (see core.ValidateName) before touching disk.
func RepoDiskPath(root, owner, name string) string {
	return filepath.Join(root, "~"+owner, name)
}

// InitStore creates a bare NBS chunk store at absPath and writes an empty repo
// into it with a single "main" branch and an initial commit authored by
// ownerName/ownerEmail.
//
// It is the OPT-IN half of database creation — the create form's "initialize
// with an empty commit" checkbox — and not what any automatic path uses. The
// initial commit it writes is history, so a client pushing a database that has
// its own history is pushing a non-fast-forward and must --force. What it buys
// in exchange is a database that can be cloned before anything is pushed to it,
// which InitEmptyStore's result cannot.
//
// absPath must be absolute. On any failure after the directory is created,
// InitStore removes absPath so a failed creation never leaves a partial store
// behind. Idempotence is NOT provided: calling InitStore on an existing store
// is a caller error and is not defended against here.
func InitStore(ctx context.Context, absPath, ownerName, ownerEmail string) (err error) {
	if !filepath.IsAbs(absPath) {
		return fmt.Errorf("storage: InitStore requires an absolute path, got %q", absPath)
	}

	if err := os.MkdirAll(absPath, 0o755); err != nil {
		return fmt.Errorf("storage: create store dir %q: %w", absPath, err)
	}
	// Any error past this point must not leave a half-written store behind.
	defer func() {
		if err != nil {
			os.RemoveAll(absPath)
		}
	}()

	fileURL := earl.FileUrlFromPath(absPath, os.PathSeparator)
	ddb, err := doltdb.LoadDoltDB(ctx, types.Format_DOLT, fileURL, filesys.LocalFS)
	if err != nil {
		return fmt.Errorf("storage: load doltdb at %q: %w", fileURL, err)
	}

	if err = ddb.WriteEmptyRepo(ctx, "main", ownerName, ownerEmail); err != nil {
		ddb.Close()
		return fmt.Errorf("storage: write empty repo at %q: %w", absPath, err)
	}

	// Release the init handle so the server can open its own store over the
	// same directory later.
	if err = ddb.Close(); err != nil {
		return fmt.Errorf("storage: close init doltdb at %q: %w", absPath, err)
	}
	return nil
}

// InitEmptyStore creates a genuinely empty bare NBS chunk store at absPath —
// a directory whose store root is the empty hash, with NO commits and NO
// working set. Unlike InitStore it deliberately does NOT call WriteEmptyRepo:
// an initial "Initialize data repository" commit would make the first push to
// this store a non-fast-forward and be rejected. An empty store lets the
// client's first push land as the initial history. Used by push-to-create.
// absPath must be absolute; on any failure the directory is removed.
func InitEmptyStore(ctx context.Context, absPath string) (err error) {
	if !filepath.IsAbs(absPath) {
		return fmt.Errorf("storage: InitEmptyStore requires an absolute path, got %q", absPath)
	}

	if err := os.MkdirAll(absPath, 0o755); err != nil {
		return fmt.Errorf("storage: create store dir %q: %w", absPath, err)
	}
	// Any error past this point must not leave a half-written store behind.
	defer func() {
		if err != nil {
			os.RemoveAll(absPath)
		}
	}()

	// Opening a plain (non-generational) NBS store over the freshly created,
	// empty directory both validates the directory (checkDir requires it to
	// exist) and confirms the store is a valid empty store (a missing manifest
	// is treated lazily as an empty store with the null root). This mirrors the
	// construction storage.Cache.Get uses to serve pushes; a plain store closes
	// cleanly, unlike the generational store LoadDoltDB routes through.
	cs, err := nbs.NewLocalStore(ctx, types.Format_DOLT.VersionString(), absPath, defaultMemTableSize, nbs.NewUnlimitedMemQuotaProvider(), false)
	if err != nil {
		return fmt.Errorf("storage: open empty store at %q: %w", absPath, err)
	}
	if err = cs.Close(); err != nil {
		return fmt.Errorf("storage: close empty store at %q: %w", absPath, err)
	}
	return nil
}

// containedPath checks that absPath is an absolute path strictly inside the
// absolute root and returns both cleaned. It is the guard every destructive
// path operation in this package runs first, so a corrupted or
// attacker-controlled path can never escape the configured repos root — and
// the root itself is never a legal target. op names the caller ("DeleteStore",
// "MoveStore") so a refusal says which operation was stopped.
func containedPath(op, root, absPath string) (cleanRoot, cleanPath string, err error) {
	if !filepath.IsAbs(root) {
		return "", "", fmt.Errorf("storage: %s requires an absolute root, got %q", op, root)
	}
	if !filepath.IsAbs(absPath) {
		return "", "", fmt.Errorf("storage: %s requires an absolute path, got %q", op, absPath)
	}

	cleanRoot = filepath.Clean(root)
	cleanPath = filepath.Clean(absPath)
	if cleanPath == cleanRoot {
		return "", "", fmt.Errorf("storage: %s refuses to act on the repos root %q", op, cleanRoot)
	}
	rel, err := filepath.Rel(cleanRoot, cleanPath)
	if err != nil {
		return "", "", fmt.Errorf("storage: %s rel(%q, %q): %w", op, cleanRoot, cleanPath, err)
	}
	if rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
		return "", "", fmt.Errorf("storage: %s refuses %q outside repos root %q", op, cleanPath, cleanRoot)
	}
	return cleanRoot, cleanPath, nil
}

// DeleteStore removes the store directory at absPath. It refuses to delete
// anything that is not strictly contained within root, guarding against a
// corrupted or attacker-controlled path escaping the configured repos root.
// Both root and absPath must be absolute.
func DeleteStore(ctx context.Context, root, absPath string) error {
	_, cleanPath, err := containedPath("DeleteStore", root, absPath)
	if err != nil {
		return err
	}

	if err := os.RemoveAll(cleanPath); err != nil {
		return fmt.Errorf("storage: remove store %q: %w", cleanPath, err)
	}
	return nil
}

// MoveStore relocates the store directory at srcPath to dstPath, the on-disk
// half of a rename. Both paths must be absolute and strictly inside root.
//
// It never overwrites: an existing dstPath is refused before anything is
// touched, because os.Rename over an empty destination directory would succeed
// silently and swallow it. A missing srcPath is likewise an error rather than a
// no-op — a rename whose store never moved would leave the metadata row
// pointing at nothing.
//
// The move itself is one os.Rename, so within a filesystem it is atomic: the
// store is either wholly at the old path or wholly at the new one, never half
// copied. Open handles on the old directory survive it (the inodes move, not
// the files), but they keep resolving new writes against the old path string,
// so the caller must still evict any memoized handle — see Cache.Evict.
func MoveStore(ctx context.Context, root, srcPath, dstPath string) error {
	_, cleanSrc, err := containedPath("MoveStore", root, srcPath)
	if err != nil {
		return err
	}
	_, cleanDst, err := containedPath("MoveStore", root, dstPath)
	if err != nil {
		return err
	}
	if cleanSrc == cleanDst {
		return fmt.Errorf("storage: MoveStore source and destination are the same path %q", cleanSrc)
	}

	if _, err := os.Stat(cleanSrc); err != nil {
		return fmt.Errorf("storage: MoveStore source %q: %w", cleanSrc, err)
	}
	if _, err := os.Stat(cleanDst); err == nil {
		return fmt.Errorf("storage: MoveStore refuses to overwrite an existing store at %q", cleanDst)
	} else if !os.IsNotExist(err) {
		return fmt.Errorf("storage: MoveStore destination %q: %w", cleanDst, err)
	}

	// The owner directory ("<root>/~<owner>") already exists for any store that
	// is being renamed within its owner's namespace, but creating it keeps the
	// operation correct if a future caller ever moves a store across owners.
	if err := os.MkdirAll(filepath.Dir(cleanDst), 0o755); err != nil {
		return fmt.Errorf("storage: create destination dir %q: %w", filepath.Dir(cleanDst), err)
	}
	if err := os.Rename(cleanSrc, cleanDst); err != nil {
		return fmt.Errorf("storage: move store %q -> %q: %w", cleanSrc, cleanDst, err)
	}
	return nil
}