// Package storage owns dolt.sr.ht's on-disk NBS chunk stores and the // remotesapi DBCache that serves them. // // A hosted database is a bare NBS chunk-store directory (no ".dolt/", no // working set) laid out at "/~/". This is exactly what // remotesrv serves and what "file://" dolt remotes consume, so InitStore can // create one with the low-level doltdb primitives and remotesrv can read and // write it directly. // // # remotesrv FS working directory (load-bearing) // // remotesrv seals its chunk-download URLs relative to the working directory of // the filesys it is given. The server MUST be constructed with // filesys.LocalFilesysWithWorkingDir(root) pointing at the repos root — NOT a // plain filesys.LocalFS. With a working-dir-rooted FS the sealed URLs carry // clean relative prefixes (e.g. "~owner/db"); with a bare LocalFS they carry // "../../.." escapes that the sealed-URL file handler rejects, and every clone // or push breaks at the chunk-transfer stage. This was proven end-to-end in the // Phase-0 spike (storage/spike_test.go). Cache in this package keys stores by // absolute disk path, so it works with either FS, but the server assembly in // remoteapi/ must still honor this rule. package storage import ( "context" "fmt" "os" "path/filepath" "strings" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/utils/earl" "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/dolt/go/store/nbs" "github.com/dolthub/dolt/go/store/types" ) // RepoDiskPath returns the absolute on-disk store directory for a database, // laid out as "/~/". Callers are responsible for validating // owner and name (see core.ValidateName) before touching disk. func RepoDiskPath(root, owner, name string) string { return filepath.Join(root, "~"+owner, name) } // InitStore creates a bare NBS chunk store at absPath and writes an empty repo // into it with a single "main" branch and an initial commit authored by // ownerName/ownerEmail. // // It is the OPT-IN half of database creation — the create form's "initialize // with an empty commit" checkbox — and not what any automatic path uses. The // initial commit it writes is history, so a client pushing a database that has // its own history is pushing a non-fast-forward and must --force. What it buys // in exchange is a database that can be cloned before anything is pushed to it, // which InitEmptyStore's result cannot. // // absPath must be absolute. On any failure after the directory is created, // InitStore removes absPath so a failed creation never leaves a partial store // behind. Idempotence is NOT provided: calling InitStore on an existing store // is a caller error and is not defended against here. func InitStore(ctx context.Context, absPath, ownerName, ownerEmail string) (err error) { if !filepath.IsAbs(absPath) { return fmt.Errorf("storage: InitStore requires an absolute path, got %q", absPath) } if err := os.MkdirAll(absPath, 0o755); err != nil { return fmt.Errorf("storage: create store dir %q: %w", absPath, err) } // Any error past this point must not leave a half-written store behind. defer func() { if err != nil { os.RemoveAll(absPath) } }() fileURL := earl.FileUrlFromPath(absPath, os.PathSeparator) ddb, err := doltdb.LoadDoltDB(ctx, types.Format_DOLT, fileURL, filesys.LocalFS) if err != nil { return fmt.Errorf("storage: load doltdb at %q: %w", fileURL, err) } if err = ddb.WriteEmptyRepo(ctx, "main", ownerName, ownerEmail); err != nil { ddb.Close() return fmt.Errorf("storage: write empty repo at %q: %w", absPath, err) } // Release the init handle so the server can open its own store over the // same directory later. if err = ddb.Close(); err != nil { return fmt.Errorf("storage: close init doltdb at %q: %w", absPath, err) } return nil } // InitEmptyStore creates a genuinely empty bare NBS chunk store at absPath — // a directory whose store root is the empty hash, with NO commits and NO // working set. Unlike InitStore it deliberately does NOT call WriteEmptyRepo: // an initial "Initialize data repository" commit would make the first push to // this store a non-fast-forward and be rejected. An empty store lets the // client's first push land as the initial history. Used by push-to-create. // absPath must be absolute; on any failure the directory is removed. func InitEmptyStore(ctx context.Context, absPath string) (err error) { if !filepath.IsAbs(absPath) { return fmt.Errorf("storage: InitEmptyStore requires an absolute path, got %q", absPath) } if err := os.MkdirAll(absPath, 0o755); err != nil { return fmt.Errorf("storage: create store dir %q: %w", absPath, err) } // Any error past this point must not leave a half-written store behind. defer func() { if err != nil { os.RemoveAll(absPath) } }() // Opening a plain (non-generational) NBS store over the freshly created, // empty directory both validates the directory (checkDir requires it to // exist) and confirms the store is a valid empty store (a missing manifest // is treated lazily as an empty store with the null root). This mirrors the // construction storage.Cache.Get uses to serve pushes; a plain store closes // cleanly, unlike the generational store LoadDoltDB routes through. cs, err := nbs.NewLocalStore(ctx, types.Format_DOLT.VersionString(), absPath, defaultMemTableSize, nbs.NewUnlimitedMemQuotaProvider(), false) if err != nil { return fmt.Errorf("storage: open empty store at %q: %w", absPath, err) } if err = cs.Close(); err != nil { return fmt.Errorf("storage: close empty store at %q: %w", absPath, err) } return nil } // containedPath checks that absPath is an absolute path strictly inside the // absolute root and returns both cleaned. It is the guard every destructive // path operation in this package runs first, so a corrupted or // attacker-controlled path can never escape the configured repos root — and // the root itself is never a legal target. op names the caller ("DeleteStore", // "MoveStore") so a refusal says which operation was stopped. func containedPath(op, root, absPath string) (cleanRoot, cleanPath string, err error) { if !filepath.IsAbs(root) { return "", "", fmt.Errorf("storage: %s requires an absolute root, got %q", op, root) } if !filepath.IsAbs(absPath) { return "", "", fmt.Errorf("storage: %s requires an absolute path, got %q", op, absPath) } cleanRoot = filepath.Clean(root) cleanPath = filepath.Clean(absPath) if cleanPath == cleanRoot { return "", "", fmt.Errorf("storage: %s refuses to act on the repos root %q", op, cleanRoot) } rel, err := filepath.Rel(cleanRoot, cleanPath) if err != nil { return "", "", fmt.Errorf("storage: %s rel(%q, %q): %w", op, cleanRoot, cleanPath, err) } if rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { return "", "", fmt.Errorf("storage: %s refuses %q outside repos root %q", op, cleanPath, cleanRoot) } return cleanRoot, cleanPath, nil } // DeleteStore removes the store directory at absPath. It refuses to delete // anything that is not strictly contained within root, guarding against a // corrupted or attacker-controlled path escaping the configured repos root. // Both root and absPath must be absolute. func DeleteStore(ctx context.Context, root, absPath string) error { _, cleanPath, err := containedPath("DeleteStore", root, absPath) if err != nil { return err } if err := os.RemoveAll(cleanPath); err != nil { return fmt.Errorf("storage: remove store %q: %w", cleanPath, err) } return nil } // MoveStore relocates the store directory at srcPath to dstPath, the on-disk // half of a rename. Both paths must be absolute and strictly inside root. // // It never overwrites: an existing dstPath is refused before anything is // touched, because os.Rename over an empty destination directory would succeed // silently and swallow it. A missing srcPath is likewise an error rather than a // no-op — a rename whose store never moved would leave the metadata row // pointing at nothing. // // The move itself is one os.Rename, so within a filesystem it is atomic: the // store is either wholly at the old path or wholly at the new one, never half // copied. Open handles on the old directory survive it (the inodes move, not // the files), but they keep resolving new writes against the old path string, // so the caller must still evict any memoized handle — see Cache.Evict. func MoveStore(ctx context.Context, root, srcPath, dstPath string) error { _, cleanSrc, err := containedPath("MoveStore", root, srcPath) if err != nil { return err } _, cleanDst, err := containedPath("MoveStore", root, dstPath) if err != nil { return err } if cleanSrc == cleanDst { return fmt.Errorf("storage: MoveStore source and destination are the same path %q", cleanSrc) } if _, err := os.Stat(cleanSrc); err != nil { return fmt.Errorf("storage: MoveStore source %q: %w", cleanSrc, err) } if _, err := os.Stat(cleanDst); err == nil { return fmt.Errorf("storage: MoveStore refuses to overwrite an existing store at %q", cleanDst) } else if !os.IsNotExist(err) { return fmt.Errorf("storage: MoveStore destination %q: %w", cleanDst, err) } // The owner directory ("/~") already exists for any store that // is being renamed within its owner's namespace, but creating it keeps the // operation correct if a future caller ever moves a store across owners. if err := os.MkdirAll(filepath.Dir(cleanDst), 0o755); err != nil { return fmt.Errorf("storage: create destination dir %q: %w", filepath.Dir(cleanDst), err) } if err := os.Rename(cleanSrc, cleanDst); err != nil { return fmt.Errorf("storage: move store %q -> %q: %w", cleanSrc, cleanDst, err) } return nil }