From 4562b448f2d5b5e1ad9570f8c2b6b6ee2db80213 Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Wed, 22 Jul 2026 13:45:57 +0300 Subject: [PATCH] =?UTF-8?q?feat:=20specsrht-migrate=20=E2=80=94=20brant=20?= =?UTF-8?q?wrapper=20for=20the=20spec.sr.ht=20schema?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Single-service wrapper around git.sr.ht/~bitfehler/brant, a close sibling of doltsrht-migrate: the brant subcommands (up, down, current, list, stamp, validate, ping) plus an extra `init` that applies schema.sql wholesale and stamps to head for a fresh install. The DSN comes from [spec.sr.ht]connection-string unless --dsn overrides it; a missing or empty value is a fatal error rather than a default connection. Migrations load from ./migrations in a checkout, else from <[sr.ht]assets>/migrations/spec.sr.ht. -a honours [spec.sr.ht]migrate-on-upgrade and exits early when it is off. lib/pq's "postgres" driver replaces brant's pgx default, which this module does not link. Tests cover flag parsing, DSN precedence and the migrations-directory resolution order without a database; the init/up round trips against a scratch schema skip unless SPECSRHT_TEST_PG is set. --- cmd/specsrht-migrate/main.go | 202 ++++++++++++++ cmd/specsrht-migrate/main_test.go | 437 ++++++++++++++++++++++++++++++ 2 files changed, 639 insertions(+) create mode 100644 cmd/specsrht-migrate/main.go create mode 100644 cmd/specsrht-migrate/main_test.go diff --git a/cmd/specsrht-migrate/main.go b/cmd/specsrht-migrate/main.go new file mode 100644 index 0000000000000000000000000000000000000000..bad16afc7b0914d6f39903417ffbb5a01520c6c6 --- /dev/null +++ b/cmd/specsrht-migrate/main.go @@ -0,0 +1,202 @@ +// Command specsrht-migrate runs the spec.sr.ht Postgres migrations. It is a +// thin single-service wrapper around git.sr.ht/~bitfehler/brant, a sibling of +// doltsrht-migrate: the brant subcommands (up, down, current, list, stamp, +// validate, ping) apply the files under the migrations directory, and an extra +// `init` subcommand loads the full schema.sql in one shot and stamps the +// database to head (used for a fresh install instead of replaying every +// migration). +// +// The connection string comes from [spec.sr.ht]connection-string unless +// overridden with --dsn. Migrations are read from ./migrations in a dev checkout +// or from the installed assets path (/usr/share/sourcehut/migrations/spec.sr.ht) +// otherwise. The lib/pq "postgres" driver this module already links is used in +// preference to brant's default pgx driver. +// +// Typical use: +// +// createdb spec.sr.ht +// specsrht-migrate init # apply schema.sql wholesale, stamp to head +// specsrht-migrate up # apply pending migrations/*.sql (upgrades) +// specsrht-migrate current # print the current schema version +// specsrht-migrate -a up # honor migrate-on-upgrade; no-op when disabled +package main + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "path/filepath" + + "git.sr.ht/~bitfehler/brant" + "git.sr.ht/~bitfehler/brant/cli" + "github.com/alexflint/go-arg" + _ "github.com/lib/pq" // registers the "postgres" database/sql driver + "github.com/vaughan0/go-ini" + + "sourcecraft.dev/bigbes/sr-ht-core/config" +) + +const ( + // progName is the binary name used in usage and log lines. + progName = "specsrht-migrate" + + // serviceName is the SourceHut service identifier and config section name. + serviceName = "spec.sr.ht" + + // driverName is the database/sql driver this binary links (lib/pq). It + // overrides brant's postgres-dialect default of "pgx", which this module + // does not import. + driverName = "postgres" + + // defaultDirectory is brant's own default for --dir. Seeing it unchanged is + // how we know the user did not override the migrations directory. + defaultDirectory = "./migrations" + + // defaultSchema is the default for `init --schema`, resolved against the + // working tree in a checkout and against the assets dir otherwise. + defaultSchema = "schema.sql" + + // defaultAssets is the fallback for [sr.ht]assets. + defaultAssets = "/usr/share/sourcehut" +) + +// InitArgs configures the `init` subcommand: load the full DDL and stamp to head. +type InitArgs struct { + Schema string `arg:"--schema" default:"schema.sql" placeholder:"FILE" help:"schema file to initialize the database with"` +} + +// Args embeds brant's CLI arguments (the up/down/... subcommands and shared +// flags such as --dir and --dsn) and adds the init subcommand and the -a +// migrate-on-upgrade gate. +type Args struct { + cli.Args + Init *InitArgs `arg:"subcommand:init" help:"initialize the database from the schema file and stamp to head"` + Auto bool `arg:"-a" help:"honor [spec.sr.ht]migrate-on-upgrade; exit early when it is disabled"` +} + +func (Args) Epilogue() string { + return "Use ` --help` for help with individual commands" +} + +// newParser builds the argument parser. Tests reuse it with IgnoreEnv set so +// that a developer's BRANT_* environment cannot skew the expected defaults. +func newParser(conf arg.Config) (*arg.Parser, *Args) { + conf.Program = progName + a := &Args{} + p, err := arg.NewParser(conf, a) + if err != nil { + panic(err) // only happens when the Args struct itself is malformed + } + return p, a +} + +func main() { + p, a := newParser(arg.Config{}) + p.MustParse(os.Args[1:]) + if p.Subcommand() == nil { + p.WriteHelp(os.Stderr) + os.Exit(1) + } + + conf := config.LoadConfig() + + if a.Auto && !config.GetBool(conf, serviceName, "migrate-on-upgrade", false) { + log.Printf("%s: [%s]migrate-on-upgrade disabled, exiting", progName, serviceName) + return + } + + dsn, err := resolveDSN(conf, a.DataSourceName) + if err != nil { + log.Fatalf("%s: %v", progName, err) + } + a.DataSourceName = dsn + + // Use lib/pq's "postgres" driver rather than brant's default "pgx". + drv := driverName + a.Driver = &drv + + if err := resolvePaths(conf, a); err != nil { + log.Fatalf("%s: %v", progName, err) + } + log.Printf("%s: loading migrations from %s", progName, a.Directory) + + if a.Init != nil { + log.Printf("%s: initializing schema from %s", progName, a.Init.Schema) + if err := initDatabase(a); err != nil { + log.Fatalf("%s: init failed: %v", progName, err) + } + return + } + + cli.RunWithArgs(&a.Args) +} + +// resolveDSN picks the connection string: an explicit --dsn (or BRANT_DSN) wins, +// otherwise [spec.sr.ht]connection-string. A missing or empty configured value +// is an error rather than a silent connection attempt against a default DSN. +func resolveDSN(conf ini.File, override string) (string, error) { + if override != "" { + return override, nil + } + dsn, ok := conf.Get(serviceName, "connection-string") + if !ok || dsn == "" { + return "", fmt.Errorf("no [%s]connection-string configured", serviceName) + } + return dsn, nil +} + +// resolvePaths picks the migrations directory and schema file when the user did +// not override --dir: a ./migrations directory in the working tree (dev +// checkout) wins, otherwise the installed assets path is used, mirroring +// sourcehut-migrate. +func resolvePaths(conf ini.File, a *Args) error { + if a.Directory != defaultDirectory { + return nil // user overrode --dir; respect it verbatim + } + + info, err := os.Stat("migrations") + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("checking ./migrations: %w", err) + } + if err == nil && info.IsDir() { + log.Printf("%s: found ./migrations, using it", progName) + return nil + } + + assetsDir := config.GetString(conf, "sr.ht", "assets", defaultAssets) + a.Directory = filepath.Join(assetsDir, "migrations", serviceName) + if a.Init != nil && a.Init.Schema == defaultSchema { + a.Init.Schema = filepath.Join(assetsDir, serviceName+".sql") + } + return nil +} + +// initDatabase applies the schema file wholesale and stamps the version table to +// head, so a fresh install skips replaying the incremental migrations. +func initDatabase(a *Args) error { + p, err := cli.ProviderFromArgs(&a.Args) + if err != nil { + return fmt.Errorf("creating provider: %w", err) + } + defer p.Close() + + statements, err := os.ReadFile(a.Init.Schema) + if err != nil { + return fmt.Errorf("reading schema %s: %w", a.Init.Schema, err) + } + + db, err := p.DB() + if err != nil { + return fmt.Errorf("connecting to database: %w", err) + } + if _, err := db.Exec(string(statements)); err != nil { + return fmt.Errorf("executing %s: %w", a.Init.Schema, err) + } + + if err := p.Stamp(context.Background(), brant.VERSION_HEAD, false); err != nil { + return fmt.Errorf("stamping database: %w", err) + } + return nil +} diff --git a/cmd/specsrht-migrate/main_test.go b/cmd/specsrht-migrate/main_test.go new file mode 100644 index 0000000000000000000000000000000000000000..6ad0083658dd189e06145875c44d1e0b0cb94a69 --- /dev/null +++ b/cmd/specsrht-migrate/main_test.go @@ -0,0 +1,437 @@ +package main + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/hex" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "git.sr.ht/~bitfehler/brant/cli" + "github.com/alexflint/go-arg" + "github.com/vaughan0/go-ini" +) + +// testParse parses argv the way main() does, except that the environment is +// ignored: BRANT_DSN / BRANT_MIGRATION_DIR / BRANT_DIALECT in a developer's +// shell would otherwise silently change the defaults under test. +func testParse(t *testing.T, argv ...string) (*arg.Parser, *Args) { + t.Helper() + p, a := newParser(arg.Config{IgnoreEnv: true, Out: os.Stderr, Exit: func(int) {}}) + if err := p.Parse(argv); err != nil { + t.Fatalf("parse %v: %v", argv, err) + } + return p, a +} + +func TestParseSubcommands(t *testing.T) { + t.Run("up", func(t *testing.T) { + p, a := testParse(t, "up") + if p.Subcommand() == nil { + t.Fatal("up: no subcommand selected") + } + if a.Up == nil { + t.Fatal("up: Up subcommand not set") + } + if a.Init != nil { + t.Error("up: Init unexpectedly set") + } + if a.Directory != defaultDirectory { + t.Errorf("up: Directory = %q, want %q", a.Directory, defaultDirectory) + } + if a.DataSourceName != "" { + t.Errorf("up: DataSourceName = %q, want empty", a.DataSourceName) + } + if a.Auto { + t.Error("up: Auto set without -a") + } + }) + + t.Run("current", func(t *testing.T) { + _, a := testParse(t, "current") + if a.Current == nil { + t.Fatal("current: Current subcommand not set") + } + }) + + t.Run("init", func(t *testing.T) { + p, a := testParse(t, "init") + if p.Subcommand() == nil { + t.Fatal("init: no subcommand selected") + } + if a.Init == nil { + t.Fatal("init: Init subcommand not set") + } + if a.Init.Schema != defaultSchema { + t.Errorf("init: Schema = %q, want %q", a.Init.Schema, defaultSchema) + } + }) + + t.Run("init --schema", func(t *testing.T) { + _, a := testParse(t, "init", "--schema", "/opt/spec.sr.ht.sql") + if a.Init == nil { + t.Fatal("init --schema: Init subcommand not set") + } + if a.Init.Schema != "/opt/spec.sr.ht.sql" { + t.Errorf("init --schema: Schema = %q", a.Init.Schema) + } + }) + + t.Run("-a up", func(t *testing.T) { + _, a := testParse(t, "-a", "up") + if !a.Auto { + t.Error("-a up: Auto not set") + } + if a.Up == nil { + t.Error("-a up: Up subcommand not set") + } + }) + + t.Run("--dsn and --dir", func(t *testing.T) { + _, a := testParse(t, "--dsn", "postgres://u@h/spec.sr.ht", "--dir", "/srv/m", "up") + if a.DataSourceName != "postgres://u@h/spec.sr.ht" { + t.Errorf("DataSourceName = %q", a.DataSourceName) + } + if a.Directory != "/srv/m" { + t.Errorf("Directory = %q", a.Directory) + } + }) + + // The brant subcommands we do not advertise in the README must still parse: + // this binary is a wrapper, not a reduced surface. + for _, name := range []string{"down", "list", "validate", "ping"} { + t.Run(name, func(t *testing.T) { + p, _ := testParse(t, name) + if p.Subcommand() == nil { + t.Fatalf("%s: no subcommand selected", name) + } + }) + } +} + +func TestParseNoSubcommandIsUsageError(t *testing.T) { + p, _ := testParse(t) + if p.Subcommand() != nil { + t.Fatal("bare invocation selected a subcommand") + } +} + +func TestResolveDSN(t *testing.T) { + confWith := func(dsn string) ini.File { + f := ini.File{serviceName: ini.Section{}} + if dsn != "" { + f[serviceName]["connection-string"] = dsn + } + return f + } + + t.Run("flag wins over config", func(t *testing.T) { + got, err := resolveDSN(confWith("postgres://from-config/x"), "postgres://from-flag/x") + if err != nil { + t.Fatalf("resolveDSN: %v", err) + } + if got != "postgres://from-flag/x" { + t.Errorf("got %q, want the --dsn value", got) + } + }) + + t.Run("config used when flag empty", func(t *testing.T) { + got, err := resolveDSN(confWith("postgres://from-config/x"), "") + if err != nil { + t.Fatalf("resolveDSN: %v", err) + } + if got != "postgres://from-config/x" { + t.Errorf("got %q, want the configured value", got) + } + }) + + t.Run("missing key is an error", func(t *testing.T) { + if _, err := resolveDSN(ini.File{}, ""); err == nil { + t.Fatal("expected an error with no config and no --dsn") + } + }) + + t.Run("empty configured value is an error", func(t *testing.T) { + if _, err := resolveDSN(confWith(""), ""); err == nil { + t.Fatal("expected an error for an empty connection-string") + } + }) +} + +func TestResolvePaths(t *testing.T) { + t.Run("checkout ./migrations wins", func(t *testing.T) { + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, "migrations"), 0o755); err != nil { + t.Fatal(err) + } + t.Chdir(dir) + + a := &Args{Init: &InitArgs{Schema: defaultSchema}} + a.Directory = defaultDirectory + if err := resolvePaths(ini.File{}, a); err != nil { + t.Fatalf("resolvePaths: %v", err) + } + if a.Directory != defaultDirectory { + t.Errorf("Directory = %q, want the checkout dir %q", a.Directory, defaultDirectory) + } + if a.Init.Schema != defaultSchema { + t.Errorf("Schema = %q, want the checkout file %q", a.Init.Schema, defaultSchema) + } + }) + + t.Run("installed assets path when no ./migrations", func(t *testing.T) { + t.Chdir(t.TempDir()) + + a := &Args{Init: &InitArgs{Schema: defaultSchema}} + a.Directory = defaultDirectory + if err := resolvePaths(ini.File{}, a); err != nil { + t.Fatalf("resolvePaths: %v", err) + } + wantDir := filepath.Join(defaultAssets, "migrations", serviceName) + if a.Directory != wantDir { + t.Errorf("Directory = %q, want %q", a.Directory, wantDir) + } + wantSchema := filepath.Join(defaultAssets, serviceName+".sql") + if a.Init.Schema != wantSchema { + t.Errorf("Schema = %q, want %q", a.Init.Schema, wantSchema) + } + }) + + t.Run("[sr.ht]assets overrides the installed path", func(t *testing.T) { + t.Chdir(t.TempDir()) + + conf := ini.File{"sr.ht": ini.Section{"assets": "/opt/sourcehut"}} + a := &Args{Init: &InitArgs{Schema: defaultSchema}} + a.Directory = defaultDirectory + if err := resolvePaths(conf, a); err != nil { + t.Fatalf("resolvePaths: %v", err) + } + if want := "/opt/sourcehut/migrations/" + serviceName; a.Directory != want { + t.Errorf("Directory = %q, want %q", a.Directory, want) + } + if want := "/opt/sourcehut/" + serviceName + ".sql"; a.Init.Schema != want { + t.Errorf("Schema = %q, want %q", a.Init.Schema, want) + } + }) + + t.Run("--dir override is respected verbatim", func(t *testing.T) { + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, "migrations"), 0o755); err != nil { + t.Fatal(err) + } + t.Chdir(dir) + + a := &Args{Init: &InitArgs{Schema: defaultSchema}} + a.Directory = "/srv/other-migrations" + if err := resolvePaths(ini.File{}, a); err != nil { + t.Fatalf("resolvePaths: %v", err) + } + if a.Directory != "/srv/other-migrations" { + t.Errorf("Directory = %q, want the --dir value", a.Directory) + } + if a.Init.Schema != defaultSchema { + t.Errorf("Schema = %q, want it untouched when --dir is given", a.Init.Schema) + } + }) + + t.Run("--schema override survives the assets fallback", func(t *testing.T) { + t.Chdir(t.TempDir()) + + a := &Args{Init: &InitArgs{Schema: "/tmp/custom.sql"}} + a.Directory = defaultDirectory + if err := resolvePaths(ini.File{}, a); err != nil { + t.Fatalf("resolvePaths: %v", err) + } + if a.Init.Schema != "/tmp/custom.sql" { + t.Errorf("Schema = %q, want the --schema value", a.Init.Schema) + } + }) + + t.Run("a ./migrations file is not a directory", func(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "migrations"), []byte("not a dir"), 0o644); err != nil { + t.Fatal(err) + } + t.Chdir(dir) + + a := &Args{} + a.Directory = defaultDirectory + if err := resolvePaths(ini.File{}, a); err != nil { + t.Fatalf("resolvePaths: %v", err) + } + if want := filepath.Join(defaultAssets, "migrations", serviceName); a.Directory != want { + t.Errorf("Directory = %q, want the assets path %q", a.Directory, want) + } + }) +} + +// testEnv names the DSN env var that gates the Postgres-backed tests, matching +// the convention used by the db package. When unset those tests skip; the flag, +// DSN and path-resolution tests above run on a machine with no database at all. +const testEnv = "SPECSRHT_TEST_PG" + +// scratchDSN creates an isolated schema on the Postgres pointed at by +// SPECSRHT_TEST_PG and returns a DSN pinned to it, plus a cleanup that drops it. +// A per-test schema avoids needing CREATE DATABASE privileges. +func scratchDSN(t *testing.T) (string, func()) { + t.Helper() + base := os.Getenv(testEnv) + if base == "" { + t.Skipf("%s not set; skipping Postgres-backed test (set it to a DSN to run)", testEnv) + } + + admin, err := sql.Open("postgres", base) + if err != nil { + t.Fatalf("open admin pool: %v", err) + } + if err := admin.Ping(); err != nil { + admin.Close() + t.Fatalf("ping %s: %v", testEnv, err) + } + + buf := make([]byte, 8) + if _, err := rand.Read(buf); err != nil { + admin.Close() + t.Fatalf("rand: %v", err) + } + schema := "specsrht_migrate_test_" + hex.EncodeToString(buf) + if _, err := admin.Exec(`CREATE SCHEMA "` + schema + `"`); err != nil { + admin.Close() + t.Fatalf("create schema %s: %v", schema, err) + } + + scoped, err := withSearchPath(base, schema) + if err != nil { + admin.Exec(`DROP SCHEMA "` + schema + `" CASCADE`) + admin.Close() + t.Fatalf("build scoped dsn: %v", err) + } + + return scoped, func() { + if _, err := admin.Exec(`DROP SCHEMA "` + schema + `" CASCADE`); err != nil { + t.Errorf("drop schema %s: %v", schema, err) + } + admin.Close() + } +} + +// withSearchPath returns base with a connection option that pins search_path to +// schema, handling both the URL and keyword DSN forms. +func withSearchPath(base, schema string) (string, error) { + opt := "-c search_path=" + schema + if strings.Contains(base, "://") { + u, err := url.Parse(base) + if err != nil { + return "", err + } + q := u.Query() + q.Set("options", opt) + u.RawQuery = q.Encode() + return u.String(), nil + } + return base + " options='" + opt + "'", nil +} + +// repoFile resolves a path relative to the repository root (this package lives +// two levels below it). +func repoFile(name string) string { + return filepath.Join("..", "..", name) +} + +func newArgs(dsn, dir string) *Args { + drv := driverName + return &Args{Args: cli.Args{ + Directory: dir, + DataSourceName: dsn, + Driver: &drv, + }} +} + +// TestInitStampsToHead exercises the `init` path against a real database: the +// full schema.sql applies and the version table lands on the head migration, so +// a subsequent `up` has nothing to do. +func TestInitStampsToHead(t *testing.T) { + dsn, cleanup := scratchDSN(t) + defer cleanup() + + a := newArgs(dsn, repoFile("migrations")) + a.Init = &InitArgs{Schema: repoFile("schema.sql")} + if err := initDatabase(a); err != nil { + t.Fatalf("initDatabase: %v", err) + } + + p, err := cli.ProviderFromArgs(&a.Args) + if err != nil { + t.Fatalf("provider: %v", err) + } + defer p.Close() + + ctx := context.Background() + cur, err := p.Current(ctx) + if err != nil { + t.Fatalf("current: %v", err) + } + sources := p.ListSources() + if len(sources) == 0 { + t.Fatal("no migration sources found") + } + if head := sources[len(sources)-1].Version; cur != head { + t.Errorf("current = %d after init, want head %d", cur, head) + } + + applied, err := p.Up(ctx) + if err != nil { + t.Fatalf("up after init: %v", err) + } + if len(applied) != 0 { + t.Errorf("up after init applied %d migrations, want 0", len(applied)) + } +} + +// TestUpFromEmpty exercises the upgrade path: replaying migrations/*.sql on an +// empty schema reaches the same head version init stamps to. +func TestUpFromEmpty(t *testing.T) { + dsn, cleanup := scratchDSN(t) + defer cleanup() + + a := newArgs(dsn, repoFile("migrations")) + p, err := cli.ProviderFromArgs(&a.Args) + if err != nil { + t.Fatalf("provider: %v", err) + } + defer p.Close() + + ctx := context.Background() + applied, err := p.Up(ctx) + if err != nil { + t.Fatalf("up: %v", err) + } + if len(applied) == 0 { + t.Fatal("up applied no migrations on an empty schema") + } + + cur, err := p.Current(ctx) + if err != nil { + t.Fatalf("current: %v", err) + } + sources := p.ListSources() + if head := sources[len(sources)-1].Version; cur != head { + t.Errorf("current = %d after up, want head %d", cur, head) + } + + // The migrations must produce the tables the design's schema declares. + db, err := p.DB() + if err != nil { + t.Fatalf("db: %v", err) + } + for _, table := range []string{"space", "document_id", "proposal", "agent_token", "index_stamp", "digest_mark"} { + var n int + if err := db.QueryRow(`SELECT count(*) FROM ` + table).Scan(&n); err != nil { + t.Errorf("table %s missing after up: %v", table, err) + } + } +}