From f1426778fcaf9edcf6e15bbf7484579712b0cc20 Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Mon, 25 May 2026 16:37:38 +0300 Subject: [PATCH] Add end-to-end test suite against real Garage via testcontainers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * internal/testutil/garage: spins up dxflrs/garage:v2.3.0 with --single-node --default-bucket so the bucket + access key are auto-created from env vars at startup, no CLI bootstrap dance. Each Start() call yields a fresh container with random creds and registers t.Cleanup teardown. * e2e_test.go gated by build tag e2e exercises the compiled cacher binary against the container — covers init/doctor parity (the regression guard for the HeadBucket+signature bugs we hit on the real bucket), single-file round-trip, exit codes (1/2/3), URL fallback + cache fill, --hash-from parity with sha256sum, directory tar+zstd round-trip, and delimited list output. * just test-e2e recipe; requires Docker on the host. Total runtime ~30s after first image pull (~1.5s/container). --- Justfile | 3 + e2e_test.go | 316 ++++++++++++++++++++++++++ go.mod | 50 ++++ go.sum | 132 +++++++++++ internal/testutil/garage/container.go | 145 ++++++++++++ 5 files changed, 646 insertions(+) create mode 100644 e2e_test.go create mode 100644 internal/testutil/garage/container.go diff --git a/Justfile b/Justfile index 9e40e4d6898b26de66bf471782dd60ad958ce49b..0318765a427ddf1adf9d82b757cb6c351ae477dc 100644 --- a/Justfile +++ b/Justfile @@ -19,6 +19,9 @@ install: test: go test ./... +test-e2e: + go test -tags=e2e -timeout=5m -v ./... + lint: golangci-lint run ./... diff --git a/e2e_test.go b/e2e_test.go new file mode 100644 index 0000000000000000000000000000000000000000..6b4cdb31a964aba0cfb318e276f0643cd158f57b --- /dev/null +++ b/e2e_test.go @@ -0,0 +1,316 @@ +//go:build e2e + +// End-to-end tests that exercise the compiled `cacher` binary against +// a real Garage container. Opt-in via build tag because they require +// Docker and pull a ~150 MB image. +// +// Run with: `just test-e2e` or `go test -tags=e2e -timeout=5m ./...` +package main_test + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "go.bigb.es/cacher/internal/testutil/garage" +) + +// cacherBinary is built once in TestMain so every subtest exec's the +// same artifact. +var cacherBinary string + +func TestMain(m *testing.M) { + tmp, err := os.MkdirTemp("", "cacher-e2e-*") + if err != nil { + fmt.Fprintln(os.Stderr, "mktmp:", err) + os.Exit(1) + } + cacherBinary = filepath.Join(tmp, "cacher") + out, err := exec.Command("go", "build", "-o", cacherBinary, ".").CombinedOutput() + if err != nil { + fmt.Fprintf(os.Stderr, "build cacher:\n%s\n%v\n", out, err) + os.RemoveAll(tmp) + os.Exit(1) + } + code := m.Run() + os.RemoveAll(tmp) + os.Exit(code) +} + +// runResult captures everything a CI script would see. +type runResult struct { + stdout, stderr string + exit int +} + +// runner returns a closure that invokes the cacher binary with the given +// args, pointing at the Garage instance via env vars and a per-test +// CACHER_CONFIG path so parallel tests don't fight. +func runner(t *testing.T, g *garage.Garage) func(args ...string) runResult { + t.Helper() + cfgPath := filepath.Join(t.TempDir(), "config.toml") + env := append(os.Environ(), + "CACHER_S3_KEY_ID="+g.AccessKey, + "CACHER_S3_SECRET="+g.SecretKey, + ) + return func(args ...string) runResult { + t.Helper() + // Use --config so each test gets its own config.toml — keeps + // parallel tests isolated and stops them from clobbering the + // developer's real ~/.config/cacher/config.toml. + full := append([]string{"--config", cfgPath}, args...) + cmd := exec.Command(cacherBinary, full...) + cmd.Env = env + var so, se bytes.Buffer + cmd.Stdout = &so + cmd.Stderr = &se + err := cmd.Run() + exit := 0 + if exitErr, ok := err.(*exec.ExitError); ok { + exit = exitErr.ExitCode() + } else if err != nil { + t.Fatalf("exec cacher: %v\nstderr: %s", err, se.String()) + } + return runResult{stdout: so.String(), stderr: se.String(), exit: exit} + } +} + +// initCacher runs `cacher init` against the Garage container. Asserts the +// doctor smoke test passes — this is the regression test for the two +// Garage quirks (HeadBucket → 403, signature region pickiness) we hit +// against real production. +func initCacher(t *testing.T, run func(...string) runResult, g *garage.Garage) { + t.Helper() + r := run("init", + "--endpoint", g.Endpoint, + "--region", g.Region, + "--bucket", g.Bucket, + "--prefix", "e2e", + "--key-file", writeFile(t, g.AccessKey), + "--secret-file", writeFile(t, g.SecretKey), + ) + if r.exit != 0 { + t.Fatalf("init exit=%d\nstdout: %s\nstderr: %s", r.exit, r.stdout, r.stderr) + } + if !strings.Contains(r.stdout, "doctor: all checks passed") { + t.Fatalf("init/doctor smoke test missing success line:\n%s", r.stdout) + } +} + +func TestFileRoundTrip(t *testing.T) { + g := garage.Start(t) + run := runner(t, g) + initCacher(t, run, g) + + src := writeFile(t, "hello world") + + // Upload, then verify exists, then download, then byte-compare. + if r := run("upload", "k1", src); r.exit != 0 { + t.Fatalf("upload: exit=%d stderr=%s", r.exit, r.stderr) + } + if r := run("exists", "k1"); r.exit != 0 { + t.Fatalf("exists(present) exit=%d, want 0", r.exit) + } + dst := filepath.Join(t.TempDir(), "out") + if r := run("download", "k1", dst); r.exit != 0 { + t.Fatalf("download exit=%d stderr=%s", r.exit, r.stderr) + } + if got := readFile(t, dst); got != "hello world" { + t.Errorf("round-trip body mismatch: %q", got) + } + + // Delete invalidates the cache; exists must now return 1. + if r := run("delete", "k1"); r.exit != 0 { + t.Fatalf("delete exit=%d stderr=%s", r.exit, r.stderr) + } + if r := run("exists", "k1"); r.exit != 1 { + t.Errorf("exists(missing) exit=%d, want 1", r.exit) + } +} + +func TestExitCodes(t *testing.T) { + g := garage.Start(t) + run := runner(t, g) + initCacher(t, run, g) + + // exists on a missing key → 1 + if r := run("exists", "nope"); r.exit != 1 { + t.Errorf("exists(missing) exit=%d, want 1", r.exit) + } + + // download with no --url and a missing key → 3 (ErrMissNoFallback) + dst := filepath.Join(t.TempDir(), "x") + if r := run("download", "nope", dst); r.exit != 3 { + t.Errorf("download(miss, no url) exit=%d, want 3", r.exit) + } +} + +func TestURLFallbackAndCacheFill(t *testing.T) { + g := garage.Start(t) + run := runner(t, g) + initCacher(t, run, g) + + want := "payload from upstream" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + io.WriteString(w, want) + })) + defer srv.Close() + + // First download misses, falls back to URL, AND populates the cache. + dst1 := filepath.Join(t.TempDir(), "first") + if r := run("download", "url-key", dst1, "--url", srv.URL); r.exit != 0 { + t.Fatalf("download(miss → url) exit=%d stderr=%s", r.exit, r.stderr) + } + if got := readFile(t, dst1); got != want { + t.Errorf("first body=%q want %q", got, want) + } + + // Second download should now HIT the cache. We pass a deliberately- + // dead URL so a regression that re-fetches would fail loudly here. + dst2 := filepath.Join(t.TempDir(), "second") + r := run("download", "url-key", dst2, "--url", "http://127.0.0.1:1/dead") + if r.exit != 0 { + t.Fatalf("download(hit) exit=%d stderr=%s", r.exit, r.stderr) + } + if !strings.Contains(r.stderr, "Cache HIT") { + t.Errorf("second call did not log Cache HIT:\n%s", r.stderr) + } + if got := readFile(t, dst2); got != want { + t.Errorf("cached body=%q want %q", got, want) + } +} + +func TestHashFromMatchesShellSha256sumCutC116(t *testing.T) { + g := garage.Start(t) + run := runner(t, g) + initCacher(t, run, g) + + src := writeFile(t, "deterministic content") + r := run("key", "img/{hash}.tar.zst", "--hash-from", src, "--hash-length", "16") + if r.exit != 0 { + t.Fatalf("key exit=%d stderr=%s", r.exit, r.stderr) + } + got := strings.TrimSpace(r.stdout) + + // Independently compute via shell to verify the parity claim. Skip + // if sha256sum isn't on PATH (macOS dev box without coreutils). + if _, err := exec.LookPath("sha256sum"); err != nil { + t.Skipf("sha256sum unavailable: %v", err) + } + out, err := exec.Command("sha256sum", src).Output() + if err != nil { + t.Fatalf("sha256sum: %v", err) + } + hash := strings.Fields(string(out))[0][:16] + want := "img/" + hash + ".tar.zst" + if got != want { + t.Errorf("key = %q\nwant = %q (sha256sum %s | cut -c1-16)", got, want, src) + } +} + +func TestDirRoundTrip(t *testing.T) { + g := garage.Start(t) + run := runner(t, g) + initCacher(t, run, g) + + src := t.TempDir() + writeFileAt(t, filepath.Join(src, "a.txt"), "alpha") + writeFileAt(t, filepath.Join(src, "sub/b.txt"), "bravo") + + if r := run("dir", "upload", "tree-key", src); r.exit != 0 { + t.Fatalf("dir upload exit=%d stderr=%s", r.exit, r.stderr) + } + + dst := t.TempDir() + if r := run("dir", "download", "tree-key", dst); r.exit != 0 { + t.Fatalf("dir download exit=%d stderr=%s", r.exit, r.stderr) + } + if got := readFile(t, filepath.Join(dst, "a.txt")); got != "alpha" { + t.Errorf("a.txt = %q want alpha", got) + } + if got := readFile(t, filepath.Join(dst, "sub/b.txt")); got != "bravo" { + t.Errorf("sub/b.txt = %q want bravo", got) + } +} + +func TestListDelimited(t *testing.T) { + g := garage.Start(t) + run := runner(t, g) + initCacher(t, run, g) + + // Seed: two "directories" plus a file at the top level. + src := writeFile(t, "x") + run("upload", "dirA/file1", src) + run("upload", "dirA/file2", src) + run("upload", "dirB/file3", src) + run("upload", "topfile", src) + + r := run("list") + if r.exit != 0 { + t.Fatalf("list exit=%d stderr=%s", r.exit, r.stderr) + } + lines := splitLines(r.stdout) + expect := map[string]bool{"dirA/": false, "dirB/": false, "topfile": false} + for _, l := range lines { + if _, ok := expect[l]; ok { + expect[l] = true + } + } + for k, seen := range expect { + if !seen { + t.Errorf("list output missing %q\nfull:\n%s", k, r.stdout) + } + } + + r = run("list", "--recursive") + if !strings.Contains(r.stdout, "dirA/file1") || !strings.Contains(r.stdout, "dirA/file2") { + t.Errorf("recursive list missing nested keys:\n%s", r.stdout) + } +} + +// --- helpers --- + +func writeFile(t *testing.T, body string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "f") + if err := os.WriteFile(p, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return p +} + +func writeFileAt(t *testing.T, path, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } +} + +func readFile(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func splitLines(s string) []string { + var out []string + for _, l := range strings.Split(strings.TrimRight(s, "\n"), "\n") { + if l != "" { + out = append(out, l) + } + } + return out +} diff --git a/go.mod b/go.mod index c24e2f59ddbf1945bbfe19696f18edce6baae91f..336a3ca5040bbed3c2f80c024840020dba204f72 100644 --- a/go.mod +++ b/go.mod @@ -11,9 +11,13 @@ require ( github.com/aws/smithy-go v1.25.1 github.com/klauspost/compress v1.18.6 github.com/spf13/cobra v1.10.2 + github.com/testcontainers/testcontainers-go v0.42.0 ) require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect @@ -22,6 +26,52 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/ebitengine/purego v0.10.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.2.0 // indirect + github.com/moby/moby/api v1.54.1 // indirect + github.com/moby/moby/client v0.4.0 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.2 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/shirou/gopsutil/v4 v4.26.3 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect github.com/spf13/pflag v1.0.9 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/tklauser/go-sysconf v0.3.16 // indirect + github.com/tklauser/numcpus v0.11.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/sys v0.42.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index f3ef29473742894c62e5729ad86ef3d1e093fbca..986953a9afdb908638d1296915b0cb1a49c59da9 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,13 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= @@ -38,15 +46,139 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOIt github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio= github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= +github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= +github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= +github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4= +github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw= +github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= +github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= +github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= +github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= +github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= +github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= +github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= +golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/internal/testutil/garage/container.go b/internal/testutil/garage/container.go new file mode 100644 index 0000000000000000000000000000000000000000..03032b2fdbc421740f7761b4a87d7d6379fad26f --- /dev/null +++ b/internal/testutil/garage/container.go @@ -0,0 +1,145 @@ +// Package garage starts a single-node Garage container for end-to-end +// tests. Garage v2.3+ exposes `server --single-node --default-bucket` +// which auto-creates the bucket + access key from env vars, so the +// usual six-step CLI bootstrap (status / layout assign / layout apply / +// bucket create / key new / bucket allow) becomes a no-op. +// +// Requires Docker on the host. Each call to Start spins up a fresh +// container with random credentials and binds host ports automatically; +// t.Cleanup tears it down. +package garage + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/base64" + "encoding/hex" + "fmt" + "testing" + "time" + + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +// Image we pull. Pin to a known-good version so test runs are reproducible. +const Image = "dxflrs/garage:v2.3.0" + +// Garage holds the connection details for the running container. +type Garage struct { + Endpoint string // http://host:port — feed to cacher --endpoint / config endpoint + AccessKey string // GK… — feed to CACHER_S3_KEY_ID + SecretKey string // hex — feed to CACHER_S3_SECRET + Bucket string // pre-created bucket the access key owns + Region string // always "garage" for this image +} + +// Start brings up a Garage container and waits for the S3 API to be +// responding. Container is automatically terminated via t.Cleanup. +func Start(t *testing.T) *Garage { + t.Helper() + ctx := context.Background() + + g := &Garage{ + AccessKey: "GK" + randHex(16), + SecretKey: randHex(32), + Bucket: "test-bucket", + Region: "garage", + } + + cfg := minimalTOML(randHex(32), randBase64(32)) + + req := testcontainers.ContainerRequest{ + Image: Image, + ExposedPorts: []string{"3900/tcp"}, + Cmd: []string{"/garage", "server", "--single-node", "--default-bucket"}, + Env: map[string]string{ + "GARAGE_DEFAULT_ACCESS_KEY": g.AccessKey, + "GARAGE_DEFAULT_SECRET_KEY": g.SecretKey, + "GARAGE_DEFAULT_BUCKET": g.Bucket, + }, + Files: []testcontainers.ContainerFile{ + { + Reader: bytes.NewReader([]byte(cfg)), + ContainerFilePath: "/etc/garage.toml", + FileMode: 0o644, + }, + }, + // Combine port-open + log-line wait so we don't race the S3 server + // starting up after the RPC port is bound. + WaitingFor: wait.ForAll( + wait.ForListeningPort("3900/tcp").WithStartupTimeout(30*time.Second), + wait.ForLog("S3 API server listening").WithStartupTimeout(30*time.Second), + ), + } + + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + if err != nil { + t.Fatalf("garage container start: %v", err) + } + t.Cleanup(func() { + if err := c.Terminate(context.Background()); err != nil { + t.Logf("garage container terminate: %v", err) + } + }) + + host, err := c.Host(ctx) + if err != nil { + t.Fatalf("container host: %v", err) + } + port, err := c.MappedPort(ctx, "3900/tcp") + if err != nil { + t.Fatalf("container port: %v", err) + } + g.Endpoint = fmt.Sprintf("http://%s:%s", host, port.Port()) + return g +} + +// minimalTOML returns the smallest garage.toml that works for a single- +// node test instance: in-memory metadata under /tmp (lost on container +// stop, which is what we want), replication_factor=1, S3 region "garage", +// random secrets for rpc + admin so the container can't conflict with +// neighbours. +func minimalTOML(rpcSecret, adminToken string) string { + return fmt.Sprintf(` +metadata_dir = "/tmp/meta" +data_dir = "/tmp/data" +db_engine = "sqlite" + +replication_factor = 1 + +rpc_bind_addr = "[::]:3901" +rpc_public_addr = "127.0.0.1:3901" +rpc_secret = %q + +[s3_api] +s3_region = "garage" +api_bind_addr = "[::]:3900" +root_domain = ".s3.garage.localhost" + +[s3_web] +bind_addr = "[::]:3902" +root_domain = ".web.garage.localhost" +index = "index.html" + +[admin] +api_bind_addr = "[::]:3903" +admin_token = %q +`, rpcSecret, adminToken) +} + +func randHex(n int) string { + b := make([]byte, n) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + +func randBase64(n int) string { + b := make([]byte, n) + _, _ = rand.Read(b) + return base64.StdEncoding.EncodeToString(b) +}