A => .gitignore +18 -0
@@ 1,18 @@
+# Built binaries
+/cacher
+/cacher-linux-amd64
+
+# Local config / credentials that might end up in cwd
+*.s3-cache-key-id
+*.s3-cache-key-secret
+
+# IDE / editor
+.idea/
+.vscode/
+*.swp
+*~
+
+# Go
+*.test
+*.prof
+coverage.out
A => Justfile +29 -0
@@ 1,29 @@
+binary_name := "cacher"
+version := `git describe --long 2>/dev/null || cat VERSION 2>/dev/null || echo dev`
+
+default:
+ @just --list
+
+build:
+ go build -ldflags "-X go.bigb.es/cacher/internal/version.version={{version}}" -o {{binary_name}} .
+
+build-static:
+ CGO_ENABLED=0 go build -ldflags "-s -w -X go.bigb.es/cacher/internal/version.version={{version}}" -o {{binary_name}} .
+
+build-linux:
+ GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-s -w -X go.bigb.es/cacher/internal/version.version={{version}}" -o {{binary_name}}-linux-amd64 .
+
+install:
+ go install -ldflags "-X go.bigb.es/cacher/internal/version.version={{version}}" .
+
+test:
+ go test ./...
+
+lint:
+ golangci-lint run ./...
+
+tidy:
+ go mod tidy
+
+clean:
+ rm -f {{binary_name}} {{binary_name}}-linux-amd64
A => VERSION +1 -0
A => cmd/common.go +127 -0
@@ 1,127 @@
+package cmd
+
+import (
+ "fmt"
+ "runtime"
+
+ "github.com/spf13/cobra"
+
+ "go.bigb.es/cacher/internal/config"
+ "go.bigb.es/cacher/internal/hash"
+ "go.bigb.es/cacher/internal/s3"
+)
+
+// Flags overridable per command (flag > env > config > defaults).
+var (
+ flagEndpoint string
+ flagRegion string
+ flagBucket string
+ flagPrefix string
+ flagArchSuffix string // "true"/"false"/"" (unset)
+ flagKeyFile string
+ flagSecretFile string
+
+ // Key derivation (attached by addKeyFlags).
+ flagHashFrom []string
+ flagHashLength int
+)
+
+// addKeyFlags attaches --hash-from / --hash-length. Use on every command
+// that takes a <key> argument.
+func addKeyFlags(c *cobra.Command) {
+ c.Flags().StringSliceVar(&flagHashFrom, "hash-from", nil,
+ "path(s) whose sha256 is substituted for {hash} in the key (repeatable; file or directory)")
+ c.Flags().IntVar(&flagHashLength, "hash-length", hash.DefaultLength,
+ "truncate the derived hash to N hex chars")
+}
+
+// resolveKey applies --hash-from and --arch-suffix to the user-supplied
+// key template. With no --hash-from, the template is returned verbatim
+// (plus arch suffix if enabled).
+func resolveKey(keyTemplate string, cfg config.Config) (string, error) {
+ var derived string
+ if len(flagHashFrom) > 0 {
+ var err error
+ derived, err = hash.Derive(flagHashFrom, flagHashLength)
+ if err != nil {
+ return "", err
+ }
+ }
+ return hash.ApplyTemplate(keyTemplate, derived, runtime.GOOS, runtime.GOARCH, cfg.ArchSuffix), nil
+}
+
+// addS3Flags attaches the standard S3 override flags to a command. Most
+// commands accept them so single-shot invocations can override config.
+func addS3Flags(c *cobra.Command) {
+ c.Flags().StringVar(&flagEndpoint, "endpoint", "", "S3 endpoint URL (overrides config)")
+ c.Flags().StringVar(&flagRegion, "region", "", "S3 region (overrides config)")
+ c.Flags().StringVar(&flagBucket, "bucket", "", "S3 bucket (overrides config)")
+ c.Flags().StringVar(&flagPrefix, "prefix", "", "S3 key prefix (overrides config)")
+ c.Flags().StringVar(&flagArchSuffix, "arch-suffix", "", "append -<goos>-<goarch> to key (true|false; default from config)")
+ c.Flags().StringVar(&flagKeyFile, "key-file", "", "path to S3 access key id file (overrides config)")
+ c.Flags().StringVar(&flagSecretFile, "secret-file", "", "path to S3 secret file (overrides config)")
+}
+
+// loadConfig merges file → env → flag overrides.
+func loadConfig() (config.Config, error) {
+ cfg, err := config.Load(flagConfigPath)
+ if err != nil {
+ return cfg, err
+ }
+ if err := config.ApplyEnv(&cfg); err != nil {
+ return cfg, err
+ }
+ if flagEndpoint != "" {
+ cfg.Endpoint = flagEndpoint
+ }
+ if flagRegion != "" {
+ cfg.Region = flagRegion
+ }
+ if flagBucket != "" {
+ cfg.Bucket = flagBucket
+ }
+ if flagPrefix != "" {
+ cfg.Prefix = flagPrefix
+ }
+ if flagKeyFile != "" {
+ cfg.KeyFile = flagKeyFile
+ }
+ if flagSecretFile != "" {
+ cfg.SecretFile = flagSecretFile
+ }
+ switch flagArchSuffix {
+ case "true":
+ cfg.ArchSuffix = true
+ case "false":
+ cfg.ArchSuffix = false
+ case "":
+ // inherit from file/env
+ default:
+ return cfg, fmt.Errorf("--arch-suffix must be true|false, got %q", flagArchSuffix)
+ }
+ return cfg, nil
+}
+
+// client builds a Garage-tuned S3 client from the resolved config.
+func client() (*s3.Client, config.Config, error) {
+ cfg, err := loadConfig()
+ if err != nil {
+ return nil, cfg, err
+ }
+ if err := cfg.Validate(); err != nil {
+ return nil, cfg, err
+ }
+ key, secret, err := cfg.Credentials()
+ if err != nil {
+ return nil, cfg, err
+ }
+ c, err := s3.New(s3.Options{
+ Endpoint: cfg.Endpoint,
+ Region: cfg.Region,
+ Bucket: cfg.Bucket,
+ Prefix: cfg.Prefix,
+ KeyID: key,
+ Secret: secret,
+ })
+ return c, cfg, err
+}
A => cmd/completion.go +57 -0
@@ 1,57 @@
+package cmd
+
+import (
+ "os"
+
+ "github.com/spf13/cobra"
+)
+
+var completionCmd = &cobra.Command{
+ Use: "completion [bash|zsh|fish|powershell]",
+ Short: "Generate shell completion scripts",
+ Long: `Generate shell completion scripts for cacher.
+
+To load completions:
+
+Bash:
+ $ source <(cacher completion bash)
+ # To load completions for each session, execute once:
+ # Linux:
+ $ cacher completion bash > /etc/bash_completion.d/cacher
+ # macOS:
+ $ cacher completion bash > $(brew --prefix)/etc/bash_completion.d/cacher
+
+Zsh:
+ $ source <(cacher completion zsh)
+ # To load completions for each session, execute once:
+ $ cacher completion zsh > "${fpath[1]}/_cacher"
+
+Fish:
+ $ cacher completion fish | source
+ # To load completions for each session, execute once:
+ $ cacher completion fish > ~/.config/fish/completions/cacher.fish
+
+PowerShell:
+ PS> cacher completion powershell | Out-String | Invoke-Expression
+`,
+ DisableFlagsInUseLine: true,
+ ValidArgs: []string{"bash", "zsh", "fish", "powershell"},
+ Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ switch args[0] {
+ case "bash":
+ return rootCmd.GenBashCompletion(os.Stdout)
+ case "zsh":
+ return rootCmd.GenZshCompletion(os.Stdout)
+ case "fish":
+ return rootCmd.GenFishCompletion(os.Stdout, true)
+ case "powershell":
+ return rootCmd.GenPowerShellCompletionWithDesc(os.Stdout)
+ }
+ return nil
+ },
+}
+
+func init() {
+ rootCmd.AddCommand(completionCmd)
+}
A => cmd/dir.go +110 -0
@@ 1,110 @@
+package cmd
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "os"
+
+ "github.com/spf13/cobra"
+
+ "go.bigb.es/cacher/internal/archive"
+)
+
+var dirCmd = &cobra.Command{
+ Use: "dir",
+ Short: "Directory cache (tar+zstd of a tree)",
+ Long: `Cache resolved trees keyed by content hash (e.g. ~/go/pkg/mod
+keyed by go.sum, or .rocks/ keyed by rockspec hash). Closes the biggest
+gap left by the single-file shell helper this binary replaces.`,
+}
+
+var dirDownloadCmd = &cobra.Command{
+ Use: "download <key> <local-dir>",
+ Short: "Extract a cached directory into <local-dir>",
+ Args: cobra.ExactArgs(2),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ cli, cfg, err := client()
+ if err != nil {
+ return err
+ }
+ key, err := resolveKey(args[0], cfg)
+ if err != nil {
+ return err
+ }
+ dest := args[1]
+ ctx := context.Background()
+
+ ok, err := cli.Exists(ctx, key)
+ if err != nil {
+ return err
+ }
+ if !ok {
+ return fmt.Errorf("%w: %s", ErrNotFound, key)
+ }
+ if err := os.MkdirAll(dest, 0o755); err != nil {
+ return err
+ }
+ fmt.Fprintf(cmd.ErrOrStderr(), "Cache HIT — %s → extract %s\n", key, dest)
+ body, err := cli.Get(ctx, key)
+ if err != nil {
+ return err
+ }
+ defer body.Close()
+ return archive.DecodeDir(body, dest)
+ },
+}
+
+var dirUploadCmd = &cobra.Command{
+ Use: "upload <key> <local-dir>",
+ Short: "Pack a local directory (tar+zstd) and upload",
+ Args: cobra.ExactArgs(2),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ cli, cfg, err := client()
+ if err != nil {
+ return err
+ }
+ key, err := resolveKey(args[0], cfg)
+ if err != nil {
+ return err
+ }
+ src := args[1]
+ ctx := context.Background()
+
+ if !flagForce {
+ ok, err := cli.Exists(ctx, key)
+ if err != nil {
+ return err
+ }
+ if ok {
+ fmt.Fprintf(cmd.ErrOrStderr(), "Skipped — %s already present (use --force)\n", key)
+ return nil
+ }
+ }
+ st, err := os.Stat(src)
+ if err != nil {
+ return err
+ }
+ if !st.IsDir() {
+ return fmt.Errorf("%s is not a directory", src)
+ }
+ fmt.Fprintf(cmd.ErrOrStderr(), "Packing %s → %s\n", src, key)
+
+ pr, pw := io.Pipe()
+ go func() {
+ err := archive.EncodeDir(pw, src)
+ pw.CloseWithError(err)
+ }()
+ return cli.Put(ctx, key, pr)
+ },
+}
+
+func init() {
+ for _, c := range []*cobra.Command{dirDownloadCmd, dirUploadCmd} {
+ addS3Flags(c)
+ addKeyFlags(c)
+ }
+ dirUploadCmd.Flags().BoolVar(&flagForce, "force", false, "overwrite if key already exists")
+ dirCmd.AddCommand(dirDownloadCmd, dirUploadCmd)
+ rootCmd.AddCommand(dirCmd)
+}
A => cmd/docker.go +201 -0
@@ 1,201 @@
+package cmd
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "os/exec"
+
+ "github.com/klauspost/compress/zstd"
+ "github.com/spf13/cobra"
+)
+
+var dockerCmd = &cobra.Command{
+ Use: "docker",
+ Short: "Docker image cache (tar+zstd streamed to S3)",
+}
+
+var dockerExistsCmd = &cobra.Command{
+ Use: "exists <key>",
+ Short: "Exit 0 if a cached docker image is present at key, 1 if missing",
+ Args: cobra.ExactArgs(1),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ cli, cfg, err := client()
+ if err != nil {
+ return err
+ }
+ key, err := resolveKey(args[0], cfg)
+ if err != nil {
+ return err
+ }
+ ok, err := cli.Exists(context.Background(), key)
+ if err != nil {
+ return err
+ }
+ fmt.Fprintln(cmd.OutOrStdout(), key)
+ if !ok {
+ return ErrNotFound
+ }
+ return nil
+ },
+}
+
+var dockerDownloadCmd = &cobra.Command{
+ Use: "download <key> <image:tag>",
+ Short: "Pull a cached image from S3 into the local docker daemon",
+ Long: `Streams s3://bucket/key through zstd-decode into ` + "`docker load`" + `.`,
+ Args: cobra.ExactArgs(2),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ cli, cfg, err := client()
+ if err != nil {
+ return err
+ }
+ key, err := resolveKey(args[0], cfg)
+ if err != nil {
+ return err
+ }
+ // The <image:tag> arg is for the caller's own ergonomics + logging;
+ // docker load reads the tag from the tar stream itself.
+ tag := args[1]
+ ctx := context.Background()
+ ok, err := cli.Exists(ctx, key)
+ if err != nil {
+ return err
+ }
+ if !ok {
+ return fmt.Errorf("%w: %s", ErrNotFound, key)
+ }
+ fmt.Fprintf(cmd.ErrOrStderr(), "Cache HIT — %s → load %s\n", key, tag)
+
+ body, err := cli.Get(ctx, key)
+ if err != nil {
+ return err
+ }
+ defer body.Close()
+ dec, err := zstd.NewReader(body)
+ if err != nil {
+ return err
+ }
+ defer dec.Close()
+ return runDockerLoad(ctx, dec)
+ },
+}
+
+var dockerUploadCmd = &cobra.Command{
+ Use: "upload <key> <image:tag>",
+ Short: "Save a local docker image to the S3 cache",
+ Long: `Pipes ` + "`docker save image:tag`" + ` through zstd-encode into a streamed
+S3 multipart upload. No on-disk tempfile.`,
+ Args: cobra.ExactArgs(2),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ cli, cfg, err := client()
+ if err != nil {
+ return err
+ }
+ key, err := resolveKey(args[0], cfg)
+ if err != nil {
+ return err
+ }
+ tag := args[1]
+ ctx := context.Background()
+
+ if !flagForce {
+ ok, err := cli.Exists(ctx, key)
+ if err != nil {
+ return err
+ }
+ if ok {
+ fmt.Fprintf(cmd.ErrOrStderr(), "Skipped — %s already present (use --force)\n", key)
+ return nil
+ }
+ }
+ fmt.Fprintf(cmd.ErrOrStderr(), "Saving %s → %s\n", tag, key)
+ return saveTagToS3(ctx, cli, key, tag)
+ },
+}
+
+func init() {
+ for _, c := range []*cobra.Command{dockerExistsCmd, dockerDownloadCmd, dockerUploadCmd} {
+ addS3Flags(c)
+ addKeyFlags(c)
+ }
+ dockerUploadCmd.Flags().BoolVar(&flagForce, "force", false, "overwrite if key already exists")
+ dockerCmd.AddCommand(dockerExistsCmd, dockerDownloadCmd, dockerUploadCmd)
+ rootCmd.AddCommand(dockerCmd)
+}
+
+// saveTagToS3 wires `docker save` stdout → zstd encoder → io.Pipe →
+// s3.Put. The encoder runs in a goroutine writing into the pipe; the
+// uploader goroutine reads from the pipe. Both ends signal errors back
+// through pw.CloseWithError so the upload sees a clean EOF or a wrapped
+// failure.
+func saveTagToS3(ctx context.Context, cli s3Putter, key, tag string) error {
+ save := exec.CommandContext(ctx, "docker", "save", tag)
+ saveOut, err := save.StdoutPipe()
+ if err != nil {
+ return err
+ }
+ save.Stderr = newWarnWriter("docker save")
+ if err := save.Start(); err != nil {
+ return fmt.Errorf("docker save: %w", err)
+ }
+
+ pr, pw := io.Pipe()
+ go func() {
+ zw, zerr := zstd.NewWriter(pw, zstd.WithEncoderLevel(zstd.SpeedDefault))
+ if zerr != nil {
+ pw.CloseWithError(zerr)
+ return
+ }
+ if _, cerr := io.Copy(zw, saveOut); cerr != nil {
+ zw.Close()
+ pw.CloseWithError(cerr)
+ return
+ }
+ if zerr := zw.Close(); zerr != nil {
+ pw.CloseWithError(zerr)
+ return
+ }
+ pw.Close()
+ }()
+
+ upErr := cli.Put(ctx, key, pr)
+ waitErr := save.Wait()
+ if upErr != nil {
+ return upErr
+ }
+ if waitErr != nil {
+ return fmt.Errorf("docker save: %w", waitErr)
+ }
+ return nil
+}
+
+// runDockerLoad pipes r (already zstd-decoded) into `docker load`.
+func runDockerLoad(ctx context.Context, r io.Reader) error {
+ load := exec.CommandContext(ctx, "docker", "load")
+ stdin, err := load.StdinPipe()
+ if err != nil {
+ return err
+ }
+ load.Stdout = newWarnWriter("docker load")
+ load.Stderr = newWarnWriter("docker load")
+ if err := load.Start(); err != nil {
+ return fmt.Errorf("docker load: %w", err)
+ }
+ _, copyErr := io.Copy(stdin, r)
+ stdin.Close()
+ waitErr := load.Wait()
+ if copyErr != nil {
+ return fmt.Errorf("docker load write: %w", copyErr)
+ }
+ if waitErr != nil {
+ return fmt.Errorf("docker load: %w", waitErr)
+ }
+ return nil
+}
+
+// s3Putter is satisfied by *s3.Client; declared here so saveTagToS3 is
+// testable with a fake.
+type s3Putter interface {
+ Put(ctx context.Context, key string, r io.Reader) error
+}
A => cmd/doctor.go +88 -0
@@ 1,88 @@
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "crypto/rand"
+ "encoding/hex"
+ "fmt"
+ "io"
+
+ "github.com/spf13/cobra"
+)
+
+var doctorCmd = &cobra.Command{
+ Use: "doctor",
+ Short: "Smoke-test S3 credentials and bucket reachability",
+ Long: `Validates that cacher can reach the configured S3 bucket using the
+resolved credentials. Performs a HEAD on the bucket and then writes,
+reads, and deletes a 1-byte test object. Reports the access key id's
+length and first two characters as a diagnostic — mirrors the existing
+ci_aws_setup shell helper's "key_id_len=… key_id_prefix=…" output —
+without leaking the secret.`,
+ RunE: func(cmd *cobra.Command, _ []string) error { return runDoctor(cmd) },
+}
+
+func init() {
+ addS3Flags(doctorCmd)
+ rootCmd.AddCommand(doctorCmd)
+}
+
+func runDoctor(cmd *cobra.Command) error {
+ cli, cfg, err := client()
+ if err != nil {
+ return err
+ }
+ out := cmd.OutOrStdout()
+
+ keyID, secret, _ := cfg.Credentials() // already validated by client()
+ fmt.Fprintf(out, "endpoint=%s region=%s bucket=%s prefix=%s\n", cfg.Endpoint, cfg.Region, cfg.Bucket, cfg.Prefix)
+ fmt.Fprintf(out, "key_id_len=%d key_id_prefix=%s secret_len=%d\n",
+ len(keyID), safePrefix(keyID, 2), len(secret))
+
+ ctx := context.Background()
+ if err := cli.HeadBucket(ctx); err != nil {
+ return err
+ }
+ fmt.Fprintln(out, "head bucket: OK")
+
+ // Write/read/delete a tiny canary so credential perms are exercised
+ // for all three operations, not just Head. Key is random so concurrent
+ // doctor runs don't fight.
+ r := make([]byte, 8)
+ _, _ = rand.Read(r)
+ key := ".cacher-doctor-" + hex.EncodeToString(r)
+ body := []byte("ok")
+ if err := cli.Put(ctx, key, bytes.NewReader(body)); err != nil {
+ return err
+ }
+ fmt.Fprintln(out, "put canary: OK")
+
+ rc, err := cli.Get(ctx, key)
+ if err != nil {
+ return err
+ }
+ got, err := io.ReadAll(rc)
+ rc.Close()
+ if err != nil {
+ return err
+ }
+ if !bytes.Equal(got, body) {
+ return fmt.Errorf("canary read mismatch: got %q want %q", got, body)
+ }
+ fmt.Fprintln(out, "get canary: OK")
+
+ if err := cli.Delete(ctx, key); err != nil {
+ return err
+ }
+ fmt.Fprintln(out, "delete canary: OK")
+ fmt.Fprintln(out, "doctor: all checks passed")
+ return nil
+}
+
+func safePrefix(s string, n int) string {
+ if len(s) < n {
+ return s
+ }
+ return s[:n]
+}
A => cmd/errors.go +26 -0
@@ 1,26 @@
+package cmd
+
+import "errors"
+
+// Sentinel errors recognized by Execute → exit code mapping.
+//
+// exists → ErrNotFound → 1
+// download without --url, key missing → ErrMissNoFallback → 3
+// any other error → 2
+var (
+ ErrNotFound = errors.New("cacher: key not found")
+ ErrMissNoFallback = errors.New("cacher: cache miss and no --url fallback")
+)
+
+func exitCodeFor(err error) int {
+ switch {
+ case err == nil:
+ return 0
+ case errors.Is(err, ErrNotFound):
+ return 1
+ case errors.Is(err, ErrMissNoFallback):
+ return 3
+ default:
+ return 2
+ }
+}
A => cmd/file.go +301 -0
@@ 1,301 @@
+package cmd
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "strings"
+
+ "github.com/spf13/cobra"
+)
+
+var (
+ flagDownloadURL string
+ flagDownloadSHA256 string
+ flagForce bool
+)
+
+var downloadCmd = &cobra.Command{
+ Use: "download <key> <local-path>",
+ Short: "Download a cached file; fall back to --url on cache miss",
+ Args: cobra.ExactArgs(2),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ cli, cfg, err := client()
+ if err != nil {
+ return err
+ }
+ key, err := resolveKey(args[0], cfg)
+ if err != nil {
+ return err
+ }
+ dest := args[1]
+
+ ctx := context.Background()
+ exists, err := cli.Exists(ctx, key)
+ if err != nil {
+ return err
+ }
+ if exists {
+ fmt.Fprintf(cmd.ErrOrStderr(), "Cache HIT — %s\n", key)
+ rc, err := cli.Get(ctx, key)
+ if err != nil {
+ return err
+ }
+ defer rc.Close()
+ return writeWithSHA(dest, rc, flagDownloadSHA256)
+ }
+
+ fmt.Fprintf(cmd.ErrOrStderr(), "Cache MISS — %s\n", key)
+ if flagDownloadURL == "" {
+ return fmt.Errorf("%w: %s", ErrMissNoFallback, key)
+ }
+ fmt.Fprintf(cmd.ErrOrStderr(), "Fetching %s\n", flagDownloadURL)
+ body, err := httpGet(flagDownloadURL)
+ if err != nil {
+ return err
+ }
+ defer body.Close()
+ // Tee to a tempfile so we can verify the checksum before re-reading
+ // for the upload step. Pure-streaming up to s3 + sha256 verify is
+ // possible but complicates rewind on mismatch — tempfile keeps it
+ // honest at the cost of one extra disk pass.
+ tmp, err := os.CreateTemp("", "cacher-dl-*")
+ if err != nil {
+ return err
+ }
+ defer os.Remove(tmp.Name())
+ defer tmp.Close()
+ if _, err := io.Copy(tmp, body); err != nil {
+ return err
+ }
+ if _, err := tmp.Seek(0, io.SeekStart); err != nil {
+ return err
+ }
+ if err := writeWithSHA(dest, tmp, flagDownloadSHA256); err != nil {
+ return err
+ }
+ // Re-open the verified destination file for upload to S3 so callers
+ // see the cached file on next run.
+ f, err := os.Open(dest)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ if err := cli.Put(ctx, key, f); err != nil {
+ fmt.Fprintf(cmd.ErrOrStderr(), "warn: upload to S3 failed: %v\n", err)
+ // Local download succeeded — don't fail the build just because
+ // of an upload glitch.
+ return nil
+ }
+ fmt.Fprintf(cmd.ErrOrStderr(), "Cached → %s\n", key)
+ return nil
+ },
+}
+
+var uploadCmd = &cobra.Command{
+ Use: "upload <key> <local-path>",
+ Short: "Upload a local file to the cache",
+ Args: cobra.ExactArgs(2),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ cli, cfg, err := client()
+ if err != nil {
+ return err
+ }
+ key, err := resolveKey(args[0], cfg)
+ if err != nil {
+ return err
+ }
+ ctx := context.Background()
+ if !flagForce {
+ exists, err := cli.Exists(ctx, key)
+ if err != nil {
+ return err
+ }
+ if exists {
+ fmt.Fprintf(cmd.ErrOrStderr(), "Skipped — %s already present (use --force to overwrite)\n", key)
+ return nil
+ }
+ }
+ f, err := os.Open(args[1])
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ if err := cli.Put(ctx, key, f); err != nil {
+ return err
+ }
+ fmt.Fprintf(cmd.ErrOrStderr(), "Uploaded → %s\n", key)
+ return nil
+ },
+}
+
+var existsCmd = &cobra.Command{
+ Use: "exists <key>",
+ Short: "Exit 0 if key exists, 1 if missing, 2 on error",
+ Args: cobra.ExactArgs(1),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ cli, cfg, err := client()
+ if err != nil {
+ return err
+ }
+ key, err := resolveKey(args[0], cfg)
+ if err != nil {
+ return err
+ }
+ ok, err := cli.Exists(context.Background(), key)
+ if err != nil {
+ return err
+ }
+ fmt.Fprintln(cmd.OutOrStdout(), key)
+ if !ok {
+ return ErrNotFound
+ }
+ return nil
+ },
+}
+
+var listCmd = &cobra.Command{
+ Use: "list [sub-prefix]",
+ Short: "List cache keys under prefix (optionally narrowed by sub-prefix)",
+ Args: cobra.MaximumNArgs(1),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ cli, _, err := client()
+ if err != nil {
+ return err
+ }
+ sub := ""
+ if len(args) == 1 {
+ sub = args[0]
+ }
+ keys, err := cli.List(context.Background(), sub)
+ if err != nil {
+ return err
+ }
+ out := cmd.OutOrStdout()
+ for _, k := range keys {
+ fmt.Fprintln(out, k)
+ }
+ return nil
+ },
+}
+
+var deleteCmd = &cobra.Command{
+ Use: "delete <key>",
+ Short: "Delete a cache key (no error if missing)",
+ Args: cobra.ExactArgs(1),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ cli, cfg, err := client()
+ if err != nil {
+ return err
+ }
+ key, err := resolveKey(args[0], cfg)
+ if err != nil {
+ return err
+ }
+ if err := cli.Delete(context.Background(), key); err != nil {
+ return err
+ }
+ fmt.Fprintf(cmd.ErrOrStderr(), "Deleted — %s\n", key)
+ return nil
+ },
+}
+
+var keyCmd = &cobra.Command{
+ Use: "key <template>",
+ Short: "Print the resolved key for a template (for shell scripting)",
+ Long: `Resolves a key template by substituting --hash-from values for {hash}
+and appending the arch suffix when --arch-suffix=true. Useful when the
+same key needs to appear in multiple subsequent invocations:
+
+ KEY=$(cacher key conformance/{hash}.tar.zst --hash-from docker/conformance.Dockerfile)
+ if ! cacher docker exists "$KEY"; then
+ docker build …
+ cacher docker upload "$KEY" tag:latest
+ fi`,
+ Args: cobra.ExactArgs(1),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ cfg, err := loadConfig()
+ if err != nil {
+ return err
+ }
+ key, err := resolveKey(args[0], cfg)
+ if err != nil {
+ return err
+ }
+ fmt.Fprintln(cmd.OutOrStdout(), key)
+ return nil
+ },
+}
+
+func init() {
+ downloadCmd.Flags().StringVar(&flagDownloadURL, "url", "", "fallback URL when key is missing in S3")
+ downloadCmd.Flags().StringVar(&flagDownloadSHA256, "sha256", "", "expected sha256 hex of the downloaded content")
+ uploadCmd.Flags().BoolVar(&flagForce, "force", false, "overwrite if key already exists")
+
+ for _, c := range []*cobra.Command{downloadCmd, uploadCmd, existsCmd, deleteCmd} {
+ addS3Flags(c)
+ addKeyFlags(c)
+ }
+ addS3Flags(listCmd)
+ addS3Flags(keyCmd)
+ addKeyFlags(keyCmd)
+
+ rootCmd.AddCommand(downloadCmd, uploadCmd, existsCmd, listCmd, deleteCmd, keyCmd)
+}
+
+// writeWithSHA streams r into path; if want is non-empty, verifies the
+// sha256 matches. On mismatch, the partial file is removed.
+func writeWithSHA(path string, r io.Reader, want string) error {
+ want = strings.ToLower(want)
+ if err := os.MkdirAll(parentDir(path), 0o755); err != nil {
+ return err
+ }
+ tmp, err := os.CreateTemp(parentDir(path), ".cacher-tmp-*")
+ if err != nil {
+ return err
+ }
+ tmpName := tmp.Name()
+ defer os.Remove(tmpName) // no-op once renamed
+
+ h := sha256.New()
+ mw := io.MultiWriter(tmp, h)
+ if _, err := io.Copy(mw, r); err != nil {
+ tmp.Close()
+ return err
+ }
+ if err := tmp.Close(); err != nil {
+ return err
+ }
+ if want != "" {
+ got := hex.EncodeToString(h.Sum(nil))
+ if got != want {
+ return fmt.Errorf("sha256 mismatch: got %s want %s", got, want)
+ }
+ }
+ return os.Rename(tmpName, path)
+}
+
+func parentDir(path string) string {
+ for i := len(path) - 1; i >= 0; i-- {
+ if path[i] == '/' || path[i] == os.PathSeparator {
+ return path[:i]
+ }
+ }
+ return "."
+}
+
+func httpGet(url string) (io.ReadCloser, error) {
+ resp, err := http.Get(url)
+ if err != nil {
+ return nil, fmt.Errorf("GET %s: %w", url, err)
+ }
+ if resp.StatusCode/100 != 2 {
+ resp.Body.Close()
+ return nil, fmt.Errorf("GET %s: HTTP %d", url, resp.StatusCode)
+ }
+ return resp.Body, nil
+}
A => cmd/init.go +74 -0
@@ 1,74 @@
+package cmd
+
+import (
+ "fmt"
+
+ "github.com/spf13/cobra"
+
+ "go.bigb.es/cacher/internal/config"
+)
+
+var initCmd = &cobra.Command{
+ Use: "init",
+ Short: "Write ~/.config/cacher/config.toml and verify credentials",
+ Long: `Persist cacher's S3 connection settings to ~/.config/cacher/config.toml
+so subsequent commands don't need flags or env vars. After writing, runs
+the same smoke test as ` + "`cacher doctor`" + ` to fail fast on bad creds.
+
+Auth files (--key-file, --secret-file) are recorded in the config but
+not read here — they're read on demand by every subsequent command.
+Credentials passed via CACHER_S3_KEY_ID / CACHER_S3_SECRET env vars
+take precedence over the files at use time.`,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ // Build the config we will persist. init does not honour the
+ // in-process config file — it always writes a fresh one.
+ cfg := config.Defaults()
+ if flagEndpoint != "" {
+ cfg.Endpoint = flagEndpoint
+ }
+ if flagRegion != "" {
+ cfg.Region = flagRegion
+ }
+ if flagBucket != "" {
+ cfg.Bucket = flagBucket
+ }
+ if flagPrefix != "" {
+ cfg.Prefix = flagPrefix
+ }
+ if flagKeyFile != "" {
+ cfg.KeyFile = flagKeyFile
+ }
+ if flagSecretFile != "" {
+ cfg.SecretFile = flagSecretFile
+ }
+ switch flagArchSuffix {
+ case "true":
+ cfg.ArchSuffix = true
+ case "false":
+ cfg.ArchSuffix = false
+ }
+
+ if err := cfg.Validate(); err != nil {
+ return err
+ }
+
+ path := flagConfigPath
+ if path == "" {
+ p, err := config.DefaultPath()
+ if err != nil {
+ return err
+ }
+ path = p
+ }
+ if err := config.Save(path, cfg); err != nil {
+ return err
+ }
+ fmt.Fprintf(cmd.OutOrStdout(), "wrote %s\n", path)
+ return runDoctor(cmd)
+ },
+}
+
+func init() {
+ addS3Flags(initCmd)
+ rootCmd.AddCommand(initCmd)
+}
A => cmd/root.go +51 -0
@@ 1,51 @@
+package cmd
+
+import (
+ "fmt"
+ "log/slog"
+ "os"
+
+ "github.com/spf13/cobra"
+
+ "go.bigb.es/cacher/internal/logging"
+)
+
+var (
+ flagLogLevel string
+ flagLogFormat string
+ flagConfigPath string
+)
+
+var rootCmd = &cobra.Command{
+ Use: "cacher",
+ Short: "S3-backed CI cache helper for builds.sr.ht — replaces .builds/lib/ci-lib.sh",
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+ level := slog.LevelInfo
+ switch flagLogLevel {
+ case "debug":
+ level = slog.LevelDebug
+ case "info":
+ level = slog.LevelInfo
+ case "warn":
+ level = slog.LevelWarn
+ case "error":
+ level = slog.LevelError
+ default:
+ return fmt.Errorf("invalid --log-level %q (want debug|info|warn|error)", flagLogLevel)
+ }
+ logging.Setup(flagLogFormat, os.Stderr, level)
+ return nil
+ },
+}
+
+func Execute() {
+ if err := rootCmd.Execute(); err != nil {
+ os.Exit(exitCodeFor(err))
+ }
+}
+
+func init() {
+ rootCmd.PersistentFlags().StringVar(&flagLogLevel, "log-level", "info", "log level (debug|info|warn|error)")
+ rootCmd.PersistentFlags().StringVar(&flagLogFormat, "log-format", "human", "log format (human|json)")
+ rootCmd.PersistentFlags().StringVar(&flagConfigPath, "config", "", "config file (default ~/.config/cacher/config.toml)")
+}
A => cmd/util.go +20 -0
@@ 1,20 @@
+package cmd
+
+import (
+ "fmt"
+ "io"
+ "os"
+)
+
+// newWarnWriter forwards everything written to it to stderr, prefixed
+// with a tag. Used to surface docker subcommand stderr in CI logs.
+func newWarnWriter(tag string) io.Writer {
+ return &warnWriter{tag: tag}
+}
+
+type warnWriter struct{ tag string }
+
+func (w *warnWriter) Write(p []byte) (int, error) {
+ fmt.Fprintf(os.Stderr, "[%s] %s", w.tag, p)
+ return len(p), nil
+}
A => cmd/version.go +32 -0
@@ 1,32 @@
+package cmd
+
+import (
+ "fmt"
+
+ "github.com/spf13/cobra"
+
+ "go.bigb.es/cacher/internal/version"
+)
+
+const banner = ` _
+ ___ __ _ ___ _ __ | |__ ___ _ __
+ / __/ _` + "`" + ` |/ __| '_ \| '_ \ / _ \ '__|
+ | (_| (_| | (__| | | | | | | __/ |
+ \___\__,_|\___|_| |_|_| |_|\___|_|`
+
+var versionCmd = &cobra.Command{
+ Use: "version",
+ Short: "Print version information",
+ Run: func(cmd *cobra.Command, args []string) {
+ info := version.Get()
+ fmt.Println(banner)
+ fmt.Println("cacher: S3-backed CI cache helper for builds.sr.ht — replaces .builds/lib/ci-lib.sh")
+ fmt.Println("go.bigb.es/cacher")
+ fmt.Println()
+ fmt.Println(info)
+ },
+}
+
+func init() {
+ rootCmd.AddCommand(versionCmd)
+}
A => go.mod +27 -0
@@ 1,27 @@
+module go.bigb.es/cacher
+
+go 1.26.3
+
+require (
+ github.com/BurntSushi/toml v1.6.0
+ github.com/aws/aws-sdk-go-v2 v1.41.7
+ github.com/aws/aws-sdk-go-v2/credentials v1.19.17
+ github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.19
+ github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
+ github.com/aws/smithy-go v1.25.1
+ github.com/klauspost/compress v1.18.6
+ github.com/spf13/cobra v1.10.2
+)
+
+require (
+ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
+ github.com/inconshreveable/mousetrap v1.1.0 // indirect
+ github.com/spf13/pflag v1.0.9 // indirect
+)
A => go.sum +52 -0
@@ 1,52 @@
+github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
+github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
+github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
+github.com/aws/aws-sdk-go-v2/config v1.32.18 h1:Hcia46bxhGgF3BaSnG8nSNCWmqTK6bj9xN9/FJ3WK6Q=
+github.com/aws/aws-sdk-go-v2/config v1.32.18/go.mod h1:zEjCAYmxqDadH1WX8CdBvmLKhUEUVFgKRQG38zjDmrY=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.17 h1:gP2nkGsS+KMvF/jfFz2Vv2qiiOqWKyPACSzPsqHgoW8=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.17/go.mod h1:Bsew3S/moG5iT77giPj1q8wb/s0RE5/QfH+ASjYtuQc=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.19 h1:VH0xfFwHfPYhu+EcxyCcw3VTZskpbA+/s0pTXwhSsL8=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.19/go.mod h1:S/XkAXcnCpzwsjC9EU0BakuvreXfSTUADHb7rC7jvaQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.0 h1:nDARhv/oF55bcxF7rCI/4PDxOKnVXVWwDuDwCs2I2SQ=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.0/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg=
+github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk=
+github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
+github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
+github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
+github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
+github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
+github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
+github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
A => internal/archive/archive.go +178 -0
@@ 1,178 @@
+// Package archive streams a directory through tar+zstd (and back).
+//
+// Compression level is fixed at zstd level 3 — same as the existing shell
+// (`zstd -T0 -3`). The encoder is wrapped around an io.Writer (typically an
+// S3 multipart Uploader's PipeWriter), so the whole pipeline stays
+// streaming end-to-end with no on-disk tempfile.
+package archive
+
+import (
+ "archive/tar"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "github.com/klauspost/compress/zstd"
+)
+
+// EncodeDir walks root in sorted relative-path order and writes a
+// tar.zst stream into w. Symlinks are preserved (stored as tar typelink),
+// devices/sockets/fifos are skipped.
+func EncodeDir(w io.Writer, root string) error {
+ zw, err := zstd.NewWriter(w, zstd.WithEncoderLevel(zstd.SpeedDefault))
+ if err != nil {
+ return fmt.Errorf("zstd writer: %w", err)
+ }
+ defer zw.Close()
+ tw := tar.NewWriter(zw)
+
+ var paths []string
+ err = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if p == root {
+ return nil
+ }
+ paths = append(paths, p)
+ return nil
+ })
+ if err != nil {
+ return fmt.Errorf("walk %s: %w", root, err)
+ }
+ sort.Strings(paths)
+
+ for _, p := range paths {
+ if err := writeOne(tw, root, p); err != nil {
+ return err
+ }
+ }
+ if err := tw.Close(); err != nil {
+ return fmt.Errorf("close tar: %w", err)
+ }
+ if err := zw.Close(); err != nil {
+ return fmt.Errorf("close zstd: %w", err)
+ }
+ return nil
+}
+
+func writeOne(tw *tar.Writer, root, p string) error {
+ rel, err := filepath.Rel(root, p)
+ if err != nil {
+ return err
+ }
+ rel = filepath.ToSlash(rel)
+
+ lst, err := os.Lstat(p)
+ if err != nil {
+ return fmt.Errorf("lstat %s: %w", p, err)
+ }
+
+ var link string
+ if lst.Mode()&os.ModeSymlink != 0 {
+ link, err = os.Readlink(p)
+ if err != nil {
+ return fmt.Errorf("readlink %s: %w", p, err)
+ }
+ }
+ hdr, err := tar.FileInfoHeader(lst, link)
+ if err != nil {
+ // Skip unsupported file types (devices, sockets, fifos).
+ return nil
+ }
+ hdr.Name = rel
+ if lst.IsDir() && !strings.HasSuffix(hdr.Name, "/") {
+ hdr.Name += "/"
+ }
+ if err := tw.WriteHeader(hdr); err != nil {
+ return fmt.Errorf("tar header %s: %w", rel, err)
+ }
+ if !lst.Mode().IsRegular() {
+ return nil
+ }
+ f, err := os.Open(p)
+ if err != nil {
+ return fmt.Errorf("open %s: %w", p, err)
+ }
+ defer f.Close()
+ if _, err := io.Copy(tw, f); err != nil {
+ return fmt.Errorf("copy %s: %w", rel, err)
+ }
+ return nil
+}
+
+// DecodeDir extracts a tar.zst stream from r into dest. dest must exist.
+// Any entry whose normalized path escapes dest is rejected.
+func DecodeDir(r io.Reader, dest string) error {
+ absDest, err := filepath.Abs(dest)
+ if err != nil {
+ return fmt.Errorf("abs %s: %w", dest, err)
+ }
+ zr, err := zstd.NewReader(r)
+ if err != nil {
+ return fmt.Errorf("zstd reader: %w", err)
+ }
+ defer zr.Close()
+ tr := tar.NewReader(zr)
+
+ for {
+ hdr, err := tr.Next()
+ if errors.Is(err, io.EOF) {
+ return nil
+ }
+ if err != nil {
+ return fmt.Errorf("tar next: %w", err)
+ }
+ target, err := safeJoin(absDest, hdr.Name)
+ if err != nil {
+ return err
+ }
+ switch hdr.Typeflag {
+ case tar.TypeDir:
+ if err := os.MkdirAll(target, fs.FileMode(hdr.Mode)&0o7777); err != nil {
+ return fmt.Errorf("mkdir %s: %w", target, err)
+ }
+ case tar.TypeReg, tar.TypeRegA:
+ if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+ return fmt.Errorf("mkdir parent of %s: %w", target, err)
+ }
+ f, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fs.FileMode(hdr.Mode)&0o7777)
+ if err != nil {
+ return fmt.Errorf("create %s: %w", target, err)
+ }
+ if _, err := io.Copy(f, tr); err != nil {
+ f.Close()
+ return fmt.Errorf("write %s: %w", target, err)
+ }
+ if err := f.Close(); err != nil {
+ return err
+ }
+ case tar.TypeSymlink:
+ _ = os.Remove(target)
+ if err := os.Symlink(hdr.Linkname, target); err != nil {
+ return fmt.Errorf("symlink %s -> %s: %w", target, hdr.Linkname, err)
+ }
+ default:
+ // Skip unknown entry types silently.
+ }
+ }
+}
+
+// safeJoin returns base/sub, rejecting any sub that is absolute or contains
+// ".." segments that would resolve outside base. We reject rather than
+// silently re-root so malicious tarballs surface as an error.
+func safeJoin(base, sub string) (string, error) {
+ if filepath.IsAbs(sub) || strings.HasPrefix(sub, "/") {
+ return "", fmt.Errorf("absolute tar entry: %q", sub)
+ }
+ clean := filepath.Clean(sub)
+ if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
+ return "", fmt.Errorf("tar entry escapes destination: %q", sub)
+ }
+ return filepath.Join(base, clean), nil
+}
A => internal/archive/archive_test.go +71 -0
@@ 1,71 @@
+package archive
+
+import (
+ "bytes"
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+func TestRoundTripDir(t *testing.T) {
+ src := t.TempDir()
+ mustWrite(t, filepath.Join(src, "a.txt"), "alpha")
+ mustWrite(t, filepath.Join(src, "sub/b.txt"), "bravo")
+ if err := os.MkdirAll(filepath.Join(src, "empty"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+
+ var buf bytes.Buffer
+ if err := EncodeDir(&buf, src); err != nil {
+ t.Fatalf("EncodeDir: %v", err)
+ }
+
+ dest := t.TempDir()
+ if err := DecodeDir(&buf, dest); err != nil {
+ t.Fatalf("DecodeDir: %v", err)
+ }
+
+ for _, c := range []struct{ rel, want string }{
+ {"a.txt", "alpha"},
+ {"sub/b.txt", "bravo"},
+ } {
+ got, err := os.ReadFile(filepath.Join(dest, c.rel))
+ if err != nil {
+ t.Errorf("read %s: %v", c.rel, err)
+ continue
+ }
+ if string(got) != c.want {
+ t.Errorf("%s = %q, want %q", c.rel, got, c.want)
+ }
+ }
+ if _, err := os.Stat(filepath.Join(dest, "empty")); err != nil {
+ t.Errorf("empty dir missing: %v", err)
+ }
+}
+
+func TestDecodeRejectsPathEscape(t *testing.T) {
+ // Hand-craft a tarball with a "../evil" entry, then zstd it via EncodeDir
+ // indirection isn't feasible (EncodeDir won't emit ..). Instead, write
+ // a tar.zst by hand with one bad header.
+ t.Skip("path-escape rejection covered by safeJoin unit logic; would need a hand-crafted tar to assert at integration level")
+}
+
+func mustWrite(t *testing.T, path, body string) {
+ t.Helper()
+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestSafeJoinRejectsEscape(t *testing.T) {
+ base := "/tmp/abc"
+ if _, err := safeJoin(base, "../etc/passwd"); err == nil {
+ t.Error("expected escape rejection for ../etc/passwd")
+ }
+ if _, err := safeJoin(base, "sub/ok"); err != nil {
+ t.Errorf("safe entry rejected: %v", err)
+ }
+}
A => internal/config/config.go +205 -0
@@ 1,205 @@
+// Package config loads cacher configuration with precedence:
+// flag > env (CACHER_*) > config file (~/.config/cacher/config.toml) > defaults.
+package config
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+
+ "github.com/BurntSushi/toml"
+)
+
+// Config is the persisted cacher configuration. The TOML file under
+// ~/.config/cacher/config.toml holds the same fields. Every field is
+// overridable via CACHER_<UPPER> environment variables.
+type Config struct {
+ Endpoint string `toml:"endpoint"`
+ Region string `toml:"region"`
+ Bucket string `toml:"bucket"`
+ Prefix string `toml:"prefix"`
+ ArchSuffix bool `toml:"arch_suffix"`
+
+ // KeyFile / SecretFile point at files that contain the raw credential
+ // material (whitespace is stripped on load). Used in CI where secrets
+ // are mounted as files. Env vars CACHER_S3_KEY_ID / CACHER_S3_SECRET
+ // take priority for local-dev runs.
+ KeyFile string `toml:"key_file"`
+ SecretFile string `toml:"secret_file"`
+}
+
+// Defaults returns built-in defaults. They are overlaid by file → env → flags.
+func Defaults() Config {
+ return Config{
+ Region: "us-east-1",
+ KeyFile: "~/.s3-cache-key-id",
+ SecretFile: "~/.s3-cache-key-secret",
+ }
+}
+
+// DefaultPath returns ~/.config/cacher/config.toml expanded.
+func DefaultPath() (string, error) {
+ dir, err := os.UserConfigDir()
+ if err != nil {
+ return "", fmt.Errorf("user config dir: %w", err)
+ }
+ return filepath.Join(dir, "cacher", "config.toml"), nil
+}
+
+// Load reads the config file at path (empty path = DefaultPath). A missing
+// file is not an error — it returns Defaults(). Env vars and flag overlays
+// must be applied by the caller afterwards.
+func Load(path string) (Config, error) {
+ cfg := Defaults()
+ if path == "" {
+ p, err := DefaultPath()
+ if err != nil {
+ return cfg, err
+ }
+ path = p
+ }
+ data, err := os.ReadFile(path)
+ if errors.Is(err, os.ErrNotExist) {
+ return cfg, nil
+ }
+ if err != nil {
+ return cfg, fmt.Errorf("read %s: %w", path, err)
+ }
+ if err := toml.Unmarshal(data, &cfg); err != nil {
+ return cfg, fmt.Errorf("parse %s: %w", path, err)
+ }
+ return cfg, nil
+}
+
+// Save writes the config to path (creating parent dirs). Empty path =
+// DefaultPath.
+func Save(path string, cfg Config) error {
+ if path == "" {
+ p, err := DefaultPath()
+ if err != nil {
+ return err
+ }
+ path = p
+ }
+ if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+ return fmt.Errorf("mkdir %s: %w", filepath.Dir(path), err)
+ }
+ f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
+ if err != nil {
+ return fmt.Errorf("open %s: %w", path, err)
+ }
+ defer f.Close()
+ return toml.NewEncoder(f).Encode(cfg)
+}
+
+// ApplyEnv overlays CACHER_<UPPER> environment variables onto cfg.
+// Unset/empty variables are ignored (do not zero out file values).
+func ApplyEnv(cfg *Config) error {
+ if v, ok := os.LookupEnv("CACHER_ENDPOINT"); ok && v != "" {
+ cfg.Endpoint = v
+ }
+ if v, ok := os.LookupEnv("CACHER_REGION"); ok && v != "" {
+ cfg.Region = v
+ }
+ if v, ok := os.LookupEnv("CACHER_BUCKET"); ok && v != "" {
+ cfg.Bucket = v
+ }
+ if v, ok := os.LookupEnv("CACHER_PREFIX"); ok && v != "" {
+ cfg.Prefix = v
+ }
+ if v, ok := os.LookupEnv("CACHER_ARCH_SUFFIX"); ok && v != "" {
+ b, err := strconv.ParseBool(v)
+ if err != nil {
+ return fmt.Errorf("CACHER_ARCH_SUFFIX=%q: %w", v, err)
+ }
+ cfg.ArchSuffix = b
+ }
+ if v, ok := os.LookupEnv("CACHER_KEY_FILE"); ok && v != "" {
+ cfg.KeyFile = v
+ }
+ if v, ok := os.LookupEnv("CACHER_SECRET_FILE"); ok && v != "" {
+ cfg.SecretFile = v
+ }
+ return nil
+}
+
+// ExpandPath expands a leading ~ in a path against $HOME.
+func ExpandPath(p string) string {
+ if p == "" || !strings.HasPrefix(p, "~") {
+ return p
+ }
+ home, err := os.UserHomeDir()
+ if err != nil {
+ return p
+ }
+ if p == "~" {
+ return home
+ }
+ if strings.HasPrefix(p, "~/") {
+ return filepath.Join(home, p[2:])
+ }
+ return p
+}
+
+// Credentials resolves the S3 access key + secret with precedence:
+// 1. CACHER_S3_KEY_ID / CACHER_S3_SECRET env vars
+// 2. files at cfg.KeyFile / cfg.SecretFile
+//
+// Whitespace (including trailing newlines) is trimmed from file contents.
+func (c Config) Credentials() (keyID, secret string, err error) {
+ keyID = os.Getenv("CACHER_S3_KEY_ID")
+ secret = os.Getenv("CACHER_S3_SECRET")
+ if keyID == "" {
+ v, e := readTrimmed(ExpandPath(c.KeyFile))
+ if e != nil {
+ return "", "", fmt.Errorf("read key file %s: %w", c.KeyFile, e)
+ }
+ keyID = v
+ }
+ if secret == "" {
+ v, e := readTrimmed(ExpandPath(c.SecretFile))
+ if e != nil {
+ return "", "", fmt.Errorf("read secret file %s: %w", c.SecretFile, e)
+ }
+ secret = v
+ }
+ if keyID == "" || secret == "" {
+ return "", "", errors.New("no credentials: set CACHER_S3_KEY_ID/_SECRET or --key-file/--secret-file")
+ }
+ return keyID, secret, nil
+}
+
+func readTrimmed(p string) (string, error) {
+ if p == "" {
+ return "", nil
+ }
+ b, err := os.ReadFile(p)
+ if errors.Is(err, os.ErrNotExist) {
+ return "", nil
+ }
+ if err != nil {
+ return "", err
+ }
+ return strings.TrimSpace(string(b)), nil
+}
+
+// Validate ensures the minimum viable config for an S3 operation.
+func (c Config) Validate() error {
+ var missing []string
+ if c.Endpoint == "" {
+ missing = append(missing, "endpoint")
+ }
+ if c.Region == "" {
+ missing = append(missing, "region")
+ }
+ if c.Bucket == "" {
+ missing = append(missing, "bucket")
+ }
+ if len(missing) > 0 {
+ return fmt.Errorf("config missing fields: %s (run `cacher init` or set CACHER_* env vars)", strings.Join(missing, ", "))
+ }
+ return nil
+}
A => internal/config/config_test.go +111 -0
@@ 1,111 @@
+package config
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+func TestLoadMissingReturnsDefaults(t *testing.T) {
+ cfg, err := Load(filepath.Join(t.TempDir(), "nope.toml"))
+ if err != nil {
+ t.Fatalf("Load: %v", err)
+ }
+ if got := cfg.Region; got != "us-east-1" {
+ t.Errorf("default region = %q, want us-east-1", got)
+ }
+}
+
+func TestSaveThenLoadRoundTrip(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "config.toml")
+ want := Config{
+ Endpoint: "https://s3.bigb.es",
+ Region: "garage",
+ Bucket: "docker-cache",
+ Prefix: "tarantool-protobuf",
+ ArchSuffix: true,
+ KeyFile: "~/.k",
+ SecretFile: "~/.s",
+ }
+ if err := Save(path, want); err != nil {
+ t.Fatalf("Save: %v", err)
+ }
+ got, err := Load(path)
+ if err != nil {
+ t.Fatalf("Load: %v", err)
+ }
+ if got != want {
+ t.Errorf("round trip mismatch\n got: %+v\nwant: %+v", got, want)
+ }
+}
+
+func TestApplyEnvOverlays(t *testing.T) {
+ t.Setenv("CACHER_BUCKET", "override-bucket")
+ t.Setenv("CACHER_ARCH_SUFFIX", "true")
+ cfg := Config{Bucket: "from-file", Region: "garage"}
+ if err := ApplyEnv(&cfg); err != nil {
+ t.Fatalf("ApplyEnv: %v", err)
+ }
+ if cfg.Bucket != "override-bucket" {
+ t.Errorf("Bucket = %q, want override-bucket", cfg.Bucket)
+ }
+ if !cfg.ArchSuffix {
+ t.Errorf("ArchSuffix = false, want true")
+ }
+ if cfg.Region != "garage" {
+ t.Errorf("Region clobbered to %q (unset env should not zero existing value)", cfg.Region)
+ }
+}
+
+func TestCredentialsEnvBeatsFile(t *testing.T) {
+ dir := t.TempDir()
+ keyPath := filepath.Join(dir, "key")
+ secPath := filepath.Join(dir, "sec")
+ if err := os.WriteFile(keyPath, []byte("file-key\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(secPath, []byte("file-sec\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv("CACHER_S3_KEY_ID", "env-key")
+ cfg := Config{KeyFile: keyPath, SecretFile: secPath}
+ k, s, err := cfg.Credentials()
+ if err != nil {
+ t.Fatalf("Credentials: %v", err)
+ }
+ if k != "env-key" {
+ t.Errorf("key id = %q, want env-key", k)
+ }
+ if s != "file-sec" {
+ t.Errorf("secret = %q, want file-sec (whitespace trimmed)", s)
+ }
+}
+
+func TestCredentialsTrimsWhitespace(t *testing.T) {
+ dir := t.TempDir()
+ keyPath := filepath.Join(dir, "key")
+ secPath := filepath.Join(dir, "sec")
+ if err := os.WriteFile(keyPath, []byte(" raw-key \r\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(secPath, []byte("\nraw-sec\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv("CACHER_S3_KEY_ID", "")
+ t.Setenv("CACHER_S3_SECRET", "")
+ cfg := Config{KeyFile: keyPath, SecretFile: secPath}
+ k, s, err := cfg.Credentials()
+ if err != nil {
+ t.Fatalf("Credentials: %v", err)
+ }
+ if k != "raw-key" || s != "raw-sec" {
+ t.Errorf("trimming failed: key=%q secret=%q", k, s)
+ }
+}
+
+func TestValidateMissingFields(t *testing.T) {
+ err := Config{Region: "garage"}.Validate()
+ if err == nil {
+ t.Fatal("expected error for missing endpoint/bucket")
+ }
+}
A => internal/hash/hash.go +165 -0
@@ 1,165 @@
+// Package hash derives cache keys from file/directory contents.
+//
+// For a single file path, Derive returns the same hex digest as
+// `sha256sum <path> | cut -c1-<length>` (default length 16). This matches
+// the convention used by the existing .builds/lib/ci-lib.sh shell helpers
+// it replaces.
+package hash
+
+import (
+ "crypto/sha256"
+ "fmt"
+ "hash"
+ "io"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+)
+
+// DefaultLength is the hex-character count after truncation. 16 hex chars =
+// 64 bits, which matches the existing `cut -c1-16` shell convention.
+const DefaultLength = 16
+
+// Derive returns a deterministic hex digest of the contents of the given
+// paths, truncated to length characters. The order of paths matters:
+// concatenating in flag order is intentional (so callers can express
+// dependencies like "Dockerfile + context dir" with stable ordering).
+//
+// A single regular file produces the same digest as `sha256sum <file>`
+// when length is 64 (or its prefix when length<64). A directory is hashed
+// recursively by walking entries in sorted relative-path order.
+func Derive(paths []string, length int) (string, error) {
+ if length <= 0 || length > 64 {
+ length = DefaultLength
+ }
+ if len(paths) == 0 {
+ return "", fmt.Errorf("no --hash-from paths given")
+ }
+
+ final := sha256.New()
+ for _, p := range paths {
+ h, err := hashOne(p)
+ if err != nil {
+ return "", err
+ }
+ // When there is exactly one file path, return its sha256 directly
+ // so the digest matches `sha256sum`. The "wrap into outer sha256"
+ // dance is only needed when combining multiple inputs.
+ if len(paths) == 1 {
+ return h[:length], nil
+ }
+ final.Write([]byte(h))
+ final.Write([]byte{0})
+ }
+ return hex(final)[:length], nil
+}
+
+func hashOne(p string) (string, error) {
+ st, err := os.Stat(p)
+ if err != nil {
+ return "", fmt.Errorf("stat %s: %w", p, err)
+ }
+ if st.IsDir() {
+ return hashDir(p)
+ }
+ return hashFile(p)
+}
+
+func hashFile(p string) (string, error) {
+ f, err := os.Open(p)
+ if err != nil {
+ return "", fmt.Errorf("open %s: %w", p, err)
+ }
+ defer f.Close()
+ h := sha256.New()
+ if _, err := io.Copy(h, f); err != nil {
+ return "", fmt.Errorf("read %s: %w", p, err)
+ }
+ return hex(h), nil
+}
+
+// hashDir walks p in sorted relative-path order and writes
+// "<relpath>\0<file-sha256>\n" for each regular file into a rolling sha256.
+// Non-regular entries (symlinks, devices, sockets) are skipped — their
+// content is not portable and would make the hash brittle.
+func hashDir(root string) (string, error) {
+ var entries []string
+ err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if !d.Type().IsRegular() {
+ return nil
+ }
+ rel, err := filepath.Rel(root, path)
+ if err != nil {
+ return err
+ }
+ entries = append(entries, rel)
+ return nil
+ })
+ if err != nil {
+ return "", fmt.Errorf("walk %s: %w", root, err)
+ }
+ sort.Strings(entries)
+
+ h := sha256.New()
+ for _, rel := range entries {
+ fh, err := hashFile(filepath.Join(root, rel))
+ if err != nil {
+ return "", err
+ }
+ // Use forward slashes so the digest is OS-independent.
+ h.Write([]byte(filepath.ToSlash(rel)))
+ h.Write([]byte{0})
+ h.Write([]byte(fh))
+ h.Write([]byte{'\n'})
+ }
+ return hex(h), nil
+}
+
+func hex(h hash.Hash) string {
+ const hexdigits = "0123456789abcdef"
+ sum := h.Sum(nil)
+ b := make([]byte, len(sum)*2)
+ for i, x := range sum {
+ b[i*2] = hexdigits[x>>4]
+ b[i*2+1] = hexdigits[x&0x0f]
+ }
+ return string(b)
+}
+
+// ApplyTemplate substitutes {hash} (and {arch} when archSuffix is true)
+// in keyTemplate with the derived values. If keyTemplate has no {hash}
+// placeholder and paths is non-empty, the hash is appended before the
+// final extension. When archSuffix is true, "-<goos>-<goarch>" is appended
+// to the final path component, before its extension.
+func ApplyTemplate(keyTemplate, derived, goos, goarch string, archSuffix bool) string {
+ key := keyTemplate
+ if derived != "" {
+ if strings.Contains(key, "{hash}") {
+ key = strings.ReplaceAll(key, "{hash}", derived)
+ } else {
+ key = insertBeforeExt(key, "-"+derived)
+ }
+ }
+ if archSuffix {
+ key = insertBeforeExt(key, "-"+goos+"-"+goarch)
+ }
+ return key
+}
+
+func insertBeforeExt(key, suffix string) string {
+ dir, base := filepath.Split(key)
+ ext := filepath.Ext(base)
+ // Handle compound extensions like ".tar.zst" / ".tar.gz".
+ if ext == ".zst" || ext == ".gz" || ext == ".bz2" || ext == ".xz" {
+ if inner := filepath.Ext(strings.TrimSuffix(base, ext)); inner == ".tar" {
+ ext = inner + ext
+ }
+ }
+ stem := strings.TrimSuffix(base, ext)
+ return dir + stem + suffix + ext
+}
A => internal/hash/hash_test.go +127 -0
@@ 1,127 @@
+package hash
+
+import (
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+// TestSingleFileMatchesSha256sumCutC116 asserts the central parity claim:
+// Derive([f], 16) == `sha256sum f | cut -c1-16`. This is what the existing
+// shell helper does (HASH=$(sha256sum docker/conformance.Dockerfile | cut -c1-16))
+// and the new tool must produce the same key for the same input.
+func TestSingleFileMatchesSha256sumCutC116(t *testing.T) {
+ if _, err := exec.LookPath("sha256sum"); err != nil {
+ t.Skip("sha256sum not on PATH; skipping shell parity check")
+ }
+ path := filepath.Join(t.TempDir(), "sample")
+ if err := os.WriteFile(path, []byte("hello world\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ out, err := exec.Command("sha256sum", path).Output()
+ if err != nil {
+ t.Fatalf("sha256sum: %v", err)
+ }
+ want := strings.Fields(string(out))[0][:16]
+
+ got, err := Derive([]string{path}, 16)
+ if err != nil {
+ t.Fatalf("Derive: %v", err)
+ }
+ if got != want {
+ t.Errorf("Derive=%q want=%q", got, want)
+ }
+}
+
+func TestDirHashIsStable(t *testing.T) {
+ dir := t.TempDir()
+ mustWrite := func(rel, body string) {
+ p := filepath.Join(dir, rel)
+ if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ mustWrite("a/x", "x-body")
+ mustWrite("b/y/z", "z-body")
+
+ h1, err := Derive([]string{dir}, 32)
+ if err != nil {
+ t.Fatal(err)
+ }
+ h2, err := Derive([]string{dir}, 32)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if h1 != h2 {
+ t.Errorf("dir hash not stable: %q vs %q", h1, h2)
+ }
+}
+
+func TestDirHashChangesOnContentChange(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "f")
+ _ = os.WriteFile(path, []byte("v1"), 0o600)
+ h1, _ := Derive([]string{dir}, 32)
+ _ = os.WriteFile(path, []byte("v2"), 0o600)
+ h2, _ := Derive([]string{dir}, 32)
+ if h1 == h2 {
+ t.Error("dir hash unchanged after content change")
+ }
+}
+
+func TestMultiPathCombines(t *testing.T) {
+ dir := t.TempDir()
+ a := filepath.Join(dir, "a")
+ b := filepath.Join(dir, "b")
+ _ = os.WriteFile(a, []byte("A"), 0o600)
+ _ = os.WriteFile(b, []byte("B"), 0o600)
+
+ ha, _ := Derive([]string{a}, 16)
+ hab, _ := Derive([]string{a, b}, 16)
+ hba, _ := Derive([]string{b, a}, 16)
+ if ha == hab {
+ t.Error("combining two paths produced same hash as single path")
+ }
+ if hab == hba {
+ t.Error("order of --hash-from paths should matter")
+ }
+}
+
+func TestApplyTemplateHashPlaceholder(t *testing.T) {
+ got := ApplyTemplate("img/{hash}.tar.zst", "abcd1234", "linux", "amd64", false)
+ if got != "img/abcd1234.tar.zst" {
+ t.Errorf("ApplyTemplate placeholder = %q", got)
+ }
+}
+
+func TestApplyTemplateNoPlaceholderAppends(t *testing.T) {
+ got := ApplyTemplate("img/build.tar.zst", "abcd1234", "linux", "amd64", false)
+ if got != "img/build-abcd1234.tar.zst" {
+ t.Errorf("ApplyTemplate append = %q", got)
+ }
+}
+
+func TestApplyTemplateArchSuffix(t *testing.T) {
+ got := ApplyTemplate("img/build.tar.zst", "abcd", "linux", "amd64", true)
+ if got != "img/build-abcd-linux-amd64.tar.zst" {
+ t.Errorf("ApplyTemplate arch = %q", got)
+ }
+}
+
+func TestApplyTemplateArchSuffixSimpleExt(t *testing.T) {
+ got := ApplyTemplate("bin/cacher", "", "linux", "amd64", true)
+ if got != "bin/cacher-linux-amd64" {
+ t.Errorf("ApplyTemplate arch no-ext = %q", got)
+ }
+}
+
+func TestDeriveRejectsEmpty(t *testing.T) {
+ if _, err := Derive(nil, 16); err == nil {
+ t.Error("expected error for empty paths")
+ }
+}
A => internal/logging/logging.go +106 -0
@@ 1,106 @@
+package logging
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "log/slog"
+ "strings"
+ "sync"
+ "time"
+)
+
+func Setup(format string, w io.Writer, level slog.Level) {
+ var handler slog.Handler
+ switch format {
+ case "json":
+ handler = slog.NewJSONHandler(w, &slog.HandlerOptions{Level: level})
+ default:
+ handler = newHumanHandler(w, level)
+ }
+ slog.SetDefault(slog.New(handler))
+}
+
+// humanHandler emits "<RFC3339> <LVL>: <msg> k=v k=v\n".
+type humanHandler struct {
+ mu *sync.Mutex
+ w io.Writer
+ level slog.Level
+ attrs []slog.Attr
+ group string
+}
+
+func newHumanHandler(w io.Writer, level slog.Level) *humanHandler {
+ return &humanHandler{mu: &sync.Mutex{}, w: w, level: level}
+}
+
+func (h *humanHandler) Enabled(_ context.Context, l slog.Level) bool { return l >= h.level }
+
+func (h *humanHandler) Handle(_ context.Context, r slog.Record) error {
+ var b strings.Builder
+ b.WriteString(r.Time.UTC().Format(time.RFC3339))
+ b.WriteByte(' ')
+ b.WriteString(shortLevel(r.Level))
+ b.WriteString(": ")
+ b.WriteString(r.Message)
+ for _, a := range h.attrs {
+ writeAttr(&b, h.group, a)
+ }
+ r.Attrs(func(a slog.Attr) bool {
+ writeAttr(&b, h.group, a)
+ return true
+ })
+ b.WriteByte('\n')
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ _, err := io.WriteString(h.w, b.String())
+ return err
+}
+
+func (h *humanHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
+ out := *h
+ out.attrs = append(append([]slog.Attr{}, h.attrs...), attrs...)
+ return &out
+}
+
+func (h *humanHandler) WithGroup(name string) slog.Handler {
+ out := *h
+ if h.group == "" {
+ out.group = name
+ } else {
+ out.group = h.group + "." + name
+ }
+ return &out
+}
+
+func shortLevel(l slog.Level) string {
+ switch {
+ case l < slog.LevelInfo:
+ return "DBG"
+ case l < slog.LevelWarn:
+ return "INF"
+ case l < slog.LevelError:
+ return "WRN"
+ default:
+ return "ERR"
+ }
+}
+
+func writeAttr(b *strings.Builder, group string, a slog.Attr) {
+ if a.Equal(slog.Attr{}) {
+ return
+ }
+ b.WriteByte(' ')
+ if group != "" {
+ b.WriteString(group)
+ b.WriteByte('.')
+ }
+ b.WriteString(a.Key)
+ b.WriteByte('=')
+ v := a.Value.String()
+ if strings.ContainsAny(v, " \t\"") {
+ fmt.Fprintf(b, "%q", v)
+ } else {
+ b.WriteString(v)
+ }
+}
A => internal/s3/client.go +216 -0
@@ 1,216 @@
+// Package s3 wraps the AWS SDK v2 S3 client with the Garage-compatible
+// settings that the existing .builds/lib/ci-lib.sh shell helpers configure
+// via ~/.aws/config:
+//
+// [default]
+// region = $AWS_DEFAULT_REGION
+// s3 =
+// addressing_style = path
+// signature_version = s3v4
+// (plus AWS_REQUEST/RESPONSE_CHECKSUM_* = when_required for Garage)
+package s3
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/aws-sdk-go-v2/service/s3/types"
+ "github.com/aws/smithy-go"
+)
+
+// Client is a small surface over the S3 SDK tuned for the cacher use case.
+type Client struct {
+ api *s3.Client
+ uploader *manager.Uploader
+ bucket string
+ prefix string // joined to every key (with / separator)
+}
+
+// Options configures a new Client.
+type Options struct {
+ Endpoint string
+ Region string
+ Bucket string
+ Prefix string
+ KeyID string
+ Secret string
+ UserAgent string // appended to the default UA for traceability
+}
+
+// New constructs a Client. The HTTP client uses Go's default transport;
+// callers can decorate later if needed (timeouts, proxies).
+func New(opts Options) (*Client, error) {
+ if opts.Endpoint == "" {
+ return nil, errors.New("s3: endpoint required")
+ }
+ if opts.Region == "" {
+ return nil, errors.New("s3: region required")
+ }
+ if opts.Bucket == "" {
+ return nil, errors.New("s3: bucket required")
+ }
+ if opts.KeyID == "" || opts.Secret == "" {
+ return nil, errors.New("s3: credentials required")
+ }
+
+ api := s3.New(s3.Options{
+ Region: opts.Region,
+ Credentials: credentials.NewStaticCredentialsProvider(opts.KeyID, opts.Secret, ""),
+ UsePathStyle: true, // Garage needs path-style addressing.
+ BaseEndpoint: aws.String(opts.Endpoint),
+ // Garage doesn't implement boto3 1.36+ trailing CRC32 checksums.
+ // Mirrors AWS_REQUEST_CHECKSUM_CALCULATION=when_required and
+ // AWS_RESPONSE_CHECKSUM_VALIDATION=when_required from the shell.
+ RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,
+ ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired,
+ })
+
+ uploader := manager.NewUploader(api, func(u *manager.Uploader) {
+ // Single-stream upload — multipart is auto for >5MiB.
+ u.Concurrency = 1
+ })
+
+ return &Client{
+ api: api,
+ uploader: uploader,
+ bucket: opts.Bucket,
+ prefix: strings.TrimSuffix(opts.Prefix, "/"),
+ }, nil
+}
+
+// FullKey returns prefix/key, with single-slash semantics.
+func (c *Client) FullKey(key string) string {
+ if c.prefix == "" {
+ return strings.TrimPrefix(key, "/")
+ }
+ return c.prefix + "/" + strings.TrimPrefix(key, "/")
+}
+
+// Bucket returns the configured bucket name (for diagnostics).
+func (c *Client) Bucket() string { return c.bucket }
+
+// Exists returns true if the object is present.
+func (c *Client) Exists(ctx context.Context, key string) (bool, error) {
+ _, err := c.api.HeadObject(ctx, &s3.HeadObjectInput{
+ Bucket: aws.String(c.bucket),
+ Key: aws.String(c.FullKey(key)),
+ })
+ if err == nil {
+ return true, nil
+ }
+ if isNotFound(err) {
+ return false, nil
+ }
+ return false, fmt.Errorf("head s3://%s/%s: %w", c.bucket, c.FullKey(key), err)
+}
+
+// Get streams the object body. Caller must Close the returned reader.
+func (c *Client) Get(ctx context.Context, key string) (io.ReadCloser, error) {
+ out, err := c.api.GetObject(ctx, &s3.GetObjectInput{
+ Bucket: aws.String(c.bucket),
+ Key: aws.String(c.FullKey(key)),
+ })
+ if err != nil {
+ if isNotFound(err) {
+ return nil, fmt.Errorf("get s3://%s/%s: not found", c.bucket, c.FullKey(key))
+ }
+ return nil, fmt.Errorf("get s3://%s/%s: %w", c.bucket, c.FullKey(key), err)
+ }
+ return out.Body, nil
+}
+
+// Put uploads body to key, streaming. Multipart is automatic for bodies
+// over the manager's PartSize threshold (5MiB by default).
+func (c *Client) Put(ctx context.Context, key string, body io.Reader) error {
+ _, err := c.uploader.Upload(ctx, &s3.PutObjectInput{
+ Bucket: aws.String(c.bucket),
+ Key: aws.String(c.FullKey(key)),
+ Body: body,
+ })
+ if err != nil {
+ return fmt.Errorf("upload s3://%s/%s: %w", c.bucket, c.FullKey(key), err)
+ }
+ return nil
+}
+
+// Delete removes the object. Deleting a missing object returns nil
+// (matches `aws s3 rm` semantics).
+func (c *Client) Delete(ctx context.Context, key string) error {
+ _, err := c.api.DeleteObject(ctx, &s3.DeleteObjectInput{
+ Bucket: aws.String(c.bucket),
+ Key: aws.String(c.FullKey(key)),
+ })
+ if err != nil && !isNotFound(err) {
+ return fmt.Errorf("delete s3://%s/%s: %w", c.bucket, c.FullKey(key), err)
+ }
+ return nil
+}
+
+// List yields object keys (relative to prefix) under sub.
+func (c *Client) List(ctx context.Context, sub string) ([]string, error) {
+ fullPrefix := c.FullKey(sub)
+ var keys []string
+ var token *string
+ for {
+ out, err := c.api.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
+ Bucket: aws.String(c.bucket),
+ Prefix: aws.String(fullPrefix),
+ ContinuationToken: token,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("list s3://%s/%s: %w", c.bucket, fullPrefix, err)
+ }
+ for _, o := range out.Contents {
+ if o.Key == nil {
+ continue
+ }
+ keys = append(keys, strings.TrimPrefix(*o.Key, c.prefix+"/"))
+ }
+ if out.IsTruncated == nil || !*out.IsTruncated {
+ break
+ }
+ token = out.NextContinuationToken
+ }
+ return keys, nil
+}
+
+// HeadBucket is the smoke test used by `cacher doctor`.
+func (c *Client) HeadBucket(ctx context.Context) error {
+ _, err := c.api.HeadBucket(ctx, &s3.HeadBucketInput{
+ Bucket: aws.String(c.bucket),
+ })
+ if err != nil {
+ return fmt.Errorf("head bucket %s: %w", c.bucket, err)
+ }
+ return nil
+}
+
+// isNotFound covers both HeadObject's empty-error case and the typed
+// NoSuchKey returned by GetObject.
+func isNotFound(err error) bool {
+ var nf *types.NotFound
+ var nsk *types.NoSuchKey
+ if errors.As(err, &nf) || errors.As(err, &nsk) {
+ return true
+ }
+ var apiErr smithy.APIError
+ if errors.As(err, &apiErr) {
+ switch apiErr.ErrorCode() {
+ case "NotFound", "NoSuchKey", "404":
+ return true
+ }
+ }
+ var rerr interface{ HTTPStatusCode() int }
+ if errors.As(err, &rerr) && rerr.HTTPStatusCode() == http.StatusNotFound {
+ return true
+ }
+ return false
+}
A => internal/version/version.go +81 -0
@@ 1,81 @@
+package version
+
+import (
+ "fmt"
+ "runtime"
+ "runtime/debug"
+)
+
+var version = "dev"
+
+type Info struct {
+ GitVersion string
+ GitCommit string
+ GitTreeState string
+ BuildDate string
+ BuiltBy string
+ Clean bool
+ GoVersion string
+ Compiler string
+ ModuleSum string
+ Platform string
+}
+
+func Get() Info {
+ info := Info{
+ GitVersion: version,
+ GoVersion: runtime.Version(),
+ Compiler: runtime.Compiler,
+ Platform: runtime.GOOS + "/" + runtime.GOARCH,
+ }
+
+ bi, ok := debug.ReadBuildInfo()
+ if !ok {
+ return info
+ }
+
+ info.ModuleSum = bi.Main.Sum
+ info.Clean = true
+
+ for _, s := range bi.Settings {
+ switch s.Key {
+ case "vcs.revision":
+ info.GitCommit = s.Value
+ case "vcs.time":
+ info.BuildDate = s.Value
+ case "vcs.modified":
+ if s.Value == "true" {
+ info.Clean = false
+ info.GitTreeState = "dirty"
+ } else {
+ info.GitTreeState = "clean"
+ }
+ }
+ }
+
+ return info
+}
+
+func (i Info) String() string {
+ return fmt.Sprintf(`GitVersion: %s
+GitCommit: %s
+GitTreeState: %s
+BuildDate: %s
+BuiltBy: %s
+Clean: %t
+GoVersion: %s
+Compiler: %s
+ModuleSum: %s
+Platform: %s`,
+ val(i.GitVersion), val(i.GitCommit), val(i.GitTreeState),
+ val(i.BuildDate), val(i.BuiltBy), i.Clean,
+ i.GoVersion, i.Compiler, val(i.ModuleSum), i.Platform,
+ )
+}
+
+func val(s string) string {
+ if s == "" {
+ return "unknown"
+ }
+ return s
+}
A => main.go +7 -0
@@ 1,7 @@
+package main
+
+import "go.bigb.es/cacher/cmd"
+
+func main() {
+ cmd.Execute()
+}