From 891eddce3a799dfa25fdce5029ef8eb7114033de Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Mon, 25 May 2026 15:57:11 +0300 Subject: [PATCH] =?UTF-8?q?Initial=20cacher=20v0.0.1-dev=20=E2=80=94=20S3-?= =?UTF-8?q?backed=20CI=20cache=20helper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces tarantool-protobuf/.builds/lib/ci-lib.sh (and its two shell helpers s3_cache_or_curl + s3_cache_docker_image) with a single static Go binary. Commands: init / doctor persist config + smoke-test credentials download / upload single-file cache with URL fallback + sha256 exists / list / delete cache management key resolve key template (for shell scripting) docker {exists,download,upload} streamed save/load via zstd dir {download,upload} tar+zstd directory caching (new) Key derivation via --hash-from (repeatable; file or dir), matching the existing sha256sum | cut -c1-16 shell convention. Garage-compatible S3 client (path-style, checksums when_required). --- .gitignore | 18 ++ Justfile | 29 +++ VERSION | 1 + cmd/common.go | 127 +++++++++++++ cmd/completion.go | 57 ++++++ cmd/dir.go | 110 +++++++++++ cmd/docker.go | 201 +++++++++++++++++++++ cmd/doctor.go | 88 +++++++++ cmd/errors.go | 26 +++ cmd/file.go | 301 +++++++++++++++++++++++++++++++ cmd/init.go | 74 ++++++++ cmd/root.go | 51 ++++++ cmd/util.go | 20 ++ cmd/version.go | 32 ++++ go.mod | 27 +++ go.sum | 52 ++++++ internal/archive/archive.go | 178 ++++++++++++++++++ internal/archive/archive_test.go | 71 ++++++++ internal/config/config.go | 205 +++++++++++++++++++++ internal/config/config_test.go | 111 ++++++++++++ internal/hash/hash.go | 165 +++++++++++++++++ internal/hash/hash_test.go | 127 +++++++++++++ internal/logging/logging.go | 106 +++++++++++ internal/s3/client.go | 216 ++++++++++++++++++++++ internal/version/version.go | 81 +++++++++ main.go | 7 + 26 files changed, 2481 insertions(+) create mode 100644 .gitignore create mode 100644 Justfile create mode 100644 VERSION create mode 100644 cmd/common.go create mode 100644 cmd/completion.go create mode 100644 cmd/dir.go create mode 100644 cmd/docker.go create mode 100644 cmd/doctor.go create mode 100644 cmd/errors.go create mode 100644 cmd/file.go create mode 100644 cmd/init.go create mode 100644 cmd/root.go create mode 100644 cmd/util.go create mode 100644 cmd/version.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/archive/archive.go create mode 100644 internal/archive/archive_test.go create mode 100644 internal/config/config.go create mode 100644 internal/config/config_test.go create mode 100644 internal/hash/hash.go create mode 100644 internal/hash/hash_test.go create mode 100644 internal/logging/logging.go create mode 100644 internal/s3/client.go create mode 100644 internal/version/version.go create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..22785465487e9cd0be483a2bd2cec06403852594 --- /dev/null +++ b/.gitignore @@ -0,0 +1,18 @@ +# Built binaries +/cacher +/cacher-linux-amd64 + +# Local config / credentials that might end up in cwd +*.s3-cache-key-id +*.s3-cache-key-secret + +# IDE / editor +.idea/ +.vscode/ +*.swp +*~ + +# Go +*.test +*.prof +coverage.out diff --git a/Justfile b/Justfile new file mode 100644 index 0000000000000000000000000000000000000000..9e40e4d6898b26de66bf471782dd60ad958ce49b --- /dev/null +++ b/Justfile @@ -0,0 +1,29 @@ +binary_name := "cacher" +version := `git describe --long 2>/dev/null || cat VERSION 2>/dev/null || echo dev` + +default: + @just --list + +build: + go build -ldflags "-X go.bigb.es/cacher/internal/version.version={{version}}" -o {{binary_name}} . + +build-static: + CGO_ENABLED=0 go build -ldflags "-s -w -X go.bigb.es/cacher/internal/version.version={{version}}" -o {{binary_name}} . + +build-linux: + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-s -w -X go.bigb.es/cacher/internal/version.version={{version}}" -o {{binary_name}}-linux-amd64 . + +install: + go install -ldflags "-X go.bigb.es/cacher/internal/version.version={{version}}" . + +test: + go test ./... + +lint: + golangci-lint run ./... + +tidy: + go mod tidy + +clean: + rm -f {{binary_name}} {{binary_name}}-linux-amd64 diff --git a/VERSION b/VERSION new file mode 100644 index 0000000000000000000000000000000000000000..c0ab82c2ff703bab28ab9487de402d334f0753c5 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.0.1-dev diff --git a/cmd/common.go b/cmd/common.go new file mode 100644 index 0000000000000000000000000000000000000000..7a6f3d607d927cc8f6173e3d46e0c7b493ccdb53 --- /dev/null +++ b/cmd/common.go @@ -0,0 +1,127 @@ +package cmd + +import ( + "fmt" + "runtime" + + "github.com/spf13/cobra" + + "go.bigb.es/cacher/internal/config" + "go.bigb.es/cacher/internal/hash" + "go.bigb.es/cacher/internal/s3" +) + +// Flags overridable per command (flag > env > config > defaults). +var ( + flagEndpoint string + flagRegion string + flagBucket string + flagPrefix string + flagArchSuffix string // "true"/"false"/"" (unset) + flagKeyFile string + flagSecretFile string + + // Key derivation (attached by addKeyFlags). + flagHashFrom []string + flagHashLength int +) + +// addKeyFlags attaches --hash-from / --hash-length. Use on every command +// that takes a argument. +func addKeyFlags(c *cobra.Command) { + c.Flags().StringSliceVar(&flagHashFrom, "hash-from", nil, + "path(s) whose sha256 is substituted for {hash} in the key (repeatable; file or directory)") + c.Flags().IntVar(&flagHashLength, "hash-length", hash.DefaultLength, + "truncate the derived hash to N hex chars") +} + +// resolveKey applies --hash-from and --arch-suffix to the user-supplied +// key template. With no --hash-from, the template is returned verbatim +// (plus arch suffix if enabled). +func resolveKey(keyTemplate string, cfg config.Config) (string, error) { + var derived string + if len(flagHashFrom) > 0 { + var err error + derived, err = hash.Derive(flagHashFrom, flagHashLength) + if err != nil { + return "", err + } + } + return hash.ApplyTemplate(keyTemplate, derived, runtime.GOOS, runtime.GOARCH, cfg.ArchSuffix), nil +} + +// addS3Flags attaches the standard S3 override flags to a command. Most +// commands accept them so single-shot invocations can override config. +func addS3Flags(c *cobra.Command) { + c.Flags().StringVar(&flagEndpoint, "endpoint", "", "S3 endpoint URL (overrides config)") + c.Flags().StringVar(&flagRegion, "region", "", "S3 region (overrides config)") + c.Flags().StringVar(&flagBucket, "bucket", "", "S3 bucket (overrides config)") + c.Flags().StringVar(&flagPrefix, "prefix", "", "S3 key prefix (overrides config)") + c.Flags().StringVar(&flagArchSuffix, "arch-suffix", "", "append -- to key (true|false; default from config)") + c.Flags().StringVar(&flagKeyFile, "key-file", "", "path to S3 access key id file (overrides config)") + c.Flags().StringVar(&flagSecretFile, "secret-file", "", "path to S3 secret file (overrides config)") +} + +// loadConfig merges file → env → flag overrides. +func loadConfig() (config.Config, error) { + cfg, err := config.Load(flagConfigPath) + if err != nil { + return cfg, err + } + if err := config.ApplyEnv(&cfg); err != nil { + return cfg, err + } + if flagEndpoint != "" { + cfg.Endpoint = flagEndpoint + } + if flagRegion != "" { + cfg.Region = flagRegion + } + if flagBucket != "" { + cfg.Bucket = flagBucket + } + if flagPrefix != "" { + cfg.Prefix = flagPrefix + } + if flagKeyFile != "" { + cfg.KeyFile = flagKeyFile + } + if flagSecretFile != "" { + cfg.SecretFile = flagSecretFile + } + switch flagArchSuffix { + case "true": + cfg.ArchSuffix = true + case "false": + cfg.ArchSuffix = false + case "": + // inherit from file/env + default: + return cfg, fmt.Errorf("--arch-suffix must be true|false, got %q", flagArchSuffix) + } + return cfg, nil +} + +// client builds a Garage-tuned S3 client from the resolved config. +func client() (*s3.Client, config.Config, error) { + cfg, err := loadConfig() + if err != nil { + return nil, cfg, err + } + if err := cfg.Validate(); err != nil { + return nil, cfg, err + } + key, secret, err := cfg.Credentials() + if err != nil { + return nil, cfg, err + } + c, err := s3.New(s3.Options{ + Endpoint: cfg.Endpoint, + Region: cfg.Region, + Bucket: cfg.Bucket, + Prefix: cfg.Prefix, + KeyID: key, + Secret: secret, + }) + return c, cfg, err +} diff --git a/cmd/completion.go b/cmd/completion.go new file mode 100644 index 0000000000000000000000000000000000000000..285851bce55cedbcb2f37012ca335eb105a44fa3 --- /dev/null +++ b/cmd/completion.go @@ -0,0 +1,57 @@ +package cmd + +import ( + "os" + + "github.com/spf13/cobra" +) + +var completionCmd = &cobra.Command{ + Use: "completion [bash|zsh|fish|powershell]", + Short: "Generate shell completion scripts", + Long: `Generate shell completion scripts for cacher. + +To load completions: + +Bash: + $ source <(cacher completion bash) + # To load completions for each session, execute once: + # Linux: + $ cacher completion bash > /etc/bash_completion.d/cacher + # macOS: + $ cacher completion bash > $(brew --prefix)/etc/bash_completion.d/cacher + +Zsh: + $ source <(cacher completion zsh) + # To load completions for each session, execute once: + $ cacher completion zsh > "${fpath[1]}/_cacher" + +Fish: + $ cacher completion fish | source + # To load completions for each session, execute once: + $ cacher completion fish > ~/.config/fish/completions/cacher.fish + +PowerShell: + PS> cacher completion powershell | Out-String | Invoke-Expression +`, + DisableFlagsInUseLine: true, + ValidArgs: []string{"bash", "zsh", "fish", "powershell"}, + Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs), + RunE: func(cmd *cobra.Command, args []string) error { + switch args[0] { + case "bash": + return rootCmd.GenBashCompletion(os.Stdout) + case "zsh": + return rootCmd.GenZshCompletion(os.Stdout) + case "fish": + return rootCmd.GenFishCompletion(os.Stdout, true) + case "powershell": + return rootCmd.GenPowerShellCompletionWithDesc(os.Stdout) + } + return nil + }, +} + +func init() { + rootCmd.AddCommand(completionCmd) +} diff --git a/cmd/dir.go b/cmd/dir.go new file mode 100644 index 0000000000000000000000000000000000000000..825d4b19a9fecdb42342937c37d523d44318c6de --- /dev/null +++ b/cmd/dir.go @@ -0,0 +1,110 @@ +package cmd + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/spf13/cobra" + + "go.bigb.es/cacher/internal/archive" +) + +var dirCmd = &cobra.Command{ + Use: "dir", + Short: "Directory cache (tar+zstd of a tree)", + Long: `Cache resolved trees keyed by content hash (e.g. ~/go/pkg/mod +keyed by go.sum, or .rocks/ keyed by rockspec hash). Closes the biggest +gap left by the single-file shell helper this binary replaces.`, +} + +var dirDownloadCmd = &cobra.Command{ + Use: "download ", + Short: "Extract a cached directory into ", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + cli, cfg, err := client() + if err != nil { + return err + } + key, err := resolveKey(args[0], cfg) + if err != nil { + return err + } + dest := args[1] + ctx := context.Background() + + ok, err := cli.Exists(ctx, key) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("%w: %s", ErrNotFound, key) + } + if err := os.MkdirAll(dest, 0o755); err != nil { + return err + } + fmt.Fprintf(cmd.ErrOrStderr(), "Cache HIT — %s → extract %s\n", key, dest) + body, err := cli.Get(ctx, key) + if err != nil { + return err + } + defer body.Close() + return archive.DecodeDir(body, dest) + }, +} + +var dirUploadCmd = &cobra.Command{ + Use: "upload ", + Short: "Pack a local directory (tar+zstd) and upload", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + cli, cfg, err := client() + if err != nil { + return err + } + key, err := resolveKey(args[0], cfg) + if err != nil { + return err + } + src := args[1] + ctx := context.Background() + + if !flagForce { + ok, err := cli.Exists(ctx, key) + if err != nil { + return err + } + if ok { + fmt.Fprintf(cmd.ErrOrStderr(), "Skipped — %s already present (use --force)\n", key) + return nil + } + } + st, err := os.Stat(src) + if err != nil { + return err + } + if !st.IsDir() { + return fmt.Errorf("%s is not a directory", src) + } + fmt.Fprintf(cmd.ErrOrStderr(), "Packing %s → %s\n", src, key) + + pr, pw := io.Pipe() + go func() { + err := archive.EncodeDir(pw, src) + pw.CloseWithError(err) + }() + return cli.Put(ctx, key, pr) + }, +} + +func init() { + for _, c := range []*cobra.Command{dirDownloadCmd, dirUploadCmd} { + addS3Flags(c) + addKeyFlags(c) + } + dirUploadCmd.Flags().BoolVar(&flagForce, "force", false, "overwrite if key already exists") + dirCmd.AddCommand(dirDownloadCmd, dirUploadCmd) + rootCmd.AddCommand(dirCmd) +} diff --git a/cmd/docker.go b/cmd/docker.go new file mode 100644 index 0000000000000000000000000000000000000000..1efae1306ead0d967f6ff98816add5cc29e1dd49 --- /dev/null +++ b/cmd/docker.go @@ -0,0 +1,201 @@ +package cmd + +import ( + "context" + "fmt" + "io" + "os/exec" + + "github.com/klauspost/compress/zstd" + "github.com/spf13/cobra" +) + +var dockerCmd = &cobra.Command{ + Use: "docker", + Short: "Docker image cache (tar+zstd streamed to S3)", +} + +var dockerExistsCmd = &cobra.Command{ + Use: "exists ", + Short: "Exit 0 if a cached docker image is present at key, 1 if missing", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cli, cfg, err := client() + if err != nil { + return err + } + key, err := resolveKey(args[0], cfg) + if err != nil { + return err + } + ok, err := cli.Exists(context.Background(), key) + if err != nil { + return err + } + fmt.Fprintln(cmd.OutOrStdout(), key) + if !ok { + return ErrNotFound + } + return nil + }, +} + +var dockerDownloadCmd = &cobra.Command{ + Use: "download ", + Short: "Pull a cached image from S3 into the local docker daemon", + Long: `Streams s3://bucket/key through zstd-decode into ` + "`docker load`" + `.`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + cli, cfg, err := client() + if err != nil { + return err + } + key, err := resolveKey(args[0], cfg) + if err != nil { + return err + } + // The arg is for the caller's own ergonomics + logging; + // docker load reads the tag from the tar stream itself. + tag := args[1] + ctx := context.Background() + ok, err := cli.Exists(ctx, key) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("%w: %s", ErrNotFound, key) + } + fmt.Fprintf(cmd.ErrOrStderr(), "Cache HIT — %s → load %s\n", key, tag) + + body, err := cli.Get(ctx, key) + if err != nil { + return err + } + defer body.Close() + dec, err := zstd.NewReader(body) + if err != nil { + return err + } + defer dec.Close() + return runDockerLoad(ctx, dec) + }, +} + +var dockerUploadCmd = &cobra.Command{ + Use: "upload ", + Short: "Save a local docker image to the S3 cache", + Long: `Pipes ` + "`docker save image:tag`" + ` through zstd-encode into a streamed +S3 multipart upload. No on-disk tempfile.`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + cli, cfg, err := client() + if err != nil { + return err + } + key, err := resolveKey(args[0], cfg) + if err != nil { + return err + } + tag := args[1] + ctx := context.Background() + + if !flagForce { + ok, err := cli.Exists(ctx, key) + if err != nil { + return err + } + if ok { + fmt.Fprintf(cmd.ErrOrStderr(), "Skipped — %s already present (use --force)\n", key) + return nil + } + } + fmt.Fprintf(cmd.ErrOrStderr(), "Saving %s → %s\n", tag, key) + return saveTagToS3(ctx, cli, key, tag) + }, +} + +func init() { + for _, c := range []*cobra.Command{dockerExistsCmd, dockerDownloadCmd, dockerUploadCmd} { + addS3Flags(c) + addKeyFlags(c) + } + dockerUploadCmd.Flags().BoolVar(&flagForce, "force", false, "overwrite if key already exists") + dockerCmd.AddCommand(dockerExistsCmd, dockerDownloadCmd, dockerUploadCmd) + rootCmd.AddCommand(dockerCmd) +} + +// saveTagToS3 wires `docker save` stdout → zstd encoder → io.Pipe → +// s3.Put. The encoder runs in a goroutine writing into the pipe; the +// uploader goroutine reads from the pipe. Both ends signal errors back +// through pw.CloseWithError so the upload sees a clean EOF or a wrapped +// failure. +func saveTagToS3(ctx context.Context, cli s3Putter, key, tag string) error { + save := exec.CommandContext(ctx, "docker", "save", tag) + saveOut, err := save.StdoutPipe() + if err != nil { + return err + } + save.Stderr = newWarnWriter("docker save") + if err := save.Start(); err != nil { + return fmt.Errorf("docker save: %w", err) + } + + pr, pw := io.Pipe() + go func() { + zw, zerr := zstd.NewWriter(pw, zstd.WithEncoderLevel(zstd.SpeedDefault)) + if zerr != nil { + pw.CloseWithError(zerr) + return + } + if _, cerr := io.Copy(zw, saveOut); cerr != nil { + zw.Close() + pw.CloseWithError(cerr) + return + } + if zerr := zw.Close(); zerr != nil { + pw.CloseWithError(zerr) + return + } + pw.Close() + }() + + upErr := cli.Put(ctx, key, pr) + waitErr := save.Wait() + if upErr != nil { + return upErr + } + if waitErr != nil { + return fmt.Errorf("docker save: %w", waitErr) + } + return nil +} + +// runDockerLoad pipes r (already zstd-decoded) into `docker load`. +func runDockerLoad(ctx context.Context, r io.Reader) error { + load := exec.CommandContext(ctx, "docker", "load") + stdin, err := load.StdinPipe() + if err != nil { + return err + } + load.Stdout = newWarnWriter("docker load") + load.Stderr = newWarnWriter("docker load") + if err := load.Start(); err != nil { + return fmt.Errorf("docker load: %w", err) + } + _, copyErr := io.Copy(stdin, r) + stdin.Close() + waitErr := load.Wait() + if copyErr != nil { + return fmt.Errorf("docker load write: %w", copyErr) + } + if waitErr != nil { + return fmt.Errorf("docker load: %w", waitErr) + } + return nil +} + +// s3Putter is satisfied by *s3.Client; declared here so saveTagToS3 is +// testable with a fake. +type s3Putter interface { + Put(ctx context.Context, key string, r io.Reader) error +} diff --git a/cmd/doctor.go b/cmd/doctor.go new file mode 100644 index 0000000000000000000000000000000000000000..7c2a18bb9ff21d275c6c7357b7fe88e9f915f4b8 --- /dev/null +++ b/cmd/doctor.go @@ -0,0 +1,88 @@ +package cmd + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "io" + + "github.com/spf13/cobra" +) + +var doctorCmd = &cobra.Command{ + Use: "doctor", + Short: "Smoke-test S3 credentials and bucket reachability", + Long: `Validates that cacher can reach the configured S3 bucket using the +resolved credentials. Performs a HEAD on the bucket and then writes, +reads, and deletes a 1-byte test object. Reports the access key id's +length and first two characters as a diagnostic — mirrors the existing +ci_aws_setup shell helper's "key_id_len=… key_id_prefix=…" output — +without leaking the secret.`, + RunE: func(cmd *cobra.Command, _ []string) error { return runDoctor(cmd) }, +} + +func init() { + addS3Flags(doctorCmd) + rootCmd.AddCommand(doctorCmd) +} + +func runDoctor(cmd *cobra.Command) error { + cli, cfg, err := client() + if err != nil { + return err + } + out := cmd.OutOrStdout() + + keyID, secret, _ := cfg.Credentials() // already validated by client() + fmt.Fprintf(out, "endpoint=%s region=%s bucket=%s prefix=%s\n", cfg.Endpoint, cfg.Region, cfg.Bucket, cfg.Prefix) + fmt.Fprintf(out, "key_id_len=%d key_id_prefix=%s secret_len=%d\n", + len(keyID), safePrefix(keyID, 2), len(secret)) + + ctx := context.Background() + if err := cli.HeadBucket(ctx); err != nil { + return err + } + fmt.Fprintln(out, "head bucket: OK") + + // Write/read/delete a tiny canary so credential perms are exercised + // for all three operations, not just Head. Key is random so concurrent + // doctor runs don't fight. + r := make([]byte, 8) + _, _ = rand.Read(r) + key := ".cacher-doctor-" + hex.EncodeToString(r) + body := []byte("ok") + if err := cli.Put(ctx, key, bytes.NewReader(body)); err != nil { + return err + } + fmt.Fprintln(out, "put canary: OK") + + rc, err := cli.Get(ctx, key) + if err != nil { + return err + } + got, err := io.ReadAll(rc) + rc.Close() + if err != nil { + return err + } + if !bytes.Equal(got, body) { + return fmt.Errorf("canary read mismatch: got %q want %q", got, body) + } + fmt.Fprintln(out, "get canary: OK") + + if err := cli.Delete(ctx, key); err != nil { + return err + } + fmt.Fprintln(out, "delete canary: OK") + fmt.Fprintln(out, "doctor: all checks passed") + return nil +} + +func safePrefix(s string, n int) string { + if len(s) < n { + return s + } + return s[:n] +} diff --git a/cmd/errors.go b/cmd/errors.go new file mode 100644 index 0000000000000000000000000000000000000000..770497893b065d285241c8e313df5ad903a8275a --- /dev/null +++ b/cmd/errors.go @@ -0,0 +1,26 @@ +package cmd + +import "errors" + +// Sentinel errors recognized by Execute → exit code mapping. +// +// exists → ErrNotFound → 1 +// download without --url, key missing → ErrMissNoFallback → 3 +// any other error → 2 +var ( + ErrNotFound = errors.New("cacher: key not found") + ErrMissNoFallback = errors.New("cacher: cache miss and no --url fallback") +) + +func exitCodeFor(err error) int { + switch { + case err == nil: + return 0 + case errors.Is(err, ErrNotFound): + return 1 + case errors.Is(err, ErrMissNoFallback): + return 3 + default: + return 2 + } +} diff --git a/cmd/file.go b/cmd/file.go new file mode 100644 index 0000000000000000000000000000000000000000..0ed7c2a354f37da60141fa389cc26b90f8b51e57 --- /dev/null +++ b/cmd/file.go @@ -0,0 +1,301 @@ +package cmd + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "strings" + + "github.com/spf13/cobra" +) + +var ( + flagDownloadURL string + flagDownloadSHA256 string + flagForce bool +) + +var downloadCmd = &cobra.Command{ + Use: "download ", + Short: "Download a cached file; fall back to --url on cache miss", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + cli, cfg, err := client() + if err != nil { + return err + } + key, err := resolveKey(args[0], cfg) + if err != nil { + return err + } + dest := args[1] + + ctx := context.Background() + exists, err := cli.Exists(ctx, key) + if err != nil { + return err + } + if exists { + fmt.Fprintf(cmd.ErrOrStderr(), "Cache HIT — %s\n", key) + rc, err := cli.Get(ctx, key) + if err != nil { + return err + } + defer rc.Close() + return writeWithSHA(dest, rc, flagDownloadSHA256) + } + + fmt.Fprintf(cmd.ErrOrStderr(), "Cache MISS — %s\n", key) + if flagDownloadURL == "" { + return fmt.Errorf("%w: %s", ErrMissNoFallback, key) + } + fmt.Fprintf(cmd.ErrOrStderr(), "Fetching %s\n", flagDownloadURL) + body, err := httpGet(flagDownloadURL) + if err != nil { + return err + } + defer body.Close() + // Tee to a tempfile so we can verify the checksum before re-reading + // for the upload step. Pure-streaming up to s3 + sha256 verify is + // possible but complicates rewind on mismatch — tempfile keeps it + // honest at the cost of one extra disk pass. + tmp, err := os.CreateTemp("", "cacher-dl-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + defer tmp.Close() + if _, err := io.Copy(tmp, body); err != nil { + return err + } + if _, err := tmp.Seek(0, io.SeekStart); err != nil { + return err + } + if err := writeWithSHA(dest, tmp, flagDownloadSHA256); err != nil { + return err + } + // Re-open the verified destination file for upload to S3 so callers + // see the cached file on next run. + f, err := os.Open(dest) + if err != nil { + return err + } + defer f.Close() + if err := cli.Put(ctx, key, f); err != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "warn: upload to S3 failed: %v\n", err) + // Local download succeeded — don't fail the build just because + // of an upload glitch. + return nil + } + fmt.Fprintf(cmd.ErrOrStderr(), "Cached → %s\n", key) + return nil + }, +} + +var uploadCmd = &cobra.Command{ + Use: "upload ", + Short: "Upload a local file to the cache", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + cli, cfg, err := client() + if err != nil { + return err + } + key, err := resolveKey(args[0], cfg) + if err != nil { + return err + } + ctx := context.Background() + if !flagForce { + exists, err := cli.Exists(ctx, key) + if err != nil { + return err + } + if exists { + fmt.Fprintf(cmd.ErrOrStderr(), "Skipped — %s already present (use --force to overwrite)\n", key) + return nil + } + } + f, err := os.Open(args[1]) + if err != nil { + return err + } + defer f.Close() + if err := cli.Put(ctx, key, f); err != nil { + return err + } + fmt.Fprintf(cmd.ErrOrStderr(), "Uploaded → %s\n", key) + return nil + }, +} + +var existsCmd = &cobra.Command{ + Use: "exists ", + Short: "Exit 0 if key exists, 1 if missing, 2 on error", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cli, cfg, err := client() + if err != nil { + return err + } + key, err := resolveKey(args[0], cfg) + if err != nil { + return err + } + ok, err := cli.Exists(context.Background(), key) + if err != nil { + return err + } + fmt.Fprintln(cmd.OutOrStdout(), key) + if !ok { + return ErrNotFound + } + return nil + }, +} + +var listCmd = &cobra.Command{ + Use: "list [sub-prefix]", + Short: "List cache keys under prefix (optionally narrowed by sub-prefix)", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cli, _, err := client() + if err != nil { + return err + } + sub := "" + if len(args) == 1 { + sub = args[0] + } + keys, err := cli.List(context.Background(), sub) + if err != nil { + return err + } + out := cmd.OutOrStdout() + for _, k := range keys { + fmt.Fprintln(out, k) + } + return nil + }, +} + +var deleteCmd = &cobra.Command{ + Use: "delete ", + Short: "Delete a cache key (no error if missing)", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cli, cfg, err := client() + if err != nil { + return err + } + key, err := resolveKey(args[0], cfg) + if err != nil { + return err + } + if err := cli.Delete(context.Background(), key); err != nil { + return err + } + fmt.Fprintf(cmd.ErrOrStderr(), "Deleted — %s\n", key) + return nil + }, +} + +var keyCmd = &cobra.Command{ + Use: "key