From aff7cc68c59442b602cb5daed2e4135ed93f33e8 Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Fri, 7 Aug 2026 00:50:19 +0300 Subject: [PATCH] cmd: add --exec fallback and --optional to the download commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --exec generalises the existing --url (file) and --pull (docker) fallbacks to anything expressible as a shell command: on a cache miss the script runs through sh -c and whatever it produced seeds the cache. That collapses the restore-or-build if/fi block CI manifests repeat around every cache into a single invocation. The destination directory is created before the script runs, so the script needs no mkdir -p of its own. The seed upload stays best-effort (the content is already on disk), but a failing script is fatal and propagates its own exit status rather than a generic 2. For docker the tag is verified with docker image inspect afterwards — with a best-effort upload, a script exiting 0 without building the image would otherwise pass silently and fail a later task. --optional turns a cache miss into exit 0 for download and dir download, so set -e manifests drop the trailing || echo "cache miss". --- cmd/dir.go | 101 +++++++++++++++++++++++++++++++++++------------- cmd/docker.go | 43 ++++++++++++++++++--- cmd/errors.go | 10 ++++- cmd/exec.go | 65 +++++++++++++++++++++++++++++++ cmd/file.go | 73 +++++++++++++++++++++++++++++++++-- e2e_test.go | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 358 insertions(+), 38 deletions(-) create mode 100644 cmd/exec.go diff --git a/cmd/dir.go b/cmd/dir.go index 825d4b19a9fecdb42342937c37d523d44318c6de..643c7b326d7d5e296d55c20c45862b6769b05fe2 100644 --- a/cmd/dir.go +++ b/cmd/dir.go @@ -22,7 +22,18 @@ gap left by the single-file shell helper this binary replaces.`, var dirDownloadCmd = &cobra.Command{ Use: "download ", Short: "Extract a cached directory into ", - Args: cobra.ExactArgs(2), + Long: `Streams s3://bucket/key through zstd-decode into . + +With --exec, a cache miss falls back to running the given script with +` + "`sh -c`" + ` and then seeds the cache from . That collapses the +restore-or-build if/fi block CI manifests otherwise repeat per cache: + + cacher dir download "$KEY" ~/scss --exec 'git clone … && cp -r …' + + is created before the script runs, so the script can write +straight into it. With --optional, a miss exits 0 instead of 1 — for +caches whose absence just means a cold build.`, + Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { cli, cfg, err := client() if err != nil { @@ -39,19 +50,45 @@ var dirDownloadCmd = &cobra.Command{ if err != nil { return err } - if !ok { - return fmt.Errorf("%w: %s", ErrNotFound, key) - } - if err := os.MkdirAll(dest, 0o755); err != nil { - return err + if ok { + if err := os.MkdirAll(dest, 0o755); err != nil { + return err + } + fmt.Fprintf(cmd.ErrOrStderr(), "Cache HIT — %s → extract %s\n", key, dest) + body, err := cli.Get(ctx, key) + if err != nil { + return err + } + defer body.Close() + return archive.DecodeDir(body, dest) } - fmt.Fprintf(cmd.ErrOrStderr(), "Cache HIT — %s → extract %s\n", key, dest) - body, err := cli.Get(ctx, key) - if err != nil { - return err + + fmt.Fprintf(cmd.ErrOrStderr(), "Cache MISS — %s\n", key) + switch { + case flagExec != "": + // Created up front so the script can `cp` into it without + // repeating a mkdir -p of its own. + if err := os.MkdirAll(dest, 0o755); err != nil { + return err + } + if err := runExecFallback(ctx, cmd, flagExec); err != nil { + return err + } + // Seed the cache so the next run hits. Best-effort: the + // script already produced the tree locally, and an S3 + // hiccup is no reason to fail a build that has its + // content. Same trade-off as `docker download --pull`. + if err := uploadDir(ctx, cli, key, dest, cmd.ErrOrStderr()); err != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "warn: seed upload failed: %v\n", err) + return nil + } + fmt.Fprintf(cmd.ErrOrStderr(), "Cached → %s\n", key) + return nil + case flagOptional: + return nil + default: + return fmt.Errorf("%w: %s", ErrNotFound, key) } - defer body.Close() - return archive.DecodeDir(body, dest) }, } @@ -81,30 +118,40 @@ var dirUploadCmd = &cobra.Command{ return nil } } - st, err := os.Stat(src) - if err != nil { - return err - } - if !st.IsDir() { - return fmt.Errorf("%s is not a directory", src) - } - fmt.Fprintf(cmd.ErrOrStderr(), "Packing %s → %s\n", src, key) - - pr, pw := io.Pipe() - go func() { - err := archive.EncodeDir(pw, src) - pw.CloseWithError(err) - }() - return cli.Put(ctx, key, pr) + return uploadDir(ctx, cli, key, src, cmd.ErrOrStderr()) }, } +// uploadDir packs src (tar+zstd) straight into an S3 upload — the +// encoder writes into one end of a pipe while the uploader reads the +// other, so nothing lands on disk. Shared by `dir upload` and the +// --exec seed path of `dir download`. +func uploadDir(ctx context.Context, cli s3Putter, key, src string, w io.Writer) error { + st, err := os.Stat(src) + if err != nil { + return err + } + if !st.IsDir() { + return fmt.Errorf("%s is not a directory", src) + } + fmt.Fprintf(w, "Packing %s → %s\n", src, key) + pr, pw := io.Pipe() + go func() { + err := archive.EncodeDir(pw, src) + pw.CloseWithError(err) + }() + return cli.Put(ctx, key, pr) +} + func init() { for _, c := range []*cobra.Command{dirDownloadCmd, dirUploadCmd} { addS3Flags(c) addKeyFlags(c) } dirUploadCmd.Flags().BoolVar(&flagForce, "force", false, "overwrite if key already exists") + addExecFlag(dirDownloadCmd, "") + addOptionalFlag(dirDownloadCmd) + dirDownloadCmd.MarkFlagsMutuallyExclusive("exec", "optional") dirCmd.AddCommand(dirDownloadCmd, dirUploadCmd) rootCmd.AddCommand(dirCmd) } diff --git a/cmd/docker.go b/cmd/docker.go index 0ef88ce066a9214eacf105a32e6b3249b3c54d3a..1fc60a70bd584620bafeb5ab6d798d31aac18369 100644 --- a/cmd/docker.go +++ b/cmd/docker.go @@ -51,7 +51,13 @@ With --pull, a cache miss falls back to ` + "`docker pull `" + ` then seeds the S3 cache via ` + "`docker save`" + `. This makes a single invocation into the full cache-or-fetch pattern callable from a CI manifest: - cacher docker download garage/v2.3.0.tar.zst dxflrs/garage:v2.3.0 --pull`, + cacher docker download garage/v2.3.0.tar.zst dxflrs/garage:v2.3.0 --pull + +--exec is the same shape for images you build rather than pull: the +script runs with ` + "`sh -c`" + ` and must leave in the local daemon, +which is then saved to the cache. + + cacher docker download "$KEY" myimage:latest --exec 'docker build -t myimage:latest .'`, Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { cli, cfg, err := client() @@ -85,13 +91,25 @@ into the full cache-or-fetch pattern callable from a CI manifest: return runDockerLoad(ctx, dec) } fmt.Fprintf(cmd.ErrOrStderr(), "Cache MISS — %s\n", key) - if !flagDockerPull { + switch { + case flagDockerPull: + fmt.Fprintf(cmd.ErrOrStderr(), "Pulling %s\n", tag) + if err := runDockerPull(ctx, tag); err != nil { + return err + } + case flagExec != "": + if err := runExecFallback(ctx, cmd, flagExec); err != nil { + return err + } + // The seed upload below is best-effort, so an --exec script + // that exits 0 without producing the tag would otherwise + // pass silently and blow up in a later task instead. + if err := dockerImagePresent(ctx, tag); err != nil { + return err + } + default: return fmt.Errorf("%w: %s", ErrNotFound, key) } - fmt.Fprintf(cmd.ErrOrStderr(), "Pulling %s\n", tag) - if err := runDockerPull(ctx, tag); err != nil { - return err - } // Image is now in the local daemon. Seed the S3 cache so the // next run hits. Treat the upload as best-effort — the pull // already gave us what we needed locally. @@ -144,6 +162,8 @@ func init() { } dockerUploadCmd.Flags().BoolVar(&flagForce, "force", false, "overwrite if key already exists") dockerDownloadCmd.Flags().BoolVar(&flagDockerPull, "pull", false, "on cache miss, docker pull and seed S3") + addExecFlag(dockerDownloadCmd, " in the local docker daemon") + dockerDownloadCmd.MarkFlagsMutuallyExclusive("pull", "exec") dockerCmd.AddCommand(dockerExistsCmd, dockerDownloadCmd, dockerUploadCmd) rootCmd.AddCommand(dockerCmd) } @@ -161,6 +181,17 @@ func runDockerPull(ctx context.Context, tag string) error { return nil } +// dockerImagePresent asserts exists in the local daemon. +func dockerImagePresent(ctx context.Context, tag string) error { + inspect := exec.CommandContext(ctx, "docker", "image", "inspect", tag) + inspect.Stdout = io.Discard + inspect.Stderr = newWarnWriter("docker image inspect") + if err := inspect.Run(); err != nil { + return fmt.Errorf("--exec finished but %s is not in the local docker daemon: %w", tag, err) + } + return nil +} + // saveTagToS3 wires `docker save` stdout → zstd encoder → io.Pipe → // s3.Put. The encoder runs in a goroutine writing into the pipe; the // uploader goroutine reads from the pipe. Both ends signal errors back diff --git a/cmd/errors.go b/cmd/errors.go index 770497893b065d285241c8e313df5ad903a8275a..ad9067aa1392e948786aa0a1886b00235818e6aa 100644 --- a/cmd/errors.go +++ b/cmd/errors.go @@ -5,14 +5,18 @@ import "errors" // Sentinel errors recognized by Execute → exit code mapping. // // exists → ErrNotFound → 1 -// download without --url, key missing → ErrMissNoFallback → 3 +// download without --url/--exec, key missing → ErrMissNoFallback → 3 +// --exec script failed → the script's own exit status // any other error → 2 var ( ErrNotFound = errors.New("cacher: key not found") - ErrMissNoFallback = errors.New("cacher: cache miss and no --url fallback") + ErrMissNoFallback = errors.New("cacher: cache miss and no --url/--exec fallback") ) func exitCodeFor(err error) int { + // A failed --exec script reports its own status: the caller wrote + // that script and its exit code says more than a generic 2. + var coded interface{ ExitCode() int } switch { case err == nil: return 0 @@ -20,6 +24,8 @@ func exitCodeFor(err error) int { return 1 case errors.Is(err, ErrMissNoFallback): return 3 + case errors.As(err, &coded): + return coded.ExitCode() default: return 2 } diff --git a/cmd/exec.go b/cmd/exec.go new file mode 100644 index 0000000000000000000000000000000000000000..c15a0bc81e667008d729ba1c260faa9dae1e3267 --- /dev/null +++ b/cmd/exec.go @@ -0,0 +1,65 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + + "github.com/spf13/cobra" +) + +// Shared by every download subcommand. Only one command runs per +// process, so package-level flag vars are safe (same convention as +// flagForce). +var ( + flagExec string + flagOptional bool +) + +// addExecFlag attaches --exec. produces describes what the script is +// expected to leave behind, so each subcommand's help text is concrete. +func addExecFlag(c *cobra.Command, produces string) { + // The backquoted word sets the value placeholder cobra prints in + // --help ("--exec script"), so keep exactly one of them. + c.Flags().StringVar(&flagExec, "exec", "", + "on cache miss, run this `script` through sh -c (it must produce "+produces+") and seed the cache with the result") +} + +// addOptionalFlag attaches --optional: a miss stops being an error, so +// `set -e` scripts don't need a trailing `|| echo "cache miss"`. +func addOptionalFlag(c *cobra.Command) { + c.Flags().BoolVar(&flagOptional, "optional", false, + "treat a cache miss as success (exit 0) instead of an error") +} + +// runExecFallback runs the --exec script through `sh -c` with stdio +// inherited, so its output lands in the CI log in place, unbuffered and +// in order. A non-zero exit is propagated verbatim by exitCodeFor — +// the script's own status is more informative than cacher's. +func runExecFallback(ctx context.Context, w *cobra.Command, script string) error { + fmt.Fprintf(w.ErrOrStderr(), "Running — sh -c %q\n", script) + c := exec.CommandContext(ctx, "sh", "-c", script) + c.Stdin = os.Stdin + c.Stdout = os.Stdout + c.Stderr = os.Stderr + if err := c.Run(); err != nil { + var ee *exec.ExitError + if errors.As(err, &ee) { + return &execError{code: ee.ExitCode(), err: err} + } + return fmt.Errorf("--exec: %w", err) + } + return nil +} + +// execError carries the --exec script's own exit status up to Execute. +type execError struct { + code int + err error +} + +func (e *execError) Error() string { return fmt.Sprintf("--exec script failed: %v", e.err) } +func (e *execError) Unwrap() error { return e.err } +func (e *execError) ExitCode() int { return e.code } diff --git a/cmd/file.go b/cmd/file.go index c7412fcda8956eb7c5b6c81b936612972969fe8f..4e8457bb8fe0625213f92da16189583bd4ffa11d 100644 --- a/cmd/file.go +++ b/cmd/file.go @@ -23,8 +23,16 @@ var ( var downloadCmd = &cobra.Command{ Use: "download ", - Short: "Download a cached file; fall back to --url on cache miss", - Args: cobra.ExactArgs(2), + Short: "Download a cached file; fall back to --url or --exec on cache miss", + Long: `Cache HIT pulls from S3. Cache MISS falls back to whichever fallback +is configured, then seeds the cache with the result: + + --url GET into + --exec run with ` + "`sh -c`" + `; it must leave behind + +--sha256 verifies either fallback's output. With --optional, a miss +with no fallback exits 0 instead of 3.`, + Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { cli, cfg, err := client() if err != nil { @@ -52,7 +60,14 @@ var downloadCmd = &cobra.Command{ } fmt.Fprintf(cmd.ErrOrStderr(), "Cache MISS — %s\n", key) - if flagDownloadURL == "" { + switch { + case flagDownloadURL != "": + // handled below + case flagExec != "": + return execFallbackFile(ctx, cmd, cli, key, dest) + case flagOptional: + return nil + default: return fmt.Errorf("%w: %s", ErrMissNoFallback, key) } fmt.Fprintf(cmd.ErrOrStderr(), "Fetching %s\n", flagDownloadURL) @@ -98,6 +113,42 @@ var downloadCmd = &cobra.Command{ }, } +// execFallbackFile runs the --exec script, checks it actually produced +// dest, verifies --sha256 against it, and seeds the cache. The upload is +// best-effort for the same reason as the --url path: dest is already on +// disk, so an S3 glitch must not fail the build. +func execFallbackFile(ctx context.Context, cmd *cobra.Command, cli s3Putter, key, dest string) error { + if err := runExecFallback(ctx, cmd, flagExec); err != nil { + return err + } + st, err := os.Stat(dest) + if err != nil { + return fmt.Errorf("--exec finished but %s is unusable: %w", dest, err) + } + if st.IsDir() { + return fmt.Errorf("--exec produced a directory at %s; use `cacher dir download` for trees", dest) + } + f, err := os.Open(dest) + if err != nil { + return err + } + defer f.Close() + if flagDownloadSHA256 != "" { + if err := verifySHA(f, flagDownloadSHA256); err != nil { + return err + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return err + } + } + if err := cli.Put(ctx, key, f); err != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "warn: seed upload failed: %v\n", err) + return nil + } + fmt.Fprintf(cmd.ErrOrStderr(), "Cached → %s\n", key) + return nil +} + var uploadCmd = &cobra.Command{ Use: "upload ", Short: "Upload a local file to the cache", @@ -247,6 +298,9 @@ same key needs to appear in multiple subsequent invocations: func init() { downloadCmd.Flags().StringVar(&flagDownloadURL, "url", "", "fallback URL when key is missing in S3") downloadCmd.Flags().StringVar(&flagDownloadSHA256, "sha256", "", "expected sha256 hex of the downloaded content") + addExecFlag(downloadCmd, "") + addOptionalFlag(downloadCmd) + downloadCmd.MarkFlagsMutuallyExclusive("url", "exec", "optional") uploadCmd.Flags().BoolVar(&flagForce, "force", false, "overwrite if key already exists") for _, c := range []*cobra.Command{downloadCmd, uploadCmd, existsCmd, deleteCmd} { @@ -262,6 +316,19 @@ func init() { rootCmd.AddCommand(downloadCmd, uploadCmd, existsCmd, listCmd, deleteCmd, keyCmd) } +// verifySHA hashes r and compares against want (hex, case-insensitive). +func verifySHA(r io.Reader, want string) error { + h := sha256.New() + if _, err := io.Copy(h, r); err != nil { + return err + } + got := hex.EncodeToString(h.Sum(nil)) + if got != strings.ToLower(want) { + return fmt.Errorf("sha256 mismatch: got %s want %s", got, strings.ToLower(want)) + } + return nil +} + // writeWithSHA streams r into path; if want is non-empty, verifies the // sha256 matches. On mismatch, the partial file is removed. func writeWithSHA(path string, r io.Reader, want string) error { diff --git a/e2e_test.go b/e2e_test.go index 6b4cdb31a964aba0cfb318e276f0643cd158f57b..7bab92dcc458c9e7abd0b48c3c265563ba187d89 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -240,6 +240,110 @@ func TestDirRoundTrip(t *testing.T) { } } +// TestDirExecFallbackAndCacheFill is the manifest-shaped path: one +// invocation that either restores the tree or builds it and seeds the +// cache, with no if/fi around it. +func TestDirExecFallbackAndCacheFill(t *testing.T) { + g := garage.Start(t) + run := runner(t, g) + initCacher(t, run, g) + + dst1 := filepath.Join(t.TempDir(), "tree") + build := fmt.Sprintf("mkdir -p %s/sub && echo built > %s/sub/marker", dst1, dst1) + r := run("dir", "download", "exec-key", dst1, "--exec", build) + if r.exit != 0 { + t.Fatalf("dir download(miss → exec) exit=%d stderr=%s", r.exit, r.stderr) + } + if got := strings.TrimSpace(readFile(t, filepath.Join(dst1, "sub/marker"))); got != "built" { + t.Errorf("marker = %q want built", got) + } + + // Second call must HIT and never reach the script — the script here + // fails loudly if a regression runs it anyway. + dst2 := filepath.Join(t.TempDir(), "tree") + r = run("dir", "download", "exec-key", dst2, "--exec", "exit 9") + if r.exit != 0 { + t.Fatalf("dir download(hit) exit=%d stderr=%s", r.exit, r.stderr) + } + if !strings.Contains(r.stderr, "Cache HIT") { + t.Errorf("second call did not log Cache HIT:\n%s", r.stderr) + } + if got := strings.TrimSpace(readFile(t, filepath.Join(dst2, "sub/marker"))); got != "built" { + t.Errorf("restored marker = %q want built", got) + } +} + +// TestFileExecFallback mirrors the dir case for single files, including +// the --sha256 check against what the script produced. +func TestFileExecFallback(t *testing.T) { + g := garage.Start(t) + run := runner(t, g) + initCacher(t, run, g) + + dst := filepath.Join(t.TempDir(), "artifact") + r := run("download", "exec-file", dst, "--exec", "printf built > "+dst) + if r.exit != 0 { + t.Fatalf("download(miss → exec) exit=%d stderr=%s", r.exit, r.stderr) + } + if got := readFile(t, dst); got != "built" { + t.Errorf("artifact = %q want built", got) + } + + // The cache was seeded, so a second call hits without running the script. + dst2 := filepath.Join(t.TempDir(), "artifact") + r = run("download", "exec-file", dst2, "--exec", "exit 9") + if r.exit != 0 || !strings.Contains(r.stderr, "Cache HIT") { + t.Fatalf("download(hit) exit=%d stderr=%s", r.exit, r.stderr) + } + + // A script that produces content not matching --sha256 must fail. + dst3 := filepath.Join(t.TempDir(), "artifact") + r = run("download", "exec-bad-sha", dst3, + "--exec", "printf wrong > "+dst3, + "--sha256", "0000000000000000000000000000000000000000000000000000000000000000") + if r.exit != 2 { + t.Errorf("download(exec, sha mismatch) exit=%d, want 2\nstderr: %s", r.exit, r.stderr) + } +} + +// TestExecExitStatusPropagates: a failing build script must surface its +// own status, not a generic operational error. +func TestExecExitStatusPropagates(t *testing.T) { + g := garage.Start(t) + run := runner(t, g) + initCacher(t, run, g) + + dst := filepath.Join(t.TempDir(), "tree") + if r := run("dir", "download", "fails", dst, "--exec", "exit 7"); r.exit != 7 { + t.Errorf("dir download(--exec 'exit 7') exit=%d, want 7\nstderr: %s", r.exit, r.stderr) + } + // And nothing was cached for the key the failed script was building. + if r := run("exists", "fails"); r.exit != 1 { + t.Errorf("exists after failed --exec: exit=%d, want 1", r.exit) + } +} + +// TestOptionalMiss: --optional turns a miss into exit 0 for both the +// file and the directory command. +func TestOptionalMiss(t *testing.T) { + g := garage.Start(t) + run := runner(t, g) + initCacher(t, run, g) + + dir := filepath.Join(t.TempDir(), "tree") + if r := run("dir", "download", "absent", dir, "--optional"); r.exit != 0 { + t.Errorf("dir download(--optional) exit=%d, want 0\nstderr: %s", r.exit, r.stderr) + } + file := filepath.Join(t.TempDir(), "f") + if r := run("download", "absent", file, "--optional"); r.exit != 0 { + t.Errorf("download(--optional) exit=%d, want 0\nstderr: %s", r.exit, r.stderr) + } + // Still a miss, though: nothing must appear on disk. + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Errorf("--optional miss left %s behind (err=%v)", file, err) + } +} + func TestListDelimited(t *testing.T) { g := garage.Start(t) run := runner(t, g)