6 files changed, 358 insertions(+), 38 deletions(-)

M cmd/dir.go
M cmd/docker.go
M cmd/errors.go
A cmd/exec.go
M cmd/file.go
M e2e_test.go

@@ 22,7 22,18 @@ gap left by the single-file shell helper this binary replaces.`,
 var dirDownloadCmd = &cobra.Command{
 	Use:   "download <key> <local-dir>",
 	Short: "Extract a cached directory into <local-dir>",
-	Args:  cobra.ExactArgs(2),
+	Long: `Streams s3://bucket/key through zstd-decode into <local-dir>.
+
+With --exec, a cache miss falls back to running the given script with
+` + "`sh -c`" + ` and then seeds the cache from <local-dir>. That collapses the
+restore-or-build if/fi block CI manifests otherwise repeat per cache:
+
+  cacher dir download "$KEY" ~/scss --exec 'git clone … && cp -r …'
+
+<local-dir> is created before the script runs, so the script can write
+straight into it. With --optional, a miss exits 0 instead of 1 — for
+caches whose absence just means a cold build.`,
+	Args: cobra.ExactArgs(2),
 	RunE: func(cmd *cobra.Command, args []string) error {
 		cli, cfg, err := client()
 		if err != nil {


@@ 39,19 50,45 @@ var dirDownloadCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		if !ok {
-			return fmt.Errorf("%w: %s", ErrNotFound, key)
-		}
-		if err := os.MkdirAll(dest, 0o755); err != nil {
-			return err
+		if ok {
+			if err := os.MkdirAll(dest, 0o755); err != nil {
+				return err
+			}
+			fmt.Fprintf(cmd.ErrOrStderr(), "Cache HIT  — %s → extract %s\n", key, dest)
+			body, err := cli.Get(ctx, key)
+			if err != nil {
+				return err
+			}
+			defer body.Close()
+			return archive.DecodeDir(body, dest)
 		}
-		fmt.Fprintf(cmd.ErrOrStderr(), "Cache HIT  — %s → extract %s\n", key, dest)
-		body, err := cli.Get(ctx, key)
-		if err != nil {
-			return err
+
+		fmt.Fprintf(cmd.ErrOrStderr(), "Cache MISS — %s\n", key)
+		switch {
+		case flagExec != "":
+			// Created up front so the script can `cp` into it without
+			// repeating a mkdir -p of its own.
+			if err := os.MkdirAll(dest, 0o755); err != nil {
+				return err
+			}
+			if err := runExecFallback(ctx, cmd, flagExec); err != nil {
+				return err
+			}
+			// Seed the cache so the next run hits. Best-effort: the
+			// script already produced the tree locally, and an S3
+			// hiccup is no reason to fail a build that has its
+			// content. Same trade-off as `docker download --pull`.
+			if err := uploadDir(ctx, cli, key, dest, cmd.ErrOrStderr()); err != nil {
+				fmt.Fprintf(cmd.ErrOrStderr(), "warn: seed upload failed: %v\n", err)
+				return nil
+			}
+			fmt.Fprintf(cmd.ErrOrStderr(), "Cached  → %s\n", key)
+			return nil
+		case flagOptional:
+			return nil
+		default:
+			return fmt.Errorf("%w: %s", ErrNotFound, key)
 		}
-		defer body.Close()
-		return archive.DecodeDir(body, dest)
 	},
 }
 


@@ 81,30 118,40 @@ var dirUploadCmd = &cobra.Command{
 				return nil
 			}
 		}
-		st, err := os.Stat(src)
-		if err != nil {
-			return err
-		}
-		if !st.IsDir() {
-			return fmt.Errorf("%s is not a directory", src)
-		}
-		fmt.Fprintf(cmd.ErrOrStderr(), "Packing %s → %s\n", src, key)
-
-		pr, pw := io.Pipe()
-		go func() {
-			err := archive.EncodeDir(pw, src)
-			pw.CloseWithError(err)
-		}()
-		return cli.Put(ctx, key, pr)
+		return uploadDir(ctx, cli, key, src, cmd.ErrOrStderr())
 	},
 }
 
+// uploadDir packs src (tar+zstd) straight into an S3 upload — the
+// encoder writes into one end of a pipe while the uploader reads the
+// other, so nothing lands on disk. Shared by `dir upload` and the
+// --exec seed path of `dir download`.
+func uploadDir(ctx context.Context, cli s3Putter, key, src string, w io.Writer) error {
+	st, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+	if !st.IsDir() {
+		return fmt.Errorf("%s is not a directory", src)
+	}
+	fmt.Fprintf(w, "Packing %s → %s\n", src, key)
+	pr, pw := io.Pipe()
+	go func() {
+		err := archive.EncodeDir(pw, src)
+		pw.CloseWithError(err)
+	}()
+	return cli.Put(ctx, key, pr)
+}
+
 func init() {
 	for _, c := range []*cobra.Command{dirDownloadCmd, dirUploadCmd} {
 		addS3Flags(c)
 		addKeyFlags(c)
 	}
 	dirUploadCmd.Flags().BoolVar(&flagForce, "force", false, "overwrite if key already exists")
+	addExecFlag(dirDownloadCmd, "<local-dir>")
+	addOptionalFlag(dirDownloadCmd)
+	dirDownloadCmd.MarkFlagsMutuallyExclusive("exec", "optional")
 	dirCmd.AddCommand(dirDownloadCmd, dirUploadCmd)
 	rootCmd.AddCommand(dirCmd)
 }

@@ 51,7 51,13 @@ With --pull, a cache miss falls back to ` + "`docker pull <image:tag>`" + ` then
 seeds the S3 cache via ` + "`docker save`" + `. This makes a single invocation
 into the full cache-or-fetch pattern callable from a CI manifest:
 
-  cacher docker download garage/v2.3.0.tar.zst dxflrs/garage:v2.3.0 --pull`,
+  cacher docker download garage/v2.3.0.tar.zst dxflrs/garage:v2.3.0 --pull
+
+--exec is the same shape for images you build rather than pull: the
+script runs with ` + "`sh -c`" + ` and must leave <image:tag> in the local daemon,
+which is then saved to the cache.
+
+  cacher docker download "$KEY" myimage:latest --exec 'docker build -t myimage:latest .'`,
 	Args: cobra.ExactArgs(2),
 	RunE: func(cmd *cobra.Command, args []string) error {
 		cli, cfg, err := client()


@@ 85,13 91,25 @@ into the full cache-or-fetch pattern callable from a CI manifest:
 			return runDockerLoad(ctx, dec)
 		}
 		fmt.Fprintf(cmd.ErrOrStderr(), "Cache MISS — %s\n", key)
-		if !flagDockerPull {
+		switch {
+		case flagDockerPull:
+			fmt.Fprintf(cmd.ErrOrStderr(), "Pulling %s\n", tag)
+			if err := runDockerPull(ctx, tag); err != nil {
+				return err
+			}
+		case flagExec != "":
+			if err := runExecFallback(ctx, cmd, flagExec); err != nil {
+				return err
+			}
+			// The seed upload below is best-effort, so an --exec script
+			// that exits 0 without producing the tag would otherwise
+			// pass silently and blow up in a later task instead.
+			if err := dockerImagePresent(ctx, tag); err != nil {
+				return err
+			}
+		default:
 			return fmt.Errorf("%w: %s", ErrNotFound, key)
 		}
-		fmt.Fprintf(cmd.ErrOrStderr(), "Pulling %s\n", tag)
-		if err := runDockerPull(ctx, tag); err != nil {
-			return err
-		}
 		// Image is now in the local daemon. Seed the S3 cache so the
 		// next run hits. Treat the upload as best-effort — the pull
 		// already gave us what we needed locally.


@@ 144,6 162,8 @@ func init() {
 	}
 	dockerUploadCmd.Flags().BoolVar(&flagForce, "force", false, "overwrite if key already exists")
 	dockerDownloadCmd.Flags().BoolVar(&flagDockerPull, "pull", false, "on cache miss, docker pull <image:tag> and seed S3")
+	addExecFlag(dockerDownloadCmd, "<image:tag> in the local docker daemon")
+	dockerDownloadCmd.MarkFlagsMutuallyExclusive("pull", "exec")
 	dockerCmd.AddCommand(dockerExistsCmd, dockerDownloadCmd, dockerUploadCmd)
 	rootCmd.AddCommand(dockerCmd)
 }


@@ 161,6 181,17 @@ func runDockerPull(ctx context.Context, tag string) error {
 	return nil
 }
 
+// dockerImagePresent asserts <tag> exists in the local daemon.
+func dockerImagePresent(ctx context.Context, tag string) error {
+	inspect := exec.CommandContext(ctx, "docker", "image", "inspect", tag)
+	inspect.Stdout = io.Discard
+	inspect.Stderr = newWarnWriter("docker image inspect")
+	if err := inspect.Run(); err != nil {
+		return fmt.Errorf("--exec finished but %s is not in the local docker daemon: %w", tag, err)
+	}
+	return nil
+}
+
 // saveTagToS3 wires `docker save` stdout → zstd encoder → io.Pipe →
 // s3.Put. The encoder runs in a goroutine writing into the pipe; the
 // uploader goroutine reads from the pipe. Both ends signal errors back

@@ 5,14 5,18 @@ import "errors"
 // Sentinel errors recognized by Execute → exit code mapping.
 //
 // exists  → ErrNotFound  →  1
-// download without --url, key missing → ErrMissNoFallback → 3
+// download without --url/--exec, key missing → ErrMissNoFallback → 3
+// --exec script failed → the script's own exit status
 // any other error → 2
 var (
 	ErrNotFound       = errors.New("cacher: key not found")
-	ErrMissNoFallback = errors.New("cacher: cache miss and no --url fallback")
+	ErrMissNoFallback = errors.New("cacher: cache miss and no --url/--exec fallback")
 )
 
 func exitCodeFor(err error) int {
+	// A failed --exec script reports its own status: the caller wrote
+	// that script and its exit code says more than a generic 2.
+	var coded interface{ ExitCode() int }
 	switch {
 	case err == nil:
 		return 0


@@ 20,6 24,8 @@ func exitCodeFor(err error) int {
 		return 1
 	case errors.Is(err, ErrMissNoFallback):
 		return 3
+	case errors.As(err, &coded):
+		return coded.ExitCode()
 	default:
 		return 2
 	}

@@ 0,0 1,65 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+
+	"github.com/spf13/cobra"
+)
+
+// Shared by every download subcommand. Only one command runs per
+// process, so package-level flag vars are safe (same convention as
+// flagForce).
+var (
+	flagExec     string
+	flagOptional bool
+)
+
+// addExecFlag attaches --exec. produces describes what the script is
+// expected to leave behind, so each subcommand's help text is concrete.
+func addExecFlag(c *cobra.Command, produces string) {
+	// The backquoted word sets the value placeholder cobra prints in
+	// --help ("--exec script"), so keep exactly one of them.
+	c.Flags().StringVar(&flagExec, "exec", "",
+		"on cache miss, run this `script` through sh -c (it must produce "+produces+") and seed the cache with the result")
+}
+
+// addOptionalFlag attaches --optional: a miss stops being an error, so
+// `set -e` scripts don't need a trailing `|| echo "cache miss"`.
+func addOptionalFlag(c *cobra.Command) {
+	c.Flags().BoolVar(&flagOptional, "optional", false,
+		"treat a cache miss as success (exit 0) instead of an error")
+}
+
+// runExecFallback runs the --exec script through `sh -c` with stdio
+// inherited, so its output lands in the CI log in place, unbuffered and
+// in order. A non-zero exit is propagated verbatim by exitCodeFor —
+// the script's own status is more informative than cacher's.
+func runExecFallback(ctx context.Context, w *cobra.Command, script string) error {
+	fmt.Fprintf(w.ErrOrStderr(), "Running  — sh -c %q\n", script)
+	c := exec.CommandContext(ctx, "sh", "-c", script)
+	c.Stdin = os.Stdin
+	c.Stdout = os.Stdout
+	c.Stderr = os.Stderr
+	if err := c.Run(); err != nil {
+		var ee *exec.ExitError
+		if errors.As(err, &ee) {
+			return &execError{code: ee.ExitCode(), err: err}
+		}
+		return fmt.Errorf("--exec: %w", err)
+	}
+	return nil
+}
+
+// execError carries the --exec script's own exit status up to Execute.
+type execError struct {
+	code int
+	err  error
+}
+
+func (e *execError) Error() string { return fmt.Sprintf("--exec script failed: %v", e.err) }
+func (e *execError) Unwrap() error { return e.err }
+func (e *execError) ExitCode() int { return e.code }

@@ 23,8 23,16 @@ var (
 
 var downloadCmd = &cobra.Command{
 	Use:   "download <key> <local-path>",
-	Short: "Download a cached file; fall back to --url on cache miss",
-	Args:  cobra.ExactArgs(2),
+	Short: "Download a cached file; fall back to --url or --exec on cache miss",
+	Long: `Cache HIT pulls from S3. Cache MISS falls back to whichever fallback
+is configured, then seeds the cache with the result:
+
+  --url  <u>   GET <u> into <local-path>
+  --exec <sh>  run <sh> with ` + "`sh -c`" + `; it must leave <local-path> behind
+
+--sha256 verifies either fallback's output. With --optional, a miss
+with no fallback exits 0 instead of 3.`,
+	Args: cobra.ExactArgs(2),
 	RunE: func(cmd *cobra.Command, args []string) error {
 		cli, cfg, err := client()
 		if err != nil {


@@ 52,7 60,14 @@ var downloadCmd = &cobra.Command{
 		}
 
 		fmt.Fprintf(cmd.ErrOrStderr(), "Cache MISS — %s\n", key)
-		if flagDownloadURL == "" {
+		switch {
+		case flagDownloadURL != "":
+			// handled below
+		case flagExec != "":
+			return execFallbackFile(ctx, cmd, cli, key, dest)
+		case flagOptional:
+			return nil
+		default:
 			return fmt.Errorf("%w: %s", ErrMissNoFallback, key)
 		}
 		fmt.Fprintf(cmd.ErrOrStderr(), "Fetching %s\n", flagDownloadURL)


@@ 98,6 113,42 @@ var downloadCmd = &cobra.Command{
 	},
 }
 
+// execFallbackFile runs the --exec script, checks it actually produced
+// dest, verifies --sha256 against it, and seeds the cache. The upload is
+// best-effort for the same reason as the --url path: dest is already on
+// disk, so an S3 glitch must not fail the build.
+func execFallbackFile(ctx context.Context, cmd *cobra.Command, cli s3Putter, key, dest string) error {
+	if err := runExecFallback(ctx, cmd, flagExec); err != nil {
+		return err
+	}
+	st, err := os.Stat(dest)
+	if err != nil {
+		return fmt.Errorf("--exec finished but %s is unusable: %w", dest, err)
+	}
+	if st.IsDir() {
+		return fmt.Errorf("--exec produced a directory at %s; use `cacher dir download` for trees", dest)
+	}
+	f, err := os.Open(dest)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if flagDownloadSHA256 != "" {
+		if err := verifySHA(f, flagDownloadSHA256); err != nil {
+			return err
+		}
+		if _, err := f.Seek(0, io.SeekStart); err != nil {
+			return err
+		}
+	}
+	if err := cli.Put(ctx, key, f); err != nil {
+		fmt.Fprintf(cmd.ErrOrStderr(), "warn: seed upload failed: %v\n", err)
+		return nil
+	}
+	fmt.Fprintf(cmd.ErrOrStderr(), "Cached  → %s\n", key)
+	return nil
+}
+
 var uploadCmd = &cobra.Command{
 	Use:   "upload <key> <local-path>",
 	Short: "Upload a local file to the cache",


@@ 247,6 298,9 @@ same key needs to appear in multiple subsequent invocations:
 func init() {
 	downloadCmd.Flags().StringVar(&flagDownloadURL, "url", "", "fallback URL when key is missing in S3")
 	downloadCmd.Flags().StringVar(&flagDownloadSHA256, "sha256", "", "expected sha256 hex of the downloaded content")
+	addExecFlag(downloadCmd, "<local-path>")
+	addOptionalFlag(downloadCmd)
+	downloadCmd.MarkFlagsMutuallyExclusive("url", "exec", "optional")
 	uploadCmd.Flags().BoolVar(&flagForce, "force", false, "overwrite if key already exists")
 
 	for _, c := range []*cobra.Command{downloadCmd, uploadCmd, existsCmd, deleteCmd} {


@@ 262,6 316,19 @@ func init() {
 	rootCmd.AddCommand(downloadCmd, uploadCmd, existsCmd, listCmd, deleteCmd, keyCmd)
 }
 
+// verifySHA hashes r and compares against want (hex, case-insensitive).
+func verifySHA(r io.Reader, want string) error {
+	h := sha256.New()
+	if _, err := io.Copy(h, r); err != nil {
+		return err
+	}
+	got := hex.EncodeToString(h.Sum(nil))
+	if got != strings.ToLower(want) {
+		return fmt.Errorf("sha256 mismatch: got %s want %s", got, strings.ToLower(want))
+	}
+	return nil
+}
+
 // writeWithSHA streams r into path; if want is non-empty, verifies the
 // sha256 matches. On mismatch, the partial file is removed.
 func writeWithSHA(path string, r io.Reader, want string) error {

@@ 240,6 240,110 @@ func TestDirRoundTrip(t *testing.T) {
 	}
 }
 
+// TestDirExecFallbackAndCacheFill is the manifest-shaped path: one
+// invocation that either restores the tree or builds it and seeds the
+// cache, with no if/fi around it.
+func TestDirExecFallbackAndCacheFill(t *testing.T) {
+	g := garage.Start(t)
+	run := runner(t, g)
+	initCacher(t, run, g)
+
+	dst1 := filepath.Join(t.TempDir(), "tree")
+	build := fmt.Sprintf("mkdir -p %s/sub && echo built > %s/sub/marker", dst1, dst1)
+	r := run("dir", "download", "exec-key", dst1, "--exec", build)
+	if r.exit != 0 {
+		t.Fatalf("dir download(miss → exec) exit=%d stderr=%s", r.exit, r.stderr)
+	}
+	if got := strings.TrimSpace(readFile(t, filepath.Join(dst1, "sub/marker"))); got != "built" {
+		t.Errorf("marker = %q want built", got)
+	}
+
+	// Second call must HIT and never reach the script — the script here
+	// fails loudly if a regression runs it anyway.
+	dst2 := filepath.Join(t.TempDir(), "tree")
+	r = run("dir", "download", "exec-key", dst2, "--exec", "exit 9")
+	if r.exit != 0 {
+		t.Fatalf("dir download(hit) exit=%d stderr=%s", r.exit, r.stderr)
+	}
+	if !strings.Contains(r.stderr, "Cache HIT") {
+		t.Errorf("second call did not log Cache HIT:\n%s", r.stderr)
+	}
+	if got := strings.TrimSpace(readFile(t, filepath.Join(dst2, "sub/marker"))); got != "built" {
+		t.Errorf("restored marker = %q want built", got)
+	}
+}
+
+// TestFileExecFallback mirrors the dir case for single files, including
+// the --sha256 check against what the script produced.
+func TestFileExecFallback(t *testing.T) {
+	g := garage.Start(t)
+	run := runner(t, g)
+	initCacher(t, run, g)
+
+	dst := filepath.Join(t.TempDir(), "artifact")
+	r := run("download", "exec-file", dst, "--exec", "printf built > "+dst)
+	if r.exit != 0 {
+		t.Fatalf("download(miss → exec) exit=%d stderr=%s", r.exit, r.stderr)
+	}
+	if got := readFile(t, dst); got != "built" {
+		t.Errorf("artifact = %q want built", got)
+	}
+
+	// The cache was seeded, so a second call hits without running the script.
+	dst2 := filepath.Join(t.TempDir(), "artifact")
+	r = run("download", "exec-file", dst2, "--exec", "exit 9")
+	if r.exit != 0 || !strings.Contains(r.stderr, "Cache HIT") {
+		t.Fatalf("download(hit) exit=%d stderr=%s", r.exit, r.stderr)
+	}
+
+	// A script that produces content not matching --sha256 must fail.
+	dst3 := filepath.Join(t.TempDir(), "artifact")
+	r = run("download", "exec-bad-sha", dst3,
+		"--exec", "printf wrong > "+dst3,
+		"--sha256", "0000000000000000000000000000000000000000000000000000000000000000")
+	if r.exit != 2 {
+		t.Errorf("download(exec, sha mismatch) exit=%d, want 2\nstderr: %s", r.exit, r.stderr)
+	}
+}
+
+// TestExecExitStatusPropagates: a failing build script must surface its
+// own status, not a generic operational error.
+func TestExecExitStatusPropagates(t *testing.T) {
+	g := garage.Start(t)
+	run := runner(t, g)
+	initCacher(t, run, g)
+
+	dst := filepath.Join(t.TempDir(), "tree")
+	if r := run("dir", "download", "fails", dst, "--exec", "exit 7"); r.exit != 7 {
+		t.Errorf("dir download(--exec 'exit 7') exit=%d, want 7\nstderr: %s", r.exit, r.stderr)
+	}
+	// And nothing was cached for the key the failed script was building.
+	if r := run("exists", "fails"); r.exit != 1 {
+		t.Errorf("exists after failed --exec: exit=%d, want 1", r.exit)
+	}
+}
+
+// TestOptionalMiss: --optional turns a miss into exit 0 for both the
+// file and the directory command.
+func TestOptionalMiss(t *testing.T) {
+	g := garage.Start(t)
+	run := runner(t, g)
+	initCacher(t, run, g)
+
+	dir := filepath.Join(t.TempDir(), "tree")
+	if r := run("dir", "download", "absent", dir, "--optional"); r.exit != 0 {
+		t.Errorf("dir download(--optional) exit=%d, want 0\nstderr: %s", r.exit, r.stderr)
+	}
+	file := filepath.Join(t.TempDir(), "f")
+	if r := run("download", "absent", file, "--optional"); r.exit != 0 {
+		t.Errorf("download(--optional) exit=%d, want 0\nstderr: %s", r.exit, r.stderr)
+	}
+	// Still a miss, though: nothing must appear on disk.
+	if _, err := os.Stat(file); !os.IsNotExist(err) {
+		t.Errorf("--optional miss left %s behind (err=%v)", file, err)
+	}
+}
+
 func TestListDelimited(t *testing.T) {
 	g := garage.Start(t)
 	run := runner(t, g)