~bigbes/ci-cacher

ref: 8851cb58dd2ccc5d3dd2cea8856d114fd69e1fff ci-cacher/cmd/dir.go -rw-r--r-- 4.3 KiB
8851cb58 — Eugene Blikh test.yml: install via install.sh, collapse cache_gomod to one dir download 11 days ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
package cmd

import (
	"context"
	"fmt"
	"io"
	"os"

	"github.com/spf13/cobra"

	"go.bigb.es/cacher/internal/archive"
)

var dirCmd = &cobra.Command{
	Use:   "dir",
	Short: "Directory cache (tar+zstd of a tree)",
	Long: `Cache resolved trees keyed by content hash (e.g. ~/go/pkg/mod
keyed by go.sum, or .rocks/ keyed by rockspec hash). Closes the biggest
gap left by the single-file shell helper this binary replaces.`,
}

var dirDownloadCmd = &cobra.Command{
	Use:   "download <key> <local-dir>",
	Short: "Extract a cached directory into <local-dir>",
	Long: `Streams s3://bucket/key through zstd-decode into <local-dir>.

With --exec, a cache miss falls back to running the given script with
` + "`sh -c`" + ` and then seeds the cache from <local-dir>. That collapses the
restore-or-build if/fi block CI manifests otherwise repeat per cache:

  cacher dir download "$KEY" ~/scss --exec 'git clone … && cp -r …'

<local-dir> is created before the script runs, so the script can write
straight into it. With --optional, a miss exits 0 instead of 1 — for
caches whose absence just means a cold build.`,
	Args: cobra.ExactArgs(2),
	RunE: func(cmd *cobra.Command, args []string) error {
		cli, cfg, err := client()
		if err != nil {
			return err
		}
		key, err := resolveKey(args[0], cfg)
		if err != nil {
			return err
		}
		dest := args[1]
		ctx := context.Background()

		ok, err := cli.Exists(ctx, key)
		if err != nil {
			return err
		}
		if ok {
			if err := os.MkdirAll(dest, 0o755); err != nil {
				return err
			}
			fmt.Fprintf(cmd.ErrOrStderr(), "Cache HIT  — %s → extract %s\n", key, dest)
			body, err := cli.Get(ctx, key)
			if err != nil {
				return err
			}
			defer body.Close()
			return archive.DecodeDir(body, dest)
		}

		fmt.Fprintf(cmd.ErrOrStderr(), "Cache MISS — %s\n", key)
		switch {
		case flagExec != "":
			// Created up front so the script can `cp` into it without
			// repeating a mkdir -p of its own.
			if err := os.MkdirAll(dest, 0o755); err != nil {
				return err
			}
			if err := runExecFallback(ctx, cmd, flagExec); err != nil {
				return err
			}
			// Seed the cache so the next run hits. Best-effort: the
			// script already produced the tree locally, and an S3
			// hiccup is no reason to fail a build that has its
			// content. Same trade-off as `docker download --pull`.
			if err := uploadDir(ctx, cli, key, dest, cmd.ErrOrStderr()); err != nil {
				fmt.Fprintf(cmd.ErrOrStderr(), "warn: seed upload failed: %v\n", err)
				return nil
			}
			fmt.Fprintf(cmd.ErrOrStderr(), "Cached  → %s\n", key)
			return nil
		case flagOptional:
			return nil
		default:
			return fmt.Errorf("%w: %s", ErrNotFound, key)
		}
	},
}

var dirUploadCmd = &cobra.Command{
	Use:   "upload <key> <local-dir>",
	Short: "Pack a local directory (tar+zstd) and upload",
	Args:  cobra.ExactArgs(2),
	RunE: func(cmd *cobra.Command, args []string) error {
		cli, cfg, err := client()
		if err != nil {
			return err
		}
		key, err := resolveKey(args[0], cfg)
		if err != nil {
			return err
		}
		src := args[1]
		ctx := context.Background()

		if !flagForce {
			ok, err := cli.Exists(ctx, key)
			if err != nil {
				return err
			}
			if ok {
				fmt.Fprintf(cmd.ErrOrStderr(), "Skipped — %s already present (use --force)\n", key)
				return nil
			}
		}
		return uploadDir(ctx, cli, key, src, cmd.ErrOrStderr())
	},
}

// uploadDir packs src (tar+zstd) straight into an S3 upload — the
// encoder writes into one end of a pipe while the uploader reads the
// other, so nothing lands on disk. Shared by `dir upload` and the
// --exec seed path of `dir download`.
func uploadDir(ctx context.Context, cli s3Putter, key, src string, w io.Writer) error {
	st, err := os.Stat(src)
	if err != nil {
		return err
	}
	if !st.IsDir() {
		return fmt.Errorf("%s is not a directory", src)
	}
	fmt.Fprintf(w, "Packing %s → %s\n", src, key)
	pr, pw := io.Pipe()
	go func() {
		err := archive.EncodeDir(pw, src)
		pw.CloseWithError(err)
	}()
	return cli.Put(ctx, key, pr)
}

func init() {
	for _, c := range []*cobra.Command{dirDownloadCmd, dirUploadCmd} {
		addS3Flags(c)
		addKeyFlags(c)
	}
	dirUploadCmd.Flags().BoolVar(&flagForce, "force", false, "overwrite if key already exists")
	addExecFlag(dirDownloadCmd, "<local-dir>")
	addOptionalFlag(dirDownloadCmd)
	dirDownloadCmd.MarkFlagsMutuallyExclusive("exec", "optional")
	dirCmd.AddCommand(dirDownloadCmd, dirUploadCmd)
	rootCmd.AddCommand(dirCmd)
}