~bigbes/ci-cacher

35e3d35fa942c39ddf0fcb3306c07458e5920391 — Eugene Blikh 2 days ago 0ad1486
Fix Garage compat: PingBucket via ListObjects, fixed config path, delimited list

Three issues surfaced when first running doctor against the real
s3.bigb.es Garage instance:

* HeadBucket returned 403 even with valid creds. Garage doesn't
  implement HeadBucket the way AWS does. Switch the doctor smoke
  test to a 1-key ListObjectsV2 — matches what aws s3 ls does, which
  the existing shell helper used.

* Config landed in ~/Library/Application Support/cacher on macOS
  via os.UserConfigDir. cacher is a CI tool that needs the same
  path on a dev mac and on a Linux build runner, so hard-code
  ~/.config/cacher/config.toml unconditionally.

* cacher list flat-listed every object under the configured prefix.
  Add Delimiter="/" semantics by default (matches aws s3 ls), with
  --recursive for the previous behaviour and --root to ignore the
  configured prefix and list at the bucket root.
4 files changed, 83 insertions(+), 21 deletions(-)

M cmd/doctor.go
M cmd/file.go
M internal/config/config.go
M internal/s3/client.go
M cmd/doctor.go => cmd/doctor.go +2 -2
@@ 41,10 41,10 @@ func runDoctor(cmd *cobra.Command) error {
		len(keyID), safePrefix(keyID, 2), len(secret))

	ctx := context.Background()
	if err := cli.HeadBucket(ctx); err != nil {
	if err := cli.PingBucket(ctx); err != nil {
		return err
	}
	fmt.Fprintln(out, "head bucket: OK")
	fmt.Fprintln(out, "ping bucket: OK")

	// Write/read/delete a tiny canary so credential perms are exercised
	// for all three operations, not just Head. Key is random so concurrent

M cmd/file.go => cmd/file.go +19 -4
@@ 17,6 17,8 @@ var (
	flagDownloadURL    string
	flagDownloadSHA256 string
	flagForce          bool
	flagListRecursive  bool
	flagListRoot       bool
)

var downloadCmd = &cobra.Command{


@@ 160,23 162,34 @@ var existsCmd = &cobra.Command{

var listCmd = &cobra.Command{
	Use:   "list [sub-prefix]",
	Short: "List cache keys under prefix (optionally narrowed by sub-prefix)",
	Args:  cobra.MaximumNArgs(1),
	Short: "List cache keys (delimited by / by default, like `aws s3 ls`)",
	Long: `Without --recursive, listing is delimited by "/" — common prefixes
appear as "dir/" lines, files at this level appear plain. Mirrors
` + "`aws s3 ls`" + ` output style.

Use --root to ignore the configured prefix and list at the bucket root.`,
	Args: cobra.MaximumNArgs(1),
	RunE: func(cmd *cobra.Command, args []string) error {
		cli, _, err := client()
		if err != nil {
			return err
		}
		if flagListRoot {
			cli.SetPrefix("")
		}
		sub := ""
		if len(args) == 1 {
			sub = args[0]
		}
		keys, err := cli.List(context.Background(), sub)
		res, err := cli.List(context.Background(), sub, flagListRecursive)
		if err != nil {
			return err
		}
		out := cmd.OutOrStdout()
		for _, k := range keys {
		for _, p := range res.Prefixes {
			fmt.Fprintln(out, p)
		}
		for _, k := range res.Keys {
			fmt.Fprintln(out, k)
		}
		return nil


@@ 241,6 254,8 @@ func init() {
		addKeyFlags(c)
	}
	addS3Flags(listCmd)
	listCmd.Flags().BoolVar(&flagListRecursive, "recursive", false, "list all keys under prefix (don't delimit by /)")
	listCmd.Flags().BoolVar(&flagListRoot, "root", false, "ignore configured prefix; list at the bucket root")
	addS3Flags(keyCmd)
	addKeyFlags(keyCmd)


M internal/config/config.go => internal/config/config.go +8 -4
@@ 40,13 40,17 @@ func Defaults() Config {
	}
}

// DefaultPath returns ~/.config/cacher/config.toml expanded.
// DefaultPath returns ~/.config/cacher/config.toml on every platform.
// We deliberately do not use os.UserConfigDir (which returns
// ~/Library/Application Support on macOS) because cacher is a CI tool
// that needs the same path on a developer's mac and on a Linux build
// runner.
func DefaultPath() (string, error) {
	dir, err := os.UserConfigDir()
	home, err := os.UserHomeDir()
	if err != nil {
		return "", fmt.Errorf("user config dir: %w", err)
		return "", fmt.Errorf("user home dir: %w", err)
	}
	return filepath.Join(dir, "cacher", "config.toml"), nil
	return filepath.Join(home, ".config", "cacher", "config.toml"), nil
}

// Load reads the config file at path (empty path = DefaultPath). A missing

M internal/s3/client.go => internal/s3/client.go +54 -11
@@ 97,6 97,12 @@ func (c *Client) FullKey(key string) string {
// Bucket returns the configured bucket name (for diagnostics).
func (c *Client) Bucket() string { return c.bucket }

// SetPrefix mutates the configured key prefix in place. Used by
// `cacher list --root` to escape the project namespace.
func (c *Client) SetPrefix(prefix string) {
	c.prefix = strings.TrimSuffix(prefix, "/")
}

// Exists returns true if the object is present.
func (c *Client) Exists(ctx context.Context, key string) (bool, error) {
	_, err := c.api.HeadObject(ctx, &s3.HeadObjectInput{


@@ 154,41 160,78 @@ func (c *Client) Delete(ctx context.Context, key string) error {
	return nil
}

// List yields object keys (relative to prefix) under sub.
func (c *Client) List(ctx context.Context, sub string) ([]string, error) {
// ListResult separates "files at this level" from "common prefixes
// (directories)". When recursive=true, Prefixes is always empty and
// Keys contains every object under sub.
type ListResult struct {
	Keys     []string // object keys relative to the client prefix
	Prefixes []string // common prefixes (each ending in "/") relative to the client prefix
}

// List walks objects under sub. When recursive is false, results are
// delimited by "/" so output matches `aws s3 ls` — common prefixes
// surface as "directories".
func (c *Client) List(ctx context.Context, sub string, recursive bool) (ListResult, error) {
	fullPrefix := c.FullKey(sub)
	var keys []string
	// When using a delimiter, the prefix must end with the delimiter
	// for "directory" semantics — otherwise S3 treats the prefix as a
	// literal substring match.
	if !recursive && fullPrefix != "" && !strings.HasSuffix(fullPrefix, "/") {
		fullPrefix += "/"
	}
	var delim *string
	if !recursive {
		delim = aws.String("/")
	}

	var res ListResult
	trim := ""
	if c.prefix != "" {
		trim = c.prefix + "/"
	}

	var token *string
	for {
		out, err := c.api.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
			Bucket:            aws.String(c.bucket),
			Prefix:            aws.String(fullPrefix),
			Delimiter:         delim,
			ContinuationToken: token,
		})
		if err != nil {
			return nil, fmt.Errorf("list s3://%s/%s: %w", c.bucket, fullPrefix, err)
			return res, fmt.Errorf("list s3://%s/%s: %w", c.bucket, fullPrefix, err)
		}
		for _, o := range out.Contents {
			if o.Key == nil {
				continue
			}
			keys = append(keys, strings.TrimPrefix(*o.Key, c.prefix+"/"))
			res.Keys = append(res.Keys, strings.TrimPrefix(*o.Key, trim))
		}
		for _, p := range out.CommonPrefixes {
			if p.Prefix == nil {
				continue
			}
			res.Prefixes = append(res.Prefixes, strings.TrimPrefix(*p.Prefix, trim))
		}
		if out.IsTruncated == nil || !*out.IsTruncated {
			break
		}
		token = out.NextContinuationToken
	}
	return keys, nil
	return res, nil
}

// HeadBucket is the smoke test used by `cacher doctor`.
func (c *Client) HeadBucket(ctx context.Context) error {
	_, err := c.api.HeadBucket(ctx, &s3.HeadBucketInput{
		Bucket: aws.String(c.bucket),
// PingBucket is the smoke test used by `cacher doctor`. Garage rejects
// HeadBucket with 403 even for valid credentials, so we use a 1-key
// ListObjectsV2 instead — matches `aws s3 ls s3://$bucket/` from the
// shell helper this replaces.
func (c *Client) PingBucket(ctx context.Context) error {
	_, err := c.api.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
		Bucket:  aws.String(c.bucket),
		MaxKeys: aws.Int32(1),
	})
	if err != nil {
		return fmt.Errorf("head bucket %s: %w", c.bucket, err)
		return fmt.Errorf("ping bucket %s: %w", c.bucket, err)
	}
	return nil
}