From 35e3d35fa942c39ddf0fcb3306c07458e5920391 Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Mon, 25 May 2026 16:37:24 +0300 Subject: [PATCH] Fix Garage compat: PingBucket via ListObjects, fixed config path, delimited list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three issues surfaced when first running doctor against the real s3.bigb.es Garage instance: * HeadBucket returned 403 even with valid creds. Garage doesn't implement HeadBucket the way AWS does. Switch the doctor smoke test to a 1-key ListObjectsV2 — matches what aws s3 ls does, which the existing shell helper used. * Config landed in ~/Library/Application Support/cacher on macOS via os.UserConfigDir. cacher is a CI tool that needs the same path on a dev mac and on a Linux build runner, so hard-code ~/.config/cacher/config.toml unconditionally. * cacher list flat-listed every object under the configured prefix. Add Delimiter="/" semantics by default (matches aws s3 ls), with --recursive for the previous behaviour and --root to ignore the configured prefix and list at the bucket root. --- cmd/doctor.go | 4 +-- cmd/file.go | 23 +++++++++++--- internal/config/config.go | 12 +++++--- internal/s3/client.go | 65 ++++++++++++++++++++++++++++++++------- 4 files changed, 83 insertions(+), 21 deletions(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index 7c2a18bb9ff21d275c6c7357b7fe88e9f915f4b8..bfeaf958f08fbaec7c94120de8e05dc004f1c523 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -41,10 +41,10 @@ func runDoctor(cmd *cobra.Command) error { len(keyID), safePrefix(keyID, 2), len(secret)) ctx := context.Background() - if err := cli.HeadBucket(ctx); err != nil { + if err := cli.PingBucket(ctx); err != nil { return err } - fmt.Fprintln(out, "head bucket: OK") + fmt.Fprintln(out, "ping bucket: OK") // Write/read/delete a tiny canary so credential perms are exercised // for all three operations, not just Head. Key is random so concurrent diff --git a/cmd/file.go b/cmd/file.go index 0ed7c2a354f37da60141fa389cc26b90f8b51e57..c7412fcda8956eb7c5b6c81b936612972969fe8f 100644 --- a/cmd/file.go +++ b/cmd/file.go @@ -17,6 +17,8 @@ var ( flagDownloadURL string flagDownloadSHA256 string flagForce bool + flagListRecursive bool + flagListRoot bool ) var downloadCmd = &cobra.Command{ @@ -160,23 +162,34 @@ var existsCmd = &cobra.Command{ var listCmd = &cobra.Command{ Use: "list [sub-prefix]", - Short: "List cache keys under prefix (optionally narrowed by sub-prefix)", - Args: cobra.MaximumNArgs(1), + Short: "List cache keys (delimited by / by default, like `aws s3 ls`)", + Long: `Without --recursive, listing is delimited by "/" — common prefixes +appear as "dir/" lines, files at this level appear plain. Mirrors +` + "`aws s3 ls`" + ` output style. + +Use --root to ignore the configured prefix and list at the bucket root.`, + Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { cli, _, err := client() if err != nil { return err } + if flagListRoot { + cli.SetPrefix("") + } sub := "" if len(args) == 1 { sub = args[0] } - keys, err := cli.List(context.Background(), sub) + res, err := cli.List(context.Background(), sub, flagListRecursive) if err != nil { return err } out := cmd.OutOrStdout() - for _, k := range keys { + for _, p := range res.Prefixes { + fmt.Fprintln(out, p) + } + for _, k := range res.Keys { fmt.Fprintln(out, k) } return nil @@ -241,6 +254,8 @@ func init() { addKeyFlags(c) } addS3Flags(listCmd) + listCmd.Flags().BoolVar(&flagListRecursive, "recursive", false, "list all keys under prefix (don't delimit by /)") + listCmd.Flags().BoolVar(&flagListRoot, "root", false, "ignore configured prefix; list at the bucket root") addS3Flags(keyCmd) addKeyFlags(keyCmd) diff --git a/internal/config/config.go b/internal/config/config.go index 47c4a7ca7be98e1ecf9161ccc17d0d980f01a6a5..1150c786fd8220bcb36606773d6b330eef0c4cac 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -40,13 +40,17 @@ func Defaults() Config { } } -// DefaultPath returns ~/.config/cacher/config.toml expanded. +// DefaultPath returns ~/.config/cacher/config.toml on every platform. +// We deliberately do not use os.UserConfigDir (which returns +// ~/Library/Application Support on macOS) because cacher is a CI tool +// that needs the same path on a developer's mac and on a Linux build +// runner. func DefaultPath() (string, error) { - dir, err := os.UserConfigDir() + home, err := os.UserHomeDir() if err != nil { - return "", fmt.Errorf("user config dir: %w", err) + return "", fmt.Errorf("user home dir: %w", err) } - return filepath.Join(dir, "cacher", "config.toml"), nil + return filepath.Join(home, ".config", "cacher", "config.toml"), nil } // Load reads the config file at path (empty path = DefaultPath). A missing diff --git a/internal/s3/client.go b/internal/s3/client.go index 6744c555429ec11d840978d0ac0b0661200c3bd3..4eed77c5e3ead5143128d1dce5ecd655a9d4d232 100644 --- a/internal/s3/client.go +++ b/internal/s3/client.go @@ -97,6 +97,12 @@ func (c *Client) FullKey(key string) string { // Bucket returns the configured bucket name (for diagnostics). func (c *Client) Bucket() string { return c.bucket } +// SetPrefix mutates the configured key prefix in place. Used by +// `cacher list --root` to escape the project namespace. +func (c *Client) SetPrefix(prefix string) { + c.prefix = strings.TrimSuffix(prefix, "/") +} + // Exists returns true if the object is present. func (c *Client) Exists(ctx context.Context, key string) (bool, error) { _, err := c.api.HeadObject(ctx, &s3.HeadObjectInput{ @@ -154,41 +160,78 @@ func (c *Client) Delete(ctx context.Context, key string) error { return nil } -// List yields object keys (relative to prefix) under sub. -func (c *Client) List(ctx context.Context, sub string) ([]string, error) { +// ListResult separates "files at this level" from "common prefixes +// (directories)". When recursive=true, Prefixes is always empty and +// Keys contains every object under sub. +type ListResult struct { + Keys []string // object keys relative to the client prefix + Prefixes []string // common prefixes (each ending in "/") relative to the client prefix +} + +// List walks objects under sub. When recursive is false, results are +// delimited by "/" so output matches `aws s3 ls` — common prefixes +// surface as "directories". +func (c *Client) List(ctx context.Context, sub string, recursive bool) (ListResult, error) { fullPrefix := c.FullKey(sub) - var keys []string + // When using a delimiter, the prefix must end with the delimiter + // for "directory" semantics — otherwise S3 treats the prefix as a + // literal substring match. + if !recursive && fullPrefix != "" && !strings.HasSuffix(fullPrefix, "/") { + fullPrefix += "/" + } + var delim *string + if !recursive { + delim = aws.String("/") + } + + var res ListResult + trim := "" + if c.prefix != "" { + trim = c.prefix + "/" + } + var token *string for { out, err := c.api.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ Bucket: aws.String(c.bucket), Prefix: aws.String(fullPrefix), + Delimiter: delim, ContinuationToken: token, }) if err != nil { - return nil, fmt.Errorf("list s3://%s/%s: %w", c.bucket, fullPrefix, err) + return res, fmt.Errorf("list s3://%s/%s: %w", c.bucket, fullPrefix, err) } for _, o := range out.Contents { if o.Key == nil { continue } - keys = append(keys, strings.TrimPrefix(*o.Key, c.prefix+"/")) + res.Keys = append(res.Keys, strings.TrimPrefix(*o.Key, trim)) + } + for _, p := range out.CommonPrefixes { + if p.Prefix == nil { + continue + } + res.Prefixes = append(res.Prefixes, strings.TrimPrefix(*p.Prefix, trim)) } if out.IsTruncated == nil || !*out.IsTruncated { break } token = out.NextContinuationToken } - return keys, nil + return res, nil } -// HeadBucket is the smoke test used by `cacher doctor`. -func (c *Client) HeadBucket(ctx context.Context) error { - _, err := c.api.HeadBucket(ctx, &s3.HeadBucketInput{ - Bucket: aws.String(c.bucket), +// PingBucket is the smoke test used by `cacher doctor`. Garage rejects +// HeadBucket with 403 even for valid credentials, so we use a 1-key +// ListObjectsV2 instead — matches `aws s3 ls s3://$bucket/` from the +// shell helper this replaces. +func (c *Client) PingBucket(ctx context.Context) error { + _, err := c.api.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(c.bucket), + MaxKeys: aws.Int32(1), }) if err != nil { - return fmt.Errorf("head bucket %s: %w", c.bucket, err) + return fmt.Errorf("ping bucket %s: %w", c.bucket, err) } return nil }