// Package archive streams a directory through tar+zstd (and back). // // Compression level is fixed at zstd level 3 — same as the existing shell // (`zstd -T0 -3`). The encoder is wrapped around an io.Writer (typically an // S3 multipart Uploader's PipeWriter), so the whole pipeline stays // streaming end-to-end with no on-disk tempfile. package archive import ( "archive/tar" "errors" "fmt" "io" "io/fs" "os" "path/filepath" "sort" "strings" "github.com/klauspost/compress/zstd" ) // EncodeDir walks root in sorted relative-path order and writes a // tar.zst stream into w. Symlinks are preserved (stored as tar typelink), // devices/sockets/fifos are skipped. func EncodeDir(w io.Writer, root string) error { zw, err := zstd.NewWriter(w, zstd.WithEncoderLevel(zstd.SpeedDefault)) if err != nil { return fmt.Errorf("zstd writer: %w", err) } defer zw.Close() tw := tar.NewWriter(zw) var paths []string err = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { if err != nil { return err } if p == root { return nil } paths = append(paths, p) return nil }) if err != nil { return fmt.Errorf("walk %s: %w", root, err) } sort.Strings(paths) for _, p := range paths { if err := writeOne(tw, root, p); err != nil { return err } } if err := tw.Close(); err != nil { return fmt.Errorf("close tar: %w", err) } if err := zw.Close(); err != nil { return fmt.Errorf("close zstd: %w", err) } return nil } func writeOne(tw *tar.Writer, root, p string) error { rel, err := filepath.Rel(root, p) if err != nil { return err } rel = filepath.ToSlash(rel) lst, err := os.Lstat(p) if err != nil { return fmt.Errorf("lstat %s: %w", p, err) } var link string if lst.Mode()&os.ModeSymlink != 0 { link, err = os.Readlink(p) if err != nil { return fmt.Errorf("readlink %s: %w", p, err) } } hdr, err := tar.FileInfoHeader(lst, link) if err != nil { // Skip unsupported file types (devices, sockets, fifos). return nil } hdr.Name = rel if lst.IsDir() && !strings.HasSuffix(hdr.Name, "/") { hdr.Name += "/" } if err := tw.WriteHeader(hdr); err != nil { return fmt.Errorf("tar header %s: %w", rel, err) } if !lst.Mode().IsRegular() { return nil } f, err := os.Open(p) if err != nil { return fmt.Errorf("open %s: %w", p, err) } defer f.Close() if _, err := io.Copy(tw, f); err != nil { return fmt.Errorf("copy %s: %w", rel, err) } return nil } // DecodeDir extracts a tar.zst stream from r into dest. dest must exist. // Any entry whose normalized path escapes dest is rejected. // // Read-only entries are the normal case, not an exotic one: the Go module // cache is 0555 directories and 0444 files throughout. So directories are // created writable and their recorded mode is applied at the very end, // and files are written writable and chmod'ed after the copy. Restoring // the archived mode as we go would lock the extraction out of the tree it // is still filling in. func DecodeDir(r io.Reader, dest string) error { absDest, err := filepath.Abs(dest) if err != nil { return fmt.Errorf("abs %s: %w", dest, err) } zr, err := zstd.NewReader(r) if err != nil { return fmt.Errorf("zstd reader: %w", err) } defer zr.Close() tr := tar.NewReader(zr) // Directory modes, in archive order, applied once everything is in // place. Deepest-first on the way back out. var dirs []dirMode for { hdr, err := tr.Next() if errors.Is(err, io.EOF) { return restoreDirModes(dirs) } if err != nil { return fmt.Errorf("tar next: %w", err) } target, err := safeJoin(absDest, hdr.Name) if err != nil { return err } mode := fs.FileMode(hdr.Mode) & 0o7777 switch hdr.Typeflag { case tar.TypeDir: if err := os.MkdirAll(target, mode|0o700); err != nil { return fmt.Errorf("mkdir %s: %w", target, err) } dirs = append(dirs, dirMode{path: target, mode: mode}) case tar.TypeReg, tar.TypeRegA: if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { return fmt.Errorf("mkdir parent of %s: %w", target, err) } f, err := createFile(target, mode) if err != nil { return fmt.Errorf("create %s: %w", target, err) } if _, err := io.Copy(f, tr); err != nil { f.Close() return fmt.Errorf("write %s: %w", target, err) } if err := f.Close(); err != nil { return err } if err := os.Chmod(target, mode); err != nil { return fmt.Errorf("chmod %s: %w", target, err) } case tar.TypeSymlink: _ = os.Remove(target) if err := os.Symlink(hdr.Linkname, target); err != nil { return fmt.Errorf("symlink %s -> %s: %w", target, hdr.Linkname, err) } default: // Skip unknown entry types silently. } } } // dirMode is a directory whose archived mode is applied after extraction. type dirMode struct { path string mode fs.FileMode } // restoreDirModes applies the archived modes deepest-first: entries arrive // parent-before-child, so walking back tightens a directory only after // everything under it is already done. func restoreDirModes(dirs []dirMode) error { for i := len(dirs) - 1; i >= 0; i-- { if err := os.Chmod(dirs[i].path, dirs[i].mode); err != nil { return fmt.Errorf("chmod %s: %w", dirs[i].path, err) } } return nil } // createFile opens target for writing, forcing the owner-write bit on so a // read-only archived mode doesn't stop us writing the content. The caller // chmods to the archived mode afterwards. A read-only file left behind by // an earlier extraction is replaced rather than treated as an error — // re-restoring over a populated module cache is a normal CI situation. func createFile(target string, mode fs.FileMode) (*os.File, error) { f, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode|0o200) if err == nil || !errors.Is(err, fs.ErrPermission) { return f, err } if rmErr := os.Remove(target); rmErr != nil { // The parent directory may itself be read-only; widen it and // retry once before giving up with the original error. parent := filepath.Dir(target) st, stErr := os.Stat(parent) if stErr != nil { return nil, err } if chErr := os.Chmod(parent, st.Mode().Perm()|0o700); chErr != nil { return nil, err } if rmErr := os.Remove(target); rmErr != nil { return nil, err } } return os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode|0o200) } // safeJoin returns base/sub, rejecting any sub that is absolute or contains // ".." segments that would resolve outside base. We reject rather than // silently re-root so malicious tarballs surface as an error. func safeJoin(base, sub string) (string, error) { if filepath.IsAbs(sub) || strings.HasPrefix(sub, "/") { return "", fmt.Errorf("absolute tar entry: %q", sub) } clean := filepath.Clean(sub) if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { return "", fmt.Errorf("tar entry escapes destination: %q", sub) } return filepath.Join(base, clean), nil }