From 8ed47c67fd2f11b540cb8caa4c84407834b973af Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Sun, 19 Jul 2026 10:15:25 +0300 Subject: [PATCH] build: pure-Go (CGO_ENABLED=0) build via a klauspost-backed gozstd shim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dolthub/dolt/go pulls in two hard cgo dependencies — go-icu-regex (SQL REGEXP) and gozstd (NBS zstd compression) — which forced a C toolchain + ICU headers on every build. Both are now avoided so the default build is pure Go and statically linkable: - ICU: build with `-tags gms_pure_go`, selecting go-mysql-server's stdlib regexp fallback. Safe because this service never runs the SQL engine (it serves bare NBS stores and browses read-only), so it never evaluates SQL REGEXP. - zstd: `replace github.com/dolthub/gozstd => ./third_party/gozstd-purego`, a pure-Go drop-in over klauspost/compress/zstd (already in the graph). It reproduces the nine gozstd symbols dolt references. dolt is unmodified. dolt uses gozstd only in its NBS archive subsystem; this binary hits only the decompress side at runtime (archive dictionary TRAINING is gc/ archive-writer code we never run — the shim implements it over klauspost but panics on the trainer errors that only that off-path use could trigger). zstd frames and dictionaries are standard-format, so libzstd-authored archives decode correctly; the shim's tests prove this by decoding plain and dictionary-compressed frames produced by the zstd CLI (libzstd). The Makefile now defaults to CGO_ENABLED=0 + -tags gms_pure_go (override with `make CGO_ENABLED=1 GO_TAGS=` for the cgo variant). Verified: CGO_ENABLED=0 build of ./..., all unit tests, the real-dolt-CLI integration + spike suites, and the shim's libzstd-interop tests, all green with no cgo. --- Makefile | 22 ++- README.md | 46 ++++-- go.mod | 3 + go.sum | 2 - third_party/gozstd-purego/go.mod | 6 + third_party/gozstd-purego/go.sum | 2 + third_party/gozstd-purego/gozstd.go | 189 +++++++++++++++++++++++ third_party/gozstd-purego/gozstd_test.go | 180 +++++++++++++++++++++ 8 files changed, 428 insertions(+), 22 deletions(-) create mode 100644 third_party/gozstd-purego/go.mod create mode 100644 third_party/gozstd-purego/go.sum create mode 100644 third_party/gozstd-purego/gozstd.go create mode 100644 third_party/gozstd-purego/gozstd_test.go diff --git a/Makefile b/Makefile index d8ac66d2a79e772f9b89fecf365830d1593cea1f..a1512ca66724dca62a2c7b418b17dd2eb465109f 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,18 @@ SASSC?=sassc SASSC_INCLUDE=-I$(ASSETS)/scss/ MINIFY?=minify +# Pure-Go build (default): no cgo, no ICU/zstd C libraries, statically linkable. +# Two things make it work: the `gms_pure_go` tag swaps go-mysql-server's ICU +# regex for the stdlib regexp (this service never runs the SQL engine), and a +# `replace github.com/dolthub/gozstd => ./third_party/gozstd-purego` directive +# backs dolt's only other hard cgo dependency with a pure-Go klauspost shim. +# To build the cgo variant instead: `make CGO_ENABLED=1 GO_TAGS=`. +GO_TAGS?=gms_pure_go +CGO_ENABLED?=0 +export CGO_ENABLED +GO=go +GOBUILD=$(GO) build $(if $(GO_TAGS),-tags "$(GO_TAGS)",) + BINARIES=\ doltsrht \ doltsrht-migrate @@ -32,23 +44,23 @@ css: all-share # Phase 3; until then these targets are no-ops rather than hard failures. doltsrht: @if [ -d ./cmd/doltsrht ]; then \ - echo "go build -o $@ ./cmd/doltsrht"; \ - go build -o $@ ./cmd/doltsrht; \ + echo "$(GOBUILD) -o $@ ./cmd/doltsrht"; \ + $(GOBUILD) -o $@ ./cmd/doltsrht; \ else \ echo "skip $@: ./cmd/doltsrht not present yet"; \ fi doltsrht-migrate: @if [ -d ./cmd/doltsrht-migrate ]; then \ - echo "go build -o $@ ./cmd/doltsrht-migrate"; \ - go build -o $@ ./cmd/doltsrht-migrate; \ + echo "$(GOBUILD) -o $@ ./cmd/doltsrht-migrate"; \ + $(GOBUILD) -o $@ ./cmd/doltsrht-migrate; \ else \ echo "skip $@: ./cmd/doltsrht-migrate not present yet"; \ fi # Compile every buildable package; used as the CI build gate. build: - go build ./... + $(GOBUILD) ./... install: install-bin install-share diff --git a/README.md b/README.md index ba928152badcc782cc02c563891fba6a5366e14d..8c3c8ffc0925fbecb507e6fc17fb0bfb54900b15 100644 --- a/README.md +++ b/README.md @@ -32,18 +32,26 @@ remotesrv assembly plus CredentialsService, the read-only `browse/` UI, the ## Build prerequisites - **Go 1.26+** -- **A C toolchain** — `github.com/dolthub/dolt/go` uses CGO for - [`gozstd`](https://github.com/valyala/gozstd) and - [`go-icu-regex`](https://github.com/dolthub/go-icu-regex). -- **ICU4C development headers** — required by `go-icu-regex`. - - Debian/Ubuntu: `apt install libicu-dev` (headers on the default path). - - macOS (Homebrew): `brew install icu4c` installs a keg-only, versioned - formula. Point CGO at it, e.g. for `icu4c@78`: - - ```sh - export CGO_CPPFLAGS="-I/opt/homebrew/opt/icu4c@78/include" - export CGO_LDFLAGS="-L/opt/homebrew/opt/icu4c@78/lib" - ``` +- **No C toolchain, no ICU, no zstd headers.** The default build is pure Go + (`CGO_ENABLED=0`, statically linkable). `github.com/dolthub/dolt/go` normally + needs CGO for two libraries; both are avoided: + - **ICU regex** — the `gms_pure_go` build tag selects go-mysql-server's stdlib + `regexp` fallback instead of `go-icu-regex`. Safe here because this service + never runs the SQL engine (it serves bare NBS stores and browses read-only), + so it never evaluates SQL `REGEXP`. + - **zstd** — a `replace github.com/dolthub/gozstd => ./third_party/gozstd-purego` + directive backs dolt's zstd dependency with a pure-Go shim over + [`klauspost/compress/zstd`](https://github.com/klauspost/compress) (see that + directory's README/tests, incl. libzstd interop). dolt itself is unmodified. + + `make` and `make build` pass `-tags gms_pure_go` and `CGO_ENABLED=0` for you; a + bare `go build` needs `-tags gms_pure_go`. + + - **Optional cgo variant** (upstream gozstd + ICU): `make CGO_ENABLED=1 GO_TAGS=`. + It then needs a C toolchain and ICU4C headers — Debian/Ubuntu + `apt install libicu-dev`; macOS keg-only `brew install icu4c` with + `CGO_CPPFLAGS="-I/opt/homebrew/opt/icu4c@78/include"` / + `CGO_LDFLAGS="-L/opt/homebrew/opt/icu4c@78/lib"`. - **sassc + minify** — only for building CSS (`make css`); not needed for the default build: @@ -78,6 +86,16 @@ remotesrv assembly plus CredentialsService, the read-only `browse/` UI, the - **`gopkg.in/go-jose/go-jose.v2` v2.6.3** — the same JOSE major/version that `dolt/go`'s `creds` package uses to sign the EdDSA keypair JWTs, so the Bearer verify path stays byte-compatible and no duplicate JOSE lib is pulled in. +- **`github.com/dolthub/gozstd`, replaced by the local `./third_party/gozstd-purego` + shim** — a pure-Go, drop-in reimplementation of the nine gozstd symbols dolt + references (Compress/CompressDict/Decompress/DecompressDict/BuildDict, the + CDict/DDict types and their constructors), backed by + `github.com/klauspost/compress/zstd` (v1.18.0, already in the graph). This is + what lets the default build be `CGO_ENABLED=0`. dolt uses gozstd only from its + NBS archive subsystem; this service only ever hits the *decompress* side at + runtime, and zstd frames/dictionaries are standard-format, so libzstd-authored + archives decode correctly (proven by the shim's libzstd-interop tests). Keep + the `replace`; to drop it, build the cgo variant (see Build prerequisites). - grpc v1.79.3, logrus v1.8.3, lib/pq v1.10.9, chi/v5, and brant round out the transport, logging, Postgres driver, HTTP router, and migration tooling. @@ -92,9 +110,7 @@ verify the rows. It skips (does not fail) when the CLI is absent, and uses an isolated `$HOME` so your real dolt config is untouched. ```sh -export CGO_CPPFLAGS="-I/opt/homebrew/opt/icu4c@78/include" -export CGO_LDFLAGS="-L/opt/homebrew/opt/icu4c@78/lib" -go test -tags spike ./storage/ -run TestSpike -v +go test -tags 'gms_pure_go spike' ./storage/ -run TestSpike -v ``` ## Dev commands diff --git a/go.mod b/go.mod index 5bdda5a4b63e32978032fc0c6cb6edfeacda45c2..de33b0ebec606d0264b829a0b2ea5e742dff6426 100644 --- a/go.mod +++ b/go.mod @@ -112,6 +112,7 @@ require ( github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d // indirect github.com/kavu/go_reuseport v1.5.0 // indirect github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.0.12 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect @@ -163,3 +164,5 @@ require ( google.golang.org/protobuf v1.36.11 // indirect gopkg.in/src-d/go-errors.v1 v1.0.0 // indirect ) + +replace github.com/dolthub/gozstd => ./third_party/gozstd-purego diff --git a/go.sum b/go.sum index 15d9a505c1d33aabde8baec5fefe6b4f940f6626..aaa5c108bac85115b19ca18e61ba999dbce0226c 100644 --- a/go.sum +++ b/go.sum @@ -178,8 +178,6 @@ github.com/dolthub/go-icu-regex v0.0.0-20260610153742-72563bc7ca83 h1:FEMjCGEroD github.com/dolthub/go-icu-regex v0.0.0-20260610153742-72563bc7ca83/go.mod h1:F3cnm+vMRK1HaU6+rNqQrOCyR03HHhR1GWG2gnPOqaE= github.com/dolthub/go-mysql-server v0.20.1-0.20260625171506-68aec8237480 h1:LTH0FHVgS2Utgi/98xzfunuXl3dJckMAgM0RLUJAzjM= github.com/dolthub/go-mysql-server v0.20.1-0.20260625171506-68aec8237480/go.mod h1:mj5/QX3V8i92REbA1w6CzyknJAFdKtdE7l931405C/E= -github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI= -github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q= github.com/dolthub/ishell v0.0.0-20260414231531-5f031e3e9037 h1:oIW9HwuWrhxv+4HZxA+QQSKHLqWFyXZ2FmNjUYwkdiM= github.com/dolthub/ishell v0.0.0-20260414231531-5f031e3e9037/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 h1:bMGS25NWAGTEtT5tOBsCuCrlYnLRKpbJVJkDbrTRhwQ= diff --git a/third_party/gozstd-purego/go.mod b/third_party/gozstd-purego/go.mod new file mode 100644 index 0000000000000000000000000000000000000000..b23ebcc7d9bfdc5df660ff99644804631909e65b --- /dev/null +++ b/third_party/gozstd-purego/go.mod @@ -0,0 +1,6 @@ +// Pure-Go drop-in replacement for github.com/dolthub/gozstd. See gozstd.go. +module github.com/dolthub/gozstd + +go 1.23 + +require github.com/klauspost/compress v1.18.0 diff --git a/third_party/gozstd-purego/go.sum b/third_party/gozstd-purego/go.sum new file mode 100644 index 0000000000000000000000000000000000000000..7afd79f02efe8841160fdbd4f964834161843c19 --- /dev/null +++ b/third_party/gozstd-purego/go.sum @@ -0,0 +1,2 @@ +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= diff --git a/third_party/gozstd-purego/gozstd.go b/third_party/gozstd-purego/gozstd.go new file mode 100644 index 0000000000000000000000000000000000000000..9457b075f262ac33063bbd04a63d154148927e33 --- /dev/null +++ b/third_party/gozstd-purego/gozstd.go @@ -0,0 +1,189 @@ +// Package gozstd is a pure-Go, drop-in replacement for the subset of +// github.com/dolthub/gozstd that github.com/dolthub/dolt/go references, backed +// by github.com/klauspost/compress/zstd. It exists so this project can build +// with CGO_ENABLED=0: upstream gozstd bundles the zstd C source and requires +// cgo for every symbol, which is the last hard cgo dependency in the dolt +// import graph (ICU is already avoided via the gms_pure_go build tag). +// +// It is wired in via a `replace github.com/dolthub/gozstd => ./third_party/ +// gozstd-purego` directive; dolt itself is not modified. +// +// # Scope and correctness +// +// dolt uses gozstd only from its NBS "archive" subsystem (store/nbs/archive_*), +// and references exactly nine symbols: Compress, CompressDict, Decompress, +// DecompressDict, BuildDict, NewCDict, NewDDict, and the CDict/DDict types. This +// file reproduces those with identical signatures. Streaming (Writer/Reader), +// the Stream* helpers, CompressLevel and NewCDictLevel are part of upstream +// gozstd's API but are unused by dolt, so they are intentionally omitted. +// +// zstd is a standard frame format (RFC 8878) and dictionaries carry an embedded +// dict-ID, so frames and dictionaries produced by upstream libzstd (e.g. by the +// real `dolt` CLI) are decodable here and vice-versa. This project only ever +// runs the DECOMPRESS side at runtime (it serves chunks over remotesapi and +// browses read-only; it never builds archives or runs gc), so the compress and +// dictionary-training paths are compiled but never executed by the server. They +// are implemented faithfully regardless, so the swap stays correct if a future +// caller does reach them. +// +// Concurrency: klauspost's Encoder.EncodeAll and Decoder.DecodeAll are safe for +// concurrent use, matching gozstd's Compress*/Decompress* which dolt calls from +// multiple goroutines. +package gozstd + +import ( + "fmt" + "hash/fnv" + "sync" + + "github.com/klauspost/compress/zstd" +) + +// DefaultCompressionLevel mirrors upstream gozstd's default (zstd level 3). +const DefaultCompressionLevel = 3 + +// Shared, dictionary-less coders. klauspost's zstd Encoder/Decoder built over a +// nil stream are the documented one-shot EncodeAll/DecodeAll coders, and both +// are safe for concurrent use, so a single lazily-initialized instance backs +// all dictionary-less Compress/Decompress calls. +var ( + encOnce sync.Once + enc *zstd.Encoder + decOnce sync.Once + dec *zstd.Decoder +) + +func sharedEncoder() *zstd.Encoder { + encOnce.Do(func() { + e, err := zstd.NewWriter(nil, + zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(DefaultCompressionLevel))) + if err != nil { + panic(fmt.Errorf("gozstd-purego: new encoder: %w", err)) + } + enc = e + }) + return enc +} + +func sharedDecoder() *zstd.Decoder { + decOnce.Do(func() { + d, err := zstd.NewReader(nil) + if err != nil { + panic(fmt.Errorf("gozstd-purego: new decoder: %w", err)) + } + dec = d + }) + return dec +} + +// Compress appends the zstd-compressed form of src to dst and returns dst. +// Signature and append semantics match upstream gozstd.Compress. Note klauspost +// takes (src, dst) where gozstd takes (dst, src); both append to dst. +func Compress(dst, src []byte) []byte { + return sharedEncoder().EncodeAll(src, dst) +} + +// Decompress appends the decompressed form of src to dst and returns dst. +func Decompress(dst, src []byte) ([]byte, error) { + return sharedDecoder().DecodeAll(src, dst) +} + +// CDict is a compression dictionary. Upstream gozstd wraps an opaque libzstd +// ZSTD_CDict; here it holds a klauspost Encoder pre-loaded with the dictionary, +// which embeds the dictionary's ID into every frame it writes (as libzstd does). +type CDict struct { + enc *zstd.Encoder +} + +// NewCDict builds a CDict from a raw zstd dictionary at the default level. +func NewCDict(dict []byte) (*CDict, error) { + e, err := zstd.NewWriter(nil, + zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(DefaultCompressionLevel)), + zstd.WithEncoderDict(dict)) + if err != nil { + return nil, fmt.Errorf("gozstd-purego: NewCDict: %w", err) + } + return &CDict{enc: e}, nil +} + +// Release frees the dictionary's coder. Upstream gozstd requires this to free C +// memory; here it closes the klauspost Encoder. dolt does not currently call it, +// but it is provided for API parity and to release coder goroutines. Safe to +// call on nil. +func (cd *CDict) Release() { + if cd != nil && cd.enc != nil { + cd.enc.Close() + } +} + +// DDict is a decompression dictionary: a klauspost Decoder pre-loaded with the +// dictionary. DecodeAll selects it for frames whose embedded dict-ID matches. +type DDict struct { + dec *zstd.Decoder +} + +// NewDDict builds a DDict from a raw zstd dictionary. The dictionary is a +// standard structured (ZDICT) dictionary carrying magic + a dict-ID, which +// klauspost parses via WithDecoderDicts. +func NewDDict(dict []byte) (*DDict, error) { + d, err := zstd.NewReader(nil, zstd.WithDecoderDicts(dict)) + if err != nil { + return nil, fmt.Errorf("gozstd-purego: NewDDict: %w", err) + } + return &DDict{dec: d}, nil +} + +// Release frees the dictionary's coder. Safe to call on nil. +func (dd *DDict) Release() { + if dd != nil && dd.dec != nil { + dd.dec.Close() + } +} + +// CompressDict appends the dictionary-compressed form of src to dst using cd. +func CompressDict(dst, src []byte, cd *CDict) []byte { + return cd.enc.EncodeAll(src, dst) +} + +// DecompressDict appends the dictionary-decompressed form of src to dst using +// dd. The frame's embedded dict-ID must match dd's dictionary. +func DecompressDict(dst, src []byte, dd *DDict) ([]byte, error) { + return dd.dec.DecodeAll(src, dst) +} + +// BuildDict trains a zstd dictionary from samples. Upstream gozstd wraps +// libzstd's ZDICT_trainFromBuffer; klauspost's trainer uses a different +// algorithm, so the produced dictionary bytes differ, but the result is a valid +// standard zstd dictionary. This function is only reached by dolt's archive- +// writer / gc code, which this project never runs, so its output is never +// consumed here; it is implemented faithfully so the swap stays correct if a +// future caller does train a dictionary. desiredDictLen has no direct klauspost +// analogue (its trainer sizes the dictionary from the content) and is ignored. +// +// It panics on failure to match upstream gozstd, which likewise cannot return +// an error from this signature; a training failure is a programmer/data error, +// not a recoverable runtime condition, and — being off this project's serving +// path — must surface loudly rather than yield a silently-bad dictionary. +func BuildDict(samples [][]byte, desiredDictLen int) []byte { + _ = desiredDictLen + // Derive a stable, non-zero dict-ID from the samples so distinct inputs get + // distinct IDs (klauspost embeds this ID; libzstd would assign its own). + h := fnv.New32a() + for _, s := range samples { + _, _ = h.Write(s) + } + id := h.Sum32() + if id == 0 { + id = 1 + } + d, err := zstd.BuildDict(zstd.BuildDictOptions{ + ID: id, + Contents: samples, + // libzstd's default repcodes; a zeroed offset set is rejected. + Offsets: [3]int{1, 4, 8}, + }) + if err != nil { + panic(fmt.Errorf("gozstd-purego: BuildDict: %w", err)) + } + return d +} diff --git a/third_party/gozstd-purego/gozstd_test.go b/third_party/gozstd-purego/gozstd_test.go new file mode 100644 index 0000000000000000000000000000000000000000..38b2873c40889d4b24c976078e4b50f03c0c49c7 --- /dev/null +++ b/third_party/gozstd-purego/gozstd_test.go @@ -0,0 +1,180 @@ +package gozstd + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" +) + +// These tests prove the load-bearing interop property of the shim: the pure-Go +// klauspost backend decodes zstd frames produced by upstream libzstd — plain +// AND dictionary-compressed — which is exactly what dolt's NBS archive reader +// asks of gozstd at runtime (it decompresses .darc archives authored by the +// real `dolt`/libzstd). They shell out to the `zstd` CLI (libzstd) to author +// the fixtures and skip cleanly when it is absent, so CI without the tool still +// passes. The internal round-trip test needs no external tool. + +func zstdBin(t *testing.T) string { + t.Helper() + p, err := exec.LookPath("zstd") + if err != nil { + t.Skip("zstd CLI (libzstd) not found; skipping libzstd-interop test") + } + return p +} + +// TestDecompressLibzstdPlain: libzstd compresses, the shim decompresses. +func TestDecompressLibzstdPlain(t *testing.T) { + zstd := zstdBin(t) + orig := bytes.Repeat([]byte("the quick brown fox jumps over the lazy dog\n"), 500) + + dir := t.TempDir() + in := filepath.Join(dir, "in") + out := filepath.Join(dir, "in.zst") + if err := os.WriteFile(in, orig, 0o644); err != nil { + t.Fatal(err) + } + run(t, zstd, "-q", "-19", in, "-o", out) + + comp, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + got, err := Decompress(nil, comp) + if err != nil { + t.Fatalf("Decompress: %v", err) + } + if !bytes.Equal(got, orig) { + t.Fatalf("plain: round-trip mismatch (%d vs %d bytes)", len(got), len(orig)) + } +} + +// TestDecompressDictLibzstd: libzstd trains a dictionary and compresses with it, +// then the shim decompresses with the same dictionary via NewDDict — +// the exact archive-read path dolt exercises. This is the interop risk the +// feasibility review flagged (klauspost matches decoder dicts by embedded +// dict-ID, structured ZDICT format), proven end-to-end here. +func TestDecompressDictLibzstd(t *testing.T) { + zstd := zstdBin(t) + dir := t.TempDir() + + // Enough varied-but-similar samples for the trainer to build a dictionary. + var samples []string + for i := 0; i < 512; i++ { + p := filepath.Join(dir, fmt.Sprintf("s%03d", i)) + line := fmt.Sprintf("issue memestudio-%04d: lorem ipsum dolor sit amet consectetur %d\n", i, i*7) + if err := os.WriteFile(p, []byte(line), 0o644); err != nil { + t.Fatal(err) + } + samples = append(samples, p) + } + dictPath := filepath.Join(dir, "dict") + if out, err := exec.Command(zstd, append([]string{"--train", "--maxdict=8192", "-o", dictPath}, samples...)...).CombinedOutput(); err != nil { + t.Skipf("zstd --train unavailable/failed (%v): %s", err, out) + } + dict, err := os.ReadFile(dictPath) + if err != nil { + t.Fatalf("read trained dict: %v", err) + } + + orig := []byte("issue memestudio-0042: lorem ipsum dolor sit amet consectetur 294 — payload body\n") + in := filepath.Join(dir, "payload") + out := filepath.Join(dir, "payload.zst") + if err := os.WriteFile(in, orig, 0o644); err != nil { + t.Fatal(err) + } + run(t, zstd, "-q", "-19", "-D", dictPath, in, "-o", out) + + comp, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + + dd, err := NewDDict(dict) + if err != nil { + t.Fatalf("NewDDict on libzstd-trained dict: %v", err) + } + defer dd.Release() + got, err := DecompressDict(nil, comp, dd) + if err != nil { + t.Fatalf("DecompressDict on libzstd frame: %v", err) + } + if !bytes.Equal(got, orig) { + t.Fatalf("dict: round-trip mismatch\n got=%q\nwant=%q", got, orig) + } +} + +// TestShimRoundTripPlain exercises the shim's own compress+decompress with no +// external tool and no dictionary. +func TestShimRoundTripPlain(t *testing.T) { + orig := bytes.Repeat([]byte("parade lane rolling lined-up stalled past-stand\n"), 200) + comp := Compress(nil, orig) + got, err := Decompress(nil, comp) + if err != nil { + t.Fatalf("Decompress: %v", err) + } + if !bytes.Equal(got, orig) { + t.Fatal("plain shim round-trip mismatch") + } +} + +// TestShimRoundTripDict exercises the shim's CompressDict + DecompressDict over +// a real structured dictionary. The dictionary is trained by libzstd (the zstd +// CLI) rather than the shim's own BuildDict, because BuildDict is off this +// project's runtime path (dolt only calls it from archive-writer/gc, which this +// binary never runs) and klauspost's trainer is stricter about corpus size than +// libzstd's ZDICT — training quality is irrelevant here; what matters is that +// the shim's encode+decode agree over a standard dictionary. +func TestShimRoundTripDict(t *testing.T) { + zstd := zstdBin(t) + dir := t.TempDir() + + var samples []string + for i := 0; i < 512; i++ { + p := filepath.Join(dir, fmt.Sprintf("s%03d", i)) + line := fmt.Sprintf("bead memestudio-%04d status open priority %d assignee eugene\n", i, i%4) + if err := os.WriteFile(p, []byte(line), 0o644); err != nil { + t.Fatal(err) + } + samples = append(samples, p) + } + dictPath := filepath.Join(dir, "dict") + if out, err := exec.Command(zstd, append([]string{"--train", "--maxdict=8192", "-o", dictPath}, samples...)...).CombinedOutput(); err != nil { + t.Skipf("zstd --train unavailable/failed (%v): %s", err, out) + } + dict, err := os.ReadFile(dictPath) + if err != nil { + t.Fatal(err) + } + + orig := bytes.Repeat([]byte("bead memestudio-0042 status open priority 2 assignee eugene\n"), 50) + cd, err := NewCDict(dict) + if err != nil { + t.Fatalf("NewCDict: %v", err) + } + defer cd.Release() + dd, err := NewDDict(dict) + if err != nil { + t.Fatalf("NewDDict: %v", err) + } + defer dd.Release() + + dcomp := CompressDict(nil, orig, cd) + dgot, err := DecompressDict(nil, dcomp, dd) + if err != nil { + t.Fatalf("DecompressDict: %v", err) + } + if !bytes.Equal(dgot, orig) { + t.Fatal("dict shim round-trip mismatch") + } +} + +func run(t *testing.T, name string, args ...string) { + t.Helper() + if out, err := exec.Command(name, args...).CombinedOutput(); err != nil { + t.Fatalf("%s %v: %v\n%s", name, args, err, out) + } +}