From c6172db991591e8bf432e27c66b5ce640076fe68 Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Sun, 19 Jul 2026 10:20:05 +0300 Subject: [PATCH] test(gozstd-purego): expand shim coverage (interop, concurrency, edges) Add to the pure-Go zstd shim's suite: reverse interop (shim output decoded by the libzstd `zstd` CLI, plain and with a dictionary); the dst-append contract for all four Compress/Decompress[Dict] funcs (dolt passes non-empty buffers); edge cases (empty, one-byte, incompressible random, 4 MiB random, large text); race-clean concurrency over the shared plain coders and a shared DDict; and error paths (garbage input errors, empty input is a documented benign no-op, wrong-dictionary decode errors rather than returning wrong bytes). All pass under -race. --- third_party/gozstd-purego/gozstd_test.go | 233 +++++++++++++++++++++++ 1 file changed, 233 insertions(+) diff --git a/third_party/gozstd-purego/gozstd_test.go b/third_party/gozstd-purego/gozstd_test.go index 38b2873c40889d4b24c976078e4b50f03c0c49c7..5dbee7b3c37746d7f71a67089f4affe92f8d74cc 100644 --- a/third_party/gozstd-purego/gozstd_test.go +++ b/third_party/gozstd-purego/gozstd_test.go @@ -3,9 +3,11 @@ package gozstd import ( "bytes" "fmt" + "math/rand" "os" "os/exec" "path/filepath" + "sync" "testing" ) @@ -172,6 +174,237 @@ func TestShimRoundTripDict(t *testing.T) { } } +// TestCompressReadableByLibzstd proves the reverse interop direction: frames the +// shim produces are decodable by upstream libzstd (the zstd CLI), both plain and +// dictionary-compressed. dolt never consumes shim-written archives at runtime, +// but this confirms the shim emits standard RFC-8878 frames, not a klauspost +// dialect. +func TestCompressReadableByLibzstd(t *testing.T) { + zstd := zstdBin(t) + dir := t.TempDir() + orig := bytes.Repeat([]byte("shim output must be standard zstd\n"), 300) + + // plain: shim compresses, zstd -d decompresses. + comp := Compress(nil, orig) + czst := filepath.Join(dir, "c.zst") + if err := os.WriteFile(czst, comp, 0o644); err != nil { + t.Fatal(err) + } + cout := filepath.Join(dir, "c.out") + run(t, zstd, "-q", "-d", czst, "-o", cout) + if got, _ := os.ReadFile(cout); !bytes.Equal(got, orig) { + t.Fatal("plain: libzstd could not round-trip shim output") + } + + // dict: shim compresses with a libzstd-trained dict, zstd -d -D decompresses. + var samples []string + for i := 0; i < 512; i++ { + p := filepath.Join(dir, fmt.Sprintf("s%03d", i)) + if err := os.WriteFile(p, []byte(fmt.Sprintf("bead memestudio-%04d status open priority %d\n", i, i%4)), 0o644); err != nil { + t.Fatal(err) + } + samples = append(samples, p) + } + dictPath := filepath.Join(dir, "dict") + if out, err := exec.Command(zstd, append([]string{"--train", "--maxdict=8192", "-o", dictPath}, samples...)...).CombinedOutput(); err != nil { + t.Skipf("zstd --train failed (%v): %s", err, out) + } + dict, err := os.ReadFile(dictPath) + if err != nil { + t.Fatal(err) + } + cd, err := NewCDict(dict) + if err != nil { + t.Fatalf("NewCDict: %v", err) + } + defer cd.Release() + dorig := bytes.Repeat([]byte("bead memestudio-0042 status open priority 2\n"), 40) + dcomp := CompressDict(nil, dorig, cd) + dzst := filepath.Join(dir, "d.zst") + if err := os.WriteFile(dzst, dcomp, 0o644); err != nil { + t.Fatal(err) + } + dout := filepath.Join(dir, "d.out") + run(t, zstd, "-q", "-d", "-D", dictPath, dzst, "-o", dout) + if got, _ := os.ReadFile(dout); !bytes.Equal(got, dorig) { + t.Fatal("dict: libzstd could not round-trip shim dict output") + } +} + +// TestAppendSemantics verifies the load-bearing dst-append contract: gozstd's +// Compress/Decompress/CompressDict/DecompressDict append to dst and return the +// grown slice, preserving any existing prefix. dolt passes non-empty buffers, so +// a klauspost mismatch here would silently corrupt its data. +func TestAppendSemantics(t *testing.T) { + orig := []byte("append into an existing buffer without clobbering the prefix") + prefix := []byte("PREFIX|") + + comp := Compress(append([]byte{}, prefix...), orig) + if !bytes.HasPrefix(comp, prefix) { + t.Fatal("Compress dropped the dst prefix") + } + got, err := Decompress(append([]byte{}, prefix...), comp[len(prefix):]) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, append(append([]byte{}, prefix...), orig...)) { + t.Fatal("Decompress did not append after the dst prefix") + } + + // same for the dict variants + cd, dd := shimDict(t) + defer cd.Release() + defer dd.Release() + dc := CompressDict(append([]byte{}, prefix...), orig, cd) + if !bytes.HasPrefix(dc, prefix) { + t.Fatal("CompressDict dropped the dst prefix") + } + dg, err := DecompressDict(append([]byte{}, prefix...), dc[len(prefix):], dd) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(dg, append(append([]byte{}, prefix...), orig...)) { + t.Fatal("DecompressDict did not append after the dst prefix") + } +} + +// TestRoundTripEdgeCases covers empty, tiny, incompressible-random, and large +// inputs through the plain path. +func TestRoundTripEdgeCases(t *testing.T) { + rng := rand.New(rand.NewSource(1)) // deterministic + big := make([]byte, 4<<20) + rng.Read(big) + cases := map[string][]byte{ + "empty": {}, + "one-byte": {0x42}, + "incompressible": randBytes(rng, 64<<10), + "large-random": big, + "large-text": bytes.Repeat([]byte("mardi gras parade "), 300000), + } + for name, orig := range cases { + got, err := Decompress(nil, Compress(nil, orig)) + if err != nil { + t.Fatalf("%s: Decompress: %v", name, err) + } + if !bytes.Equal(got, orig) { + t.Fatalf("%s: round-trip mismatch (%d vs %d bytes)", name, len(got), len(orig)) + } + } +} + +// TestConcurrent exercises the concurrency-safety the shim claims (dolt calls +// these from many goroutines): shared coders for plain Compress/Decompress and a +// single shared DDict for DecompressDict. +func TestConcurrent(t *testing.T) { + cd, dd := shimDict(t) + defer cd.Release() + defer dd.Release() + + payload := bytes.Repeat([]byte("concurrent parade "), 500) + dictFrame := CompressDict(nil, payload, cd) + + const n = 64 + errs := make(chan error, n) + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + // plain round-trip on distinct data + src := []byte(fmt.Sprintf("goroutine-%d-%s", i, payload)) + if got, err := Decompress(nil, Compress(nil, src)); err != nil || !bytes.Equal(got, src) { + errs <- fmt.Errorf("plain g%d: err=%v equal=%v", i, err, bytes.Equal(got, src)) + return + } + // concurrent DecompressDict on the shared DDict + if got, err := DecompressDict(nil, dictFrame, dd); err != nil || !bytes.Equal(got, payload) { + errs <- fmt.Errorf("dict g%d: err=%v equal=%v", i, err, bytes.Equal(got, payload)) + return + } + }(i) + } + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } +} + +// TestErrorPaths confirms bad input yields an error, never a panic — dolt's +// archive reader handles the error return. +func TestErrorPaths(t *testing.T) { + if _, err := Decompress(nil, []byte("not a zstd frame at all")); err == nil { + t.Fatal("Decompress of garbage: want error, got nil") + } + // Empty input is a benign divergence from libzstd (which errors): klauspost + // yields empty output, no error, no panic. dolt never decompresses empty + // input, so this is documented rather than treated as a fault. + if got, err := Decompress(nil, nil); err != nil || len(got) != 0 { + t.Fatalf("Decompress of empty input: want (empty, nil), got (%d bytes, %v)", len(got), err) + } + // A frame compressed with dict A must not decode under dict B (dict-ID + // mismatch) — and must fail cleanly rather than return wrong bytes. + a := buildShimDict(t, "alpha") + b := buildShimDict(t, "bravo") + cdA, _ := NewCDict(a) + defer cdA.Release() + ddB, _ := NewDDict(b) + defer ddB.Release() + frame := CompressDict(nil, bytes.Repeat([]byte("alpha payload "), 50), cdA) + if _, err := DecompressDict(nil, frame, ddB); err == nil { + t.Fatal("DecompressDict under the wrong dict: want error, got nil") + } +} + +// --- test helpers --- + +// shimDict builds a matching CDict/DDict pair from a libzstd-trained dictionary +// (via the zstd CLI); it skips the calling test if the CLI is unavailable. +func shimDict(t *testing.T) (*CDict, *DDict) { + t.Helper() + d := buildShimDict(t, "shared") + cd, err := NewCDict(d) + if err != nil { + t.Fatalf("NewCDict: %v", err) + } + dd, err := NewDDict(d) + if err != nil { + t.Fatalf("NewDDict: %v", err) + } + return cd, dd +} + +// buildShimDict trains a distinct structured dictionary via the zstd CLI, seeded +// by tag so different tags yield different dict-IDs. +func buildShimDict(t *testing.T, tag string) []byte { + t.Helper() + zstd := zstdBin(t) + dir := t.TempDir() + var samples []string + for i := 0; i < 512; i++ { + p := filepath.Join(dir, fmt.Sprintf("s%03d", i)) + if err := os.WriteFile(p, []byte(fmt.Sprintf("%s bead-%04d status open priority %d assignee eugene\n", tag, i, i%4)), 0o644); err != nil { + t.Fatal(err) + } + samples = append(samples, p) + } + dictPath := filepath.Join(dir, "dict") + if out, err := exec.Command(zstd, append([]string{"--train", "--maxdict=8192", "-o", dictPath}, samples...)...).CombinedOutput(); err != nil { + t.Skipf("zstd --train failed (%v): %s", err, out) + } + d, err := os.ReadFile(dictPath) + if err != nil { + t.Fatal(err) + } + return d +} + +func randBytes(rng *rand.Rand, n int) []byte { + b := make([]byte, n) + rng.Read(b) + return b +} + func run(t *testing.T, name string, args ...string) { t.Helper() if out, err := exec.Command(name, args...).CombinedOutput(); err != nil {