@@ 3,9 3,11 @@ package gozstd
import (
"bytes"
"fmt"
+ "math/rand"
"os"
"os/exec"
"path/filepath"
+ "sync"
"testing"
)
@@ 172,6 174,237 @@ func TestShimRoundTripDict(t *testing.T) {
}
}
+// TestCompressReadableByLibzstd proves the reverse interop direction: frames the
+// shim produces are decodable by upstream libzstd (the zstd CLI), both plain and
+// dictionary-compressed. dolt never consumes shim-written archives at runtime,
+// but this confirms the shim emits standard RFC-8878 frames, not a klauspost
+// dialect.
+func TestCompressReadableByLibzstd(t *testing.T) {
+ zstd := zstdBin(t)
+ dir := t.TempDir()
+ orig := bytes.Repeat([]byte("shim output must be standard zstd\n"), 300)
+
+ // plain: shim compresses, zstd -d decompresses.
+ comp := Compress(nil, orig)
+ czst := filepath.Join(dir, "c.zst")
+ if err := os.WriteFile(czst, comp, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ cout := filepath.Join(dir, "c.out")
+ run(t, zstd, "-q", "-d", czst, "-o", cout)
+ if got, _ := os.ReadFile(cout); !bytes.Equal(got, orig) {
+ t.Fatal("plain: libzstd could not round-trip shim output")
+ }
+
+ // dict: shim compresses with a libzstd-trained dict, zstd -d -D decompresses.
+ var samples []string
+ for i := 0; i < 512; i++ {
+ p := filepath.Join(dir, fmt.Sprintf("s%03d", i))
+ if err := os.WriteFile(p, []byte(fmt.Sprintf("bead memestudio-%04d status open priority %d\n", i, i%4)), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ samples = append(samples, p)
+ }
+ dictPath := filepath.Join(dir, "dict")
+ if out, err := exec.Command(zstd, append([]string{"--train", "--maxdict=8192", "-o", dictPath}, samples...)...).CombinedOutput(); err != nil {
+ t.Skipf("zstd --train failed (%v): %s", err, out)
+ }
+ dict, err := os.ReadFile(dictPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ cd, err := NewCDict(dict)
+ if err != nil {
+ t.Fatalf("NewCDict: %v", err)
+ }
+ defer cd.Release()
+ dorig := bytes.Repeat([]byte("bead memestudio-0042 status open priority 2\n"), 40)
+ dcomp := CompressDict(nil, dorig, cd)
+ dzst := filepath.Join(dir, "d.zst")
+ if err := os.WriteFile(dzst, dcomp, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ dout := filepath.Join(dir, "d.out")
+ run(t, zstd, "-q", "-d", "-D", dictPath, dzst, "-o", dout)
+ if got, _ := os.ReadFile(dout); !bytes.Equal(got, dorig) {
+ t.Fatal("dict: libzstd could not round-trip shim dict output")
+ }
+}
+
+// TestAppendSemantics verifies the load-bearing dst-append contract: gozstd's
+// Compress/Decompress/CompressDict/DecompressDict append to dst and return the
+// grown slice, preserving any existing prefix. dolt passes non-empty buffers, so
+// a klauspost mismatch here would silently corrupt its data.
+func TestAppendSemantics(t *testing.T) {
+ orig := []byte("append into an existing buffer without clobbering the prefix")
+ prefix := []byte("PREFIX|")
+
+ comp := Compress(append([]byte{}, prefix...), orig)
+ if !bytes.HasPrefix(comp, prefix) {
+ t.Fatal("Compress dropped the dst prefix")
+ }
+ got, err := Decompress(append([]byte{}, prefix...), comp[len(prefix):])
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(got, append(append([]byte{}, prefix...), orig...)) {
+ t.Fatal("Decompress did not append after the dst prefix")
+ }
+
+ // same for the dict variants
+ cd, dd := shimDict(t)
+ defer cd.Release()
+ defer dd.Release()
+ dc := CompressDict(append([]byte{}, prefix...), orig, cd)
+ if !bytes.HasPrefix(dc, prefix) {
+ t.Fatal("CompressDict dropped the dst prefix")
+ }
+ dg, err := DecompressDict(append([]byte{}, prefix...), dc[len(prefix):], dd)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(dg, append(append([]byte{}, prefix...), orig...)) {
+ t.Fatal("DecompressDict did not append after the dst prefix")
+ }
+}
+
+// TestRoundTripEdgeCases covers empty, tiny, incompressible-random, and large
+// inputs through the plain path.
+func TestRoundTripEdgeCases(t *testing.T) {
+ rng := rand.New(rand.NewSource(1)) // deterministic
+ big := make([]byte, 4<<20)
+ rng.Read(big)
+ cases := map[string][]byte{
+ "empty": {},
+ "one-byte": {0x42},
+ "incompressible": randBytes(rng, 64<<10),
+ "large-random": big,
+ "large-text": bytes.Repeat([]byte("mardi gras parade "), 300000),
+ }
+ for name, orig := range cases {
+ got, err := Decompress(nil, Compress(nil, orig))
+ if err != nil {
+ t.Fatalf("%s: Decompress: %v", name, err)
+ }
+ if !bytes.Equal(got, orig) {
+ t.Fatalf("%s: round-trip mismatch (%d vs %d bytes)", name, len(got), len(orig))
+ }
+ }
+}
+
+// TestConcurrent exercises the concurrency-safety the shim claims (dolt calls
+// these from many goroutines): shared coders for plain Compress/Decompress and a
+// single shared DDict for DecompressDict.
+func TestConcurrent(t *testing.T) {
+ cd, dd := shimDict(t)
+ defer cd.Release()
+ defer dd.Release()
+
+ payload := bytes.Repeat([]byte("concurrent parade "), 500)
+ dictFrame := CompressDict(nil, payload, cd)
+
+ const n = 64
+ errs := make(chan error, n)
+ var wg sync.WaitGroup
+ for i := 0; i < n; i++ {
+ wg.Add(1)
+ go func(i int) {
+ defer wg.Done()
+ // plain round-trip on distinct data
+ src := []byte(fmt.Sprintf("goroutine-%d-%s", i, payload))
+ if got, err := Decompress(nil, Compress(nil, src)); err != nil || !bytes.Equal(got, src) {
+ errs <- fmt.Errorf("plain g%d: err=%v equal=%v", i, err, bytes.Equal(got, src))
+ return
+ }
+ // concurrent DecompressDict on the shared DDict
+ if got, err := DecompressDict(nil, dictFrame, dd); err != nil || !bytes.Equal(got, payload) {
+ errs <- fmt.Errorf("dict g%d: err=%v equal=%v", i, err, bytes.Equal(got, payload))
+ return
+ }
+ }(i)
+ }
+ wg.Wait()
+ close(errs)
+ for err := range errs {
+ t.Error(err)
+ }
+}
+
+// TestErrorPaths confirms bad input yields an error, never a panic — dolt's
+// archive reader handles the error return.
+func TestErrorPaths(t *testing.T) {
+ if _, err := Decompress(nil, []byte("not a zstd frame at all")); err == nil {
+ t.Fatal("Decompress of garbage: want error, got nil")
+ }
+ // Empty input is a benign divergence from libzstd (which errors): klauspost
+ // yields empty output, no error, no panic. dolt never decompresses empty
+ // input, so this is documented rather than treated as a fault.
+ if got, err := Decompress(nil, nil); err != nil || len(got) != 0 {
+ t.Fatalf("Decompress of empty input: want (empty, nil), got (%d bytes, %v)", len(got), err)
+ }
+ // A frame compressed with dict A must not decode under dict B (dict-ID
+ // mismatch) — and must fail cleanly rather than return wrong bytes.
+ a := buildShimDict(t, "alpha")
+ b := buildShimDict(t, "bravo")
+ cdA, _ := NewCDict(a)
+ defer cdA.Release()
+ ddB, _ := NewDDict(b)
+ defer ddB.Release()
+ frame := CompressDict(nil, bytes.Repeat([]byte("alpha payload "), 50), cdA)
+ if _, err := DecompressDict(nil, frame, ddB); err == nil {
+ t.Fatal("DecompressDict under the wrong dict: want error, got nil")
+ }
+}
+
+// --- test helpers ---
+
+// shimDict builds a matching CDict/DDict pair from a libzstd-trained dictionary
+// (via the zstd CLI); it skips the calling test if the CLI is unavailable.
+func shimDict(t *testing.T) (*CDict, *DDict) {
+ t.Helper()
+ d := buildShimDict(t, "shared")
+ cd, err := NewCDict(d)
+ if err != nil {
+ t.Fatalf("NewCDict: %v", err)
+ }
+ dd, err := NewDDict(d)
+ if err != nil {
+ t.Fatalf("NewDDict: %v", err)
+ }
+ return cd, dd
+}
+
+// buildShimDict trains a distinct structured dictionary via the zstd CLI, seeded
+// by tag so different tags yield different dict-IDs.
+func buildShimDict(t *testing.T, tag string) []byte {
+ t.Helper()
+ zstd := zstdBin(t)
+ dir := t.TempDir()
+ var samples []string
+ for i := 0; i < 512; i++ {
+ p := filepath.Join(dir, fmt.Sprintf("s%03d", i))
+ if err := os.WriteFile(p, []byte(fmt.Sprintf("%s bead-%04d status open priority %d assignee eugene\n", tag, i, i%4)), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ samples = append(samples, p)
+ }
+ dictPath := filepath.Join(dir, "dict")
+ if out, err := exec.Command(zstd, append([]string{"--train", "--maxdict=8192", "-o", dictPath}, samples...)...).CombinedOutput(); err != nil {
+ t.Skipf("zstd --train failed (%v): %s", err, out)
+ }
+ d, err := os.ReadFile(dictPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return d
+}
+
+func randBytes(rng *rand.Rand, n int) []byte {
+ b := make([]byte, n)
+ rng.Read(b)
+ return b
+}
+
func run(t *testing.T, name string, args ...string) {
t.Helper()
if out, err := exec.Command(name, args...).CombinedOutput(); err != nil {