package gozstd import ( "bytes" "fmt" "os" "os/exec" "path/filepath" "testing" ) // These tests prove the load-bearing interop property of the shim: the pure-Go // klauspost backend decodes zstd frames produced by upstream libzstd — plain // AND dictionary-compressed — which is exactly what dolt's NBS archive reader // asks of gozstd at runtime (it decompresses .darc archives authored by the // real `dolt`/libzstd). They shell out to the `zstd` CLI (libzstd) to author // the fixtures and skip cleanly when it is absent, so CI without the tool still // passes. The internal round-trip test needs no external tool. func zstdBin(t *testing.T) string { t.Helper() p, err := exec.LookPath("zstd") if err != nil { t.Skip("zstd CLI (libzstd) not found; skipping libzstd-interop test") } return p } // TestDecompressLibzstdPlain: libzstd compresses, the shim decompresses. func TestDecompressLibzstdPlain(t *testing.T) { zstd := zstdBin(t) orig := bytes.Repeat([]byte("the quick brown fox jumps over the lazy dog\n"), 500) dir := t.TempDir() in := filepath.Join(dir, "in") out := filepath.Join(dir, "in.zst") if err := os.WriteFile(in, orig, 0o644); err != nil { t.Fatal(err) } run(t, zstd, "-q", "-19", in, "-o", out) comp, err := os.ReadFile(out) if err != nil { t.Fatal(err) } got, err := Decompress(nil, comp) if err != nil { t.Fatalf("Decompress: %v", err) } if !bytes.Equal(got, orig) { t.Fatalf("plain: round-trip mismatch (%d vs %d bytes)", len(got), len(orig)) } } // TestDecompressDictLibzstd: libzstd trains a dictionary and compresses with it, // then the shim decompresses with the same dictionary via NewDDict — // the exact archive-read path dolt exercises. This is the interop risk the // feasibility review flagged (klauspost matches decoder dicts by embedded // dict-ID, structured ZDICT format), proven end-to-end here. func TestDecompressDictLibzstd(t *testing.T) { zstd := zstdBin(t) dir := t.TempDir() // Enough varied-but-similar samples for the trainer to build a dictionary. var samples []string for i := 0; i < 512; i++ { p := filepath.Join(dir, fmt.Sprintf("s%03d", i)) line := fmt.Sprintf("issue memestudio-%04d: lorem ipsum dolor sit amet consectetur %d\n", i, i*7) if err := os.WriteFile(p, []byte(line), 0o644); err != nil { t.Fatal(err) } samples = append(samples, p) } dictPath := filepath.Join(dir, "dict") if out, err := exec.Command(zstd, append([]string{"--train", "--maxdict=8192", "-o", dictPath}, samples...)...).CombinedOutput(); err != nil { t.Skipf("zstd --train unavailable/failed (%v): %s", err, out) } dict, err := os.ReadFile(dictPath) if err != nil { t.Fatalf("read trained dict: %v", err) } orig := []byte("issue memestudio-0042: lorem ipsum dolor sit amet consectetur 294 — payload body\n") in := filepath.Join(dir, "payload") out := filepath.Join(dir, "payload.zst") if err := os.WriteFile(in, orig, 0o644); err != nil { t.Fatal(err) } run(t, zstd, "-q", "-19", "-D", dictPath, in, "-o", out) comp, err := os.ReadFile(out) if err != nil { t.Fatal(err) } dd, err := NewDDict(dict) if err != nil { t.Fatalf("NewDDict on libzstd-trained dict: %v", err) } defer dd.Release() got, err := DecompressDict(nil, comp, dd) if err != nil { t.Fatalf("DecompressDict on libzstd frame: %v", err) } if !bytes.Equal(got, orig) { t.Fatalf("dict: round-trip mismatch\n got=%q\nwant=%q", got, orig) } } // TestShimRoundTripPlain exercises the shim's own compress+decompress with no // external tool and no dictionary. func TestShimRoundTripPlain(t *testing.T) { orig := bytes.Repeat([]byte("parade lane rolling lined-up stalled past-stand\n"), 200) comp := Compress(nil, orig) got, err := Decompress(nil, comp) if err != nil { t.Fatalf("Decompress: %v", err) } if !bytes.Equal(got, orig) { t.Fatal("plain shim round-trip mismatch") } } // TestShimRoundTripDict exercises the shim's CompressDict + DecompressDict over // a real structured dictionary. The dictionary is trained by libzstd (the zstd // CLI) rather than the shim's own BuildDict, because BuildDict is off this // project's runtime path (dolt only calls it from archive-writer/gc, which this // binary never runs) and klauspost's trainer is stricter about corpus size than // libzstd's ZDICT — training quality is irrelevant here; what matters is that // the shim's encode+decode agree over a standard dictionary. func TestShimRoundTripDict(t *testing.T) { zstd := zstdBin(t) dir := t.TempDir() var samples []string for i := 0; i < 512; i++ { p := filepath.Join(dir, fmt.Sprintf("s%03d", i)) line := fmt.Sprintf("bead memestudio-%04d status open priority %d assignee eugene\n", i, i%4) if err := os.WriteFile(p, []byte(line), 0o644); err != nil { t.Fatal(err) } samples = append(samples, p) } dictPath := filepath.Join(dir, "dict") if out, err := exec.Command(zstd, append([]string{"--train", "--maxdict=8192", "-o", dictPath}, samples...)...).CombinedOutput(); err != nil { t.Skipf("zstd --train unavailable/failed (%v): %s", err, out) } dict, err := os.ReadFile(dictPath) if err != nil { t.Fatal(err) } orig := bytes.Repeat([]byte("bead memestudio-0042 status open priority 2 assignee eugene\n"), 50) cd, err := NewCDict(dict) if err != nil { t.Fatalf("NewCDict: %v", err) } defer cd.Release() dd, err := NewDDict(dict) if err != nil { t.Fatalf("NewDDict: %v", err) } defer dd.Release() dcomp := CompressDict(nil, orig, cd) dgot, err := DecompressDict(nil, dcomp, dd) if err != nil { t.Fatalf("DecompressDict: %v", err) } if !bytes.Equal(dgot, orig) { t.Fatal("dict shim round-trip mismatch") } } func run(t *testing.T, name string, args ...string) { t.Helper() if out, err := exec.Command(name, args...).CombinedOutput(); err != nil { t.Fatalf("%s %v: %v\n%s", name, args, err, out) } }