~bigbes/sr-ht-dolt

ref: 8ed47c67fd2f11b540cb8caa4c84407834b973af sr-ht-dolt/third_party/gozstd-purego/gozstd_test.go -rw-r--r-- 5.7 KiB
8ed47c67 — Eugene Blikh build: pure-Go (CGO_ENABLED=0) build via a klauspost-backed gozstd shim 30 days ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
package gozstd

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"testing"
)

// These tests prove the load-bearing interop property of the shim: the pure-Go
// klauspost backend decodes zstd frames produced by upstream libzstd — plain
// AND dictionary-compressed — which is exactly what dolt's NBS archive reader
// asks of gozstd at runtime (it decompresses .darc archives authored by the
// real `dolt`/libzstd). They shell out to the `zstd` CLI (libzstd) to author
// the fixtures and skip cleanly when it is absent, so CI without the tool still
// passes. The internal round-trip test needs no external tool.

func zstdBin(t *testing.T) string {
	t.Helper()
	p, err := exec.LookPath("zstd")
	if err != nil {
		t.Skip("zstd CLI (libzstd) not found; skipping libzstd-interop test")
	}
	return p
}

// TestDecompressLibzstdPlain: libzstd compresses, the shim decompresses.
func TestDecompressLibzstdPlain(t *testing.T) {
	zstd := zstdBin(t)
	orig := bytes.Repeat([]byte("the quick brown fox jumps over the lazy dog\n"), 500)

	dir := t.TempDir()
	in := filepath.Join(dir, "in")
	out := filepath.Join(dir, "in.zst")
	if err := os.WriteFile(in, orig, 0o644); err != nil {
		t.Fatal(err)
	}
	run(t, zstd, "-q", "-19", in, "-o", out)

	comp, err := os.ReadFile(out)
	if err != nil {
		t.Fatal(err)
	}
	got, err := Decompress(nil, comp)
	if err != nil {
		t.Fatalf("Decompress: %v", err)
	}
	if !bytes.Equal(got, orig) {
		t.Fatalf("plain: round-trip mismatch (%d vs %d bytes)", len(got), len(orig))
	}
}

// TestDecompressDictLibzstd: libzstd trains a dictionary and compresses with it,
// then the shim decompresses with the same dictionary via NewDDict —
// the exact archive-read path dolt exercises. This is the interop risk the
// feasibility review flagged (klauspost matches decoder dicts by embedded
// dict-ID, structured ZDICT format), proven end-to-end here.
func TestDecompressDictLibzstd(t *testing.T) {
	zstd := zstdBin(t)
	dir := t.TempDir()

	// Enough varied-but-similar samples for the trainer to build a dictionary.
	var samples []string
	for i := 0; i < 512; i++ {
		p := filepath.Join(dir, fmt.Sprintf("s%03d", i))
		line := fmt.Sprintf("issue memestudio-%04d: lorem ipsum dolor sit amet consectetur %d\n", i, i*7)
		if err := os.WriteFile(p, []byte(line), 0o644); err != nil {
			t.Fatal(err)
		}
		samples = append(samples, p)
	}
	dictPath := filepath.Join(dir, "dict")
	if out, err := exec.Command(zstd, append([]string{"--train", "--maxdict=8192", "-o", dictPath}, samples...)...).CombinedOutput(); err != nil {
		t.Skipf("zstd --train unavailable/failed (%v): %s", err, out)
	}
	dict, err := os.ReadFile(dictPath)
	if err != nil {
		t.Fatalf("read trained dict: %v", err)
	}

	orig := []byte("issue memestudio-0042: lorem ipsum dolor sit amet consectetur 294 — payload body\n")
	in := filepath.Join(dir, "payload")
	out := filepath.Join(dir, "payload.zst")
	if err := os.WriteFile(in, orig, 0o644); err != nil {
		t.Fatal(err)
	}
	run(t, zstd, "-q", "-19", "-D", dictPath, in, "-o", out)

	comp, err := os.ReadFile(out)
	if err != nil {
		t.Fatal(err)
	}

	dd, err := NewDDict(dict)
	if err != nil {
		t.Fatalf("NewDDict on libzstd-trained dict: %v", err)
	}
	defer dd.Release()
	got, err := DecompressDict(nil, comp, dd)
	if err != nil {
		t.Fatalf("DecompressDict on libzstd frame: %v", err)
	}
	if !bytes.Equal(got, orig) {
		t.Fatalf("dict: round-trip mismatch\n got=%q\nwant=%q", got, orig)
	}
}

// TestShimRoundTripPlain exercises the shim's own compress+decompress with no
// external tool and no dictionary.
func TestShimRoundTripPlain(t *testing.T) {
	orig := bytes.Repeat([]byte("parade lane rolling lined-up stalled past-stand\n"), 200)
	comp := Compress(nil, orig)
	got, err := Decompress(nil, comp)
	if err != nil {
		t.Fatalf("Decompress: %v", err)
	}
	if !bytes.Equal(got, orig) {
		t.Fatal("plain shim round-trip mismatch")
	}
}

// TestShimRoundTripDict exercises the shim's CompressDict + DecompressDict over
// a real structured dictionary. The dictionary is trained by libzstd (the zstd
// CLI) rather than the shim's own BuildDict, because BuildDict is off this
// project's runtime path (dolt only calls it from archive-writer/gc, which this
// binary never runs) and klauspost's trainer is stricter about corpus size than
// libzstd's ZDICT — training quality is irrelevant here; what matters is that
// the shim's encode+decode agree over a standard dictionary.
func TestShimRoundTripDict(t *testing.T) {
	zstd := zstdBin(t)
	dir := t.TempDir()

	var samples []string
	for i := 0; i < 512; i++ {
		p := filepath.Join(dir, fmt.Sprintf("s%03d", i))
		line := fmt.Sprintf("bead memestudio-%04d status open priority %d assignee eugene\n", i, i%4)
		if err := os.WriteFile(p, []byte(line), 0o644); err != nil {
			t.Fatal(err)
		}
		samples = append(samples, p)
	}
	dictPath := filepath.Join(dir, "dict")
	if out, err := exec.Command(zstd, append([]string{"--train", "--maxdict=8192", "-o", dictPath}, samples...)...).CombinedOutput(); err != nil {
		t.Skipf("zstd --train unavailable/failed (%v): %s", err, out)
	}
	dict, err := os.ReadFile(dictPath)
	if err != nil {
		t.Fatal(err)
	}

	orig := bytes.Repeat([]byte("bead memestudio-0042 status open priority 2 assignee eugene\n"), 50)
	cd, err := NewCDict(dict)
	if err != nil {
		t.Fatalf("NewCDict: %v", err)
	}
	defer cd.Release()
	dd, err := NewDDict(dict)
	if err != nil {
		t.Fatalf("NewDDict: %v", err)
	}
	defer dd.Release()

	dcomp := CompressDict(nil, orig, cd)
	dgot, err := DecompressDict(nil, dcomp, dd)
	if err != nil {
		t.Fatalf("DecompressDict: %v", err)
	}
	if !bytes.Equal(dgot, orig) {
		t.Fatal("dict shim round-trip mismatch")
	}
}

func run(t *testing.T, name string, args ...string) {
	t.Helper()
	if out, err := exec.Command(name, args...).CombinedOutput(); err != nil {
		t.Fatalf("%s %v: %v\n%s", name, args, err, out)
	}
}