// Package gozstd is a pure-Go, drop-in replacement for the subset of // github.com/dolthub/gozstd that github.com/dolthub/dolt/go references, backed // by github.com/klauspost/compress/zstd. It exists so this project can build // with CGO_ENABLED=0: upstream gozstd bundles the zstd C source and requires // cgo for every symbol, which is the last hard cgo dependency in the dolt // import graph (ICU is already avoided via the gms_pure_go build tag). // // It is wired in via a `replace github.com/dolthub/gozstd => ./third_party/ // gozstd-purego` directive; dolt itself is not modified. // // # Scope and correctness // // dolt uses gozstd only from its NBS "archive" subsystem (store/nbs/archive_*), // and references exactly nine symbols: Compress, CompressDict, Decompress, // DecompressDict, BuildDict, NewCDict, NewDDict, and the CDict/DDict types. This // file reproduces those with identical signatures. Streaming (Writer/Reader), // the Stream* helpers, CompressLevel and NewCDictLevel are part of upstream // gozstd's API but are unused by dolt, so they are intentionally omitted. // // zstd is a standard frame format (RFC 8878) and dictionaries carry an embedded // dict-ID, so frames and dictionaries produced by upstream libzstd (e.g. by the // real `dolt` CLI) are decodable here and vice-versa. This project only ever // runs the DECOMPRESS side at runtime (it serves chunks over remotesapi and // browses read-only; it never builds archives or runs gc), so the compress and // dictionary-training paths are compiled but never executed by the server. They // are implemented faithfully regardless, so the swap stays correct if a future // caller does reach them. // // Concurrency: klauspost's Encoder.EncodeAll and Decoder.DecodeAll are safe for // concurrent use, matching gozstd's Compress*/Decompress* which dolt calls from // multiple goroutines. package gozstd import ( "fmt" "hash/fnv" "sync" "github.com/klauspost/compress/zstd" ) // DefaultCompressionLevel mirrors upstream gozstd's default (zstd level 3). const DefaultCompressionLevel = 3 // Shared, dictionary-less coders. klauspost's zstd Encoder/Decoder built over a // nil stream are the documented one-shot EncodeAll/DecodeAll coders, and both // are safe for concurrent use, so a single lazily-initialized instance backs // all dictionary-less Compress/Decompress calls. var ( encOnce sync.Once enc *zstd.Encoder decOnce sync.Once dec *zstd.Decoder ) func sharedEncoder() *zstd.Encoder { encOnce.Do(func() { e, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(DefaultCompressionLevel))) if err != nil { panic(fmt.Errorf("gozstd-purego: new encoder: %w", err)) } enc = e }) return enc } func sharedDecoder() *zstd.Decoder { decOnce.Do(func() { d, err := zstd.NewReader(nil) if err != nil { panic(fmt.Errorf("gozstd-purego: new decoder: %w", err)) } dec = d }) return dec } // Compress appends the zstd-compressed form of src to dst and returns dst. // Signature and append semantics match upstream gozstd.Compress. Note klauspost // takes (src, dst) where gozstd takes (dst, src); both append to dst. func Compress(dst, src []byte) []byte { return sharedEncoder().EncodeAll(src, dst) } // Decompress appends the decompressed form of src to dst and returns dst. func Decompress(dst, src []byte) ([]byte, error) { return sharedDecoder().DecodeAll(src, dst) } // CDict is a compression dictionary. Upstream gozstd wraps an opaque libzstd // ZSTD_CDict; here it holds a klauspost Encoder pre-loaded with the dictionary, // which embeds the dictionary's ID into every frame it writes (as libzstd does). type CDict struct { enc *zstd.Encoder } // NewCDict builds a CDict from a raw zstd dictionary at the default level. func NewCDict(dict []byte) (*CDict, error) { e, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(DefaultCompressionLevel)), zstd.WithEncoderDict(dict)) if err != nil { return nil, fmt.Errorf("gozstd-purego: NewCDict: %w", err) } return &CDict{enc: e}, nil } // Release frees the dictionary's coder. Upstream gozstd requires this to free C // memory; here it closes the klauspost Encoder. dolt does not currently call it, // but it is provided for API parity and to release coder goroutines. Safe to // call on nil. func (cd *CDict) Release() { if cd != nil && cd.enc != nil { cd.enc.Close() } } // DDict is a decompression dictionary: a klauspost Decoder pre-loaded with the // dictionary. DecodeAll selects it for frames whose embedded dict-ID matches. type DDict struct { dec *zstd.Decoder } // NewDDict builds a DDict from a raw zstd dictionary. The dictionary is a // standard structured (ZDICT) dictionary carrying magic + a dict-ID, which // klauspost parses via WithDecoderDicts. func NewDDict(dict []byte) (*DDict, error) { d, err := zstd.NewReader(nil, zstd.WithDecoderDicts(dict)) if err != nil { return nil, fmt.Errorf("gozstd-purego: NewDDict: %w", err) } return &DDict{dec: d}, nil } // Release frees the dictionary's coder. Safe to call on nil. func (dd *DDict) Release() { if dd != nil && dd.dec != nil { dd.dec.Close() } } // CompressDict appends the dictionary-compressed form of src to dst using cd. func CompressDict(dst, src []byte, cd *CDict) []byte { return cd.enc.EncodeAll(src, dst) } // DecompressDict appends the dictionary-decompressed form of src to dst using // dd. The frame's embedded dict-ID must match dd's dictionary. func DecompressDict(dst, src []byte, dd *DDict) ([]byte, error) { return dd.dec.DecodeAll(src, dst) } // BuildDict trains a zstd dictionary from samples. Upstream gozstd wraps // libzstd's ZDICT_trainFromBuffer; klauspost's trainer uses a different // algorithm, so the produced dictionary bytes differ, but the result is a valid // standard zstd dictionary. This function is only reached by dolt's archive- // writer / gc code, which this project never runs, so its output is never // consumed here; it is implemented faithfully so the swap stays correct if a // future caller does train a dictionary. desiredDictLen has no direct klauspost // analogue (its trainer sizes the dictionary from the content) and is ignored. // // It panics on failure to match upstream gozstd, which likewise cannot return // an error from this signature; a training failure is a programmer/data error, // not a recoverable runtime condition, and — being off this project's serving // path — must surface loudly rather than yield a silently-bad dictionary. func BuildDict(samples [][]byte, desiredDictLen int) []byte { _ = desiredDictLen // Derive a stable, non-zero dict-ID from the samples so distinct inputs get // distinct IDs (klauspost embeds this ID; libzstd would assign its own). h := fnv.New32a() for _, s := range samples { _, _ = h.Write(s) } id := h.Sum32() if id == 0 { id = 1 } d, err := zstd.BuildDict(zstd.BuildDictOptions{ ID: id, Contents: samples, // libzstd's default repcodes; a zeroed offset set is rejected. Offsets: [3]int{1, 4, 8}, }) if err != nil { panic(fmt.Errorf("gozstd-purego: BuildDict: %w", err)) } return d }