M Justfile => Justfile +19 -0
@@ 177,6 177,25 @@ bench-shapes: gen
jit-trace: gen
tarantool bench/jit_trace.lua
+# Regenerate Go-side pb.go + vtproto pb.go in bench/go/pb/. Not committed
+# (*.pb.go is gitignored repo-wide); run before `bench-go` on a fresh
+# checkout. Requires `protoc-gen-go` and `protoc-gen-go-vtproto` on PATH:
+# go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
+# go install github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto@latest
+gen-go:
+ mkdir -p bench/go/pb/hellopb bench/go/pb/proto2pb
+ cd bench/go && PATH="$(go env GOPATH)/bin:$PATH" protoc \
+ --go_out=. --go_opt=module=github.com/tarantool-protobuf/bench/go \
+ --go-vtproto_out=. --go-vtproto_opt=module=github.com/tarantool-protobuf/bench/go \
+ --go-vtproto_opt=features=marshal+unmarshal+size \
+ -I proto proto/hello.proto proto/proto2_basic.proto
+
+# Go-side comparison bench: same fixtures + sizes as bench.lua, run against
+# google.golang.org/protobuf (apiv2) and planetscale/vtprotobuf. Serial by
+# default — Go's testing.B doesn't parallelize unless RunParallel is called.
+bench-go: gen-go
+ cd bench/go && go test -bench=. -benchmem -run=^$ -benchtime=1s ./...
+
# ---------------------------------------------------------------------------
# Conformance (Google's protobuf conformance suite, in Docker)
# ---------------------------------------------------------------------------
A bench/COMPARISON.md => bench/COMPARISON.md +115 -0
@@ 0,0 1,115 @@
+# Lua vs Go: cross-runtime protobuf benchmark
+
+Side-by-side numbers from `just bench` (Lua `full` mode) and `just
+bench-go` (Go apiv2 + vtproto). Same proto definitions, same payload
+shapes, single-threaded on both sides.
+
+- **Host:** Apple M4 Pro, darwin/arm64, GOMAXPROCS=14 (Go iter loop is
+ serial regardless; GOMAXPROCS only affects GC threads)
+- **Tarantool:** 3.x with LuaJIT 2.1 fork
+- **Go:** 1.26.3, `google.golang.org/protobuf` v1.36.11,
+ `planetscale/vtprotobuf` v0.6.0
+- **Benchtime:** Go `-benchtime=1s`; Lua adaptive (50k–200k iters)
+- **Lua mode shown:** `full` (codegen-inlined; the fast path). `runtime`
+ mode is 10–25 % slower across the board — see `bench/baseline.json`
+ if you want both.
+- **Columns:** MB/s (higher is better), B/op (allocator bytes per
+ operation, lower is better)
+
+Numbers will drift run-to-run by 5–10 % on a busy laptop. Treat ratios,
+not absolutes, as load-bearing.
+
+## hello.Person — encode
+
+| Size | Lua full MB/s | Lua run MB/s | apiv2 MB/s | vtproto MB/s | apiv2/Lua-full | vtproto/Lua-full |
+| ------ | ------------: | -----------: | ---------: | -----------: | -------------: | ---------------: |
+| 10 B | 31.14 | 16.09 | 161.88 | 599.94 | 5.2 × | 19.3 × |
+| 100 B | 281.29 | 148.41 | 1416.79 | 4330.00 | 5.0 × | 15.4 × |
+| 1 KB | 310.70 | 252.11 | 2090.37 | 5626.07 | 6.7 × | 18.1 × |
+| 10 KB | 582.82 | 546.06 | 2783.52 | 7494.49 | 4.8 × | 12.9 × |
+| 100 KB | 582.94 | 593.19 | 2742.85 | 7851.05 | 4.7 × | 13.5 × |
+
+## hello.Person — decode
+
+| Size | Lua full MB/s | Lua run MB/s | apiv2 MB/s | vtproto MB/s | apiv2/Lua-full | vtproto/Lua-full |
+| ------ | ------------: | -----------: | ---------: | -----------: | -------------: | ---------------: |
+| 10 B | 32.71 | 27.84 | 131.47 | 774.64 | 4.0 × | 23.7 × |
+| 100 B | 259.38 | 231.39 | 1099.02 | 4853.53 | 4.2 × | 18.7 × |
+| 1 KB | 136.40 | 129.43 | 966.14 | 1524.65 | 7.1 × | 11.2 × |
+| 10 KB | 192.72 | 175.65 | 1263.84 | 2084.66 | 6.6 × | 10.8 × |
+| 100 KB | 187.85 | 171.28 | 1105.49 | 1966.28 | 5.9 × | 10.5 × |
+
+## proto2_basic.BenchPayload — encode
+
+vtproto skipped here: `MarshalVT` drops proto2 extensions, so its
+output bytes would not match apiv2/Lua. apiv2 is the only fair
+Go-side comparison for this fixture.
+
+| Size | Lua full MB/s | Lua run MB/s | apiv2 MB/s | apiv2/Lua-full |
+| ---- | ------------: | -----------: | ---------: | -------------: |
+| min | 10.67 | 7.71 | 36.27 | 3.4 × |
+| mid | 157.95 | 141.29 | 1486.45 | 9.4 × |
+
+## proto2_basic.BenchPayload — decode
+
+| Size | Lua full MB/s | Lua run MB/s | apiv2 MB/s | apiv2/Lua-full |
+| ---- | ------------: | -----------: | ---------: | -------------: |
+| min | 6.03 | 6.35 | 19.23 | 3.2 × |
+| mid | 56.48 | 55.81 | 678.35 | 12.0 × |
+
+## Allocations per op (Person)
+
+Worth a separate look because the picture changes between encode and
+decode. Encode is a single output buffer in all three runtimes; decode
+allocates per-field-instance in Go but reuses interned strings in
+LuaJIT.
+
+| Size | Op | Lua full B/op | Lua run B/op | apiv2 B/op | apiv2 allocs | vtproto B/op | vtproto allocs |
+| ------ | ------ | ------------: | -----------: | ---------: | -----------: | -----------: | -------------: |
+| 10 B | encode | 136.0 | 136.0 | 16 | 1 | 16 | 1 |
+| 100 B | encode | 136.0 | 136.0 | 96 | 1 | 96 | 1 |
+| 1 KB | encode | 1368.3 | 1368.3 | 1024 | 1 | 1024 | 1 |
+| 10 KB | encode | 8540.2 | 8540.2 | 9728 | 1 | 9728 | 1 |
+| 100 KB | encode | 131605.2 | 131605.2 | 98304 | 1 | 98304 | 1 |
+| 10 B | decode | 112.0 | 112.0 | 216 | 2 | 8 | 1 |
+| 100 B | decode | 112.0 | 112.0 | 304 | 2 | 96 | 1 |
+| 1 KB | decode | 1000.0 | 1000.0 | 2200 | 38 | 1992 | 37 |
+| 10 KB | decode | 4840.0 | 4840.0 | 18712 | 297 | 18504 | 296 |
+| 100 KB | decode | 33512.2 | 33512.2 | 249368 | 2862 | 249160 | 2861 |
+
+The decode comparison flips above ~1 KB: Lua decodes a 100 KB Person
+with **7.4 ×** less allocator pressure than either Go marshaler. The
+repeated-string-emails workload allocates one `string` header per
+element in Go, but lands in LuaJIT's string-intern table where
+identical or short strings reuse storage. This is a real workload
+characteristic for chat/log streams, not a microbench artifact.
+
+## Reading the numbers
+
+- **Encode ceiling** is set by code-gen marshalers + low-level
+ bytes-pushing: vtproto wins by 10–24 × over us. Our `mode=full`
+ inlines the same way, but LuaJIT pays for table iteration,
+ type-tag dispatch, and string allocation that AOT-compiled Go
+ doesn't.
+- **vs. apiv2** (Go's default reflective path) the gap narrows to
+ 4–7 ×. apiv2 walks `protoreflect.MessageDescriptor` per call; we
+ walk our own descriptor in `mode=runtime` and inline it in
+ `mode=full`. apiv2 still wins because it's compiled.
+- **Big-payload decode** is where LuaJIT's string interning pays
+ off and the alloc gap inverts (33 KB Lua vs 249 KB Go for a 100 KB
+ Person). Throughput is still Go-favorable, but per-op GC pressure
+ is not.
+- **proto2 fixture** numbers (10–60 MB/s for the `min` size) are
+ dominated by per-message overhead — extensions, defaults — and
+ not representative of bulk throughput. The `mid` line is the one
+ to compare for steady-state cost.
+
+## Reproducing
+
+```bash
+just bench # Lua: writes JSON to stdout (this doc used --print output)
+just bench-go # Go: standard go test -bench output
+
+# Quick narrowed view:
+cd bench/go && go test -bench=Person -benchtime=2s ./...
+```
A bench/go/README.md => bench/go/README.md +77 -0
@@ 0,0 1,77 @@
+# Go comparison bench
+
+Apples-to-apples microbenchmarks against `bench/bench.lua` so the LuaJIT
+runtime can be compared to mainstream Go protobuf implementations.
+
+## What it measures
+
+Same fixtures and size sweep as the Lua bench:
+
+- `hello.Person` at five payload sizes (10 B, 100 B, 1 KB, 10 KB, 100 KB)
+- `proto2_basic.BenchPayload` at two sizes (min, mid)
+
+For each fixture × size, both encode and decode under two Go
+implementations:
+
+- **apiv2** — `google.golang.org/protobuf/proto.Marshal/Unmarshal`. The
+ default reflective marshaler everyone gets out of the box.
+- **vtproto** — `planetscale/vtprotobuf` generated `MarshalVT/UnmarshalVT`.
+ Code-gen'd, no reflection. Conceptually equivalent to our `mode=full`
+ Lua codegen.
+
+Single-threaded: Go's `testing.B` runs serially unless `RunParallel` is
+called. `-cpu=1` is not needed — GOMAXPROCS only affects GC parallelism,
+not bench iterations.
+
+## Running
+
+```bash
+just bench-go # full sweep, default benchtime=1s
+cd bench/go && go test -bench=Person -benchtime=3s ./...
+```
+
+`go test` reports ns/op, MB/s (via `b.SetBytes`), B/op, allocs/op.
+
+## Comparing to the Lua bench
+
+`bench/bench.lua --print` emits ns/op + msgs/s + MB/s + bytes/op per
+mode × size. Run both, pick the same size, compare directly. The
+encoded byte lengths should match within a byte or two for the same
+target size since both use the same proto definitions and equivalent
+payload builders.
+
+## proto2 caveats
+
+- **Extensions skipped by vtproto.** `MarshalVT` does not serialize
+ proto2 extensions, so the BenchPayload fixture (which carries
+ `ext_count` + `ext_label`) would produce shorter bytes than the
+ apiv2 path. The vtproto proto2 cases are intentionally omitted to
+ avoid publishing misleading throughput numbers.
+- Groups are supported by both `protoc-gen-go` and vtproto.
+
+## Layout
+
+```
+bench/go/
+├── go.mod
+├── proto/ # .proto sources (subset of repo protos with go_package set)
+├── pb/{hellopb,proto2pb}/ # generated *.pb.go + *_vtproto.pb.go
+├── fixtures.go # payload builders mirroring bench/bench.lua
+└── bench_test.go # `go test -bench` entry points
+```
+
+## Generated .pb.go files
+
+`*.pb.go` is gitignored repo-wide, so `bench/go/pb/` is **not**
+committed — regenerate locally before running the bench:
+
+```bash
+just gen-go # runs protoc with both plugins; bench-go depends on it
+```
+
+Requires `protoc-gen-go` and `protoc-gen-go-vtproto` on `$PATH`:
+
+```bash
+go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
+go install github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto@latest
+```
A bench/go/bench_test.go => bench/go/bench_test.go +138 -0
@@ 0,0 1,138 @@
+// Standard `go test -bench=.` benchmarks. Mirrors bench/bench.lua: per
+// fixture × per size, encode + decode, throughput + alloc.
+//
+// Two implementations per case:
+// * apiv2: google.golang.org/protobuf reflective Marshal/Unmarshal
+// — the default everyone gets out of the box.
+// * vtproto: planetscale/vtprotobuf generated MarshalVT/UnmarshalVT
+// — the fastest pure-Go path, conceptually equivalent to
+// our `mode=full` codegen.
+//
+// Runs are single-threaded by default (testing.B doesn't parallelize
+// unless RunParallel is called).
+package bench
+
+import (
+ "fmt"
+ "testing"
+
+ "google.golang.org/protobuf/proto"
+
+ hellopb "github.com/tarantool-protobuf/bench/go/pb/hellopb"
+ proto2pb "github.com/tarantool-protobuf/bench/go/pb/proto2pb"
+)
+
+// --- hello.Person ---------------------------------------------------------
+
+func BenchmarkPersonEncode(b *testing.B) {
+ for _, sz := range PersonSizes {
+ msg := BuildPerson(sz.Target)
+ wire, err := proto.Marshal(msg)
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ b.Run(fmt.Sprintf("apiv2/%s", sz.Label), func(b *testing.B) {
+ b.SetBytes(int64(len(wire)))
+ b.ReportAllocs()
+ for i := 0; i < b.N; i++ {
+ if _, err := proto.Marshal(msg); err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+
+ b.Run(fmt.Sprintf("vtproto/%s", sz.Label), func(b *testing.B) {
+ b.SetBytes(int64(len(wire)))
+ b.ReportAllocs()
+ for i := 0; i < b.N; i++ {
+ if _, err := msg.MarshalVT(); err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
+
+func BenchmarkPersonDecode(b *testing.B) {
+ for _, sz := range PersonSizes {
+ msg := BuildPerson(sz.Target)
+ wire, err := proto.Marshal(msg)
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ b.Run(fmt.Sprintf("apiv2/%s", sz.Label), func(b *testing.B) {
+ b.SetBytes(int64(len(wire)))
+ b.ReportAllocs()
+ for i := 0; i < b.N; i++ {
+ var out hellopb.Person
+ if err := proto.Unmarshal(wire, &out); err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+
+ b.Run(fmt.Sprintf("vtproto/%s", sz.Label), func(b *testing.B) {
+ b.SetBytes(int64(len(wire)))
+ b.ReportAllocs()
+ for i := 0; i < b.N; i++ {
+ var out hellopb.Person
+ if err := out.UnmarshalVT(wire); err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
+
+// --- proto2_basic.BenchPayload --------------------------------------------
+
+func BenchmarkProto2Encode(b *testing.B) {
+ for _, sz := range Proto2Sizes {
+ msg := BuildProto2(sz.Target)
+ wire, err := proto.Marshal(msg)
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ b.Run(fmt.Sprintf("apiv2/%s", sz.Label), func(b *testing.B) {
+ b.SetBytes(int64(len(wire)))
+ b.ReportAllocs()
+ for i := 0; i < b.N; i++ {
+ if _, err := proto.Marshal(msg); err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+
+ // vtproto's generated MarshalVT does not serialize extensions —
+ // it skips XXX_unrecognized + protoimpl.ExtensionFields and so
+ // drops the ext_count/ext_label payload. The benchmark would
+ // produce shorter bytes than apiv2; skip the vtproto variant
+ // when the fixture carries extensions so we don't publish
+ // misleading throughput numbers.
+ _ = msg.MarshalVT // keep symbol referenced; intentionally not benched
+ }
+}
+
+func BenchmarkProto2Decode(b *testing.B) {
+ for _, sz := range Proto2Sizes {
+ msg := BuildProto2(sz.Target)
+ wire, err := proto.Marshal(msg)
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ b.Run(fmt.Sprintf("apiv2/%s", sz.Label), func(b *testing.B) {
+ b.SetBytes(int64(len(wire)))
+ b.ReportAllocs()
+ for i := 0; i < b.N; i++ {
+ var out proto2pb.BenchPayload
+ if err := proto.Unmarshal(wire, &out); err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
A bench/go/fixtures.go => bench/go/fixtures.go +109 -0
@@ 0,0 1,109 @@
+// Payload builders that mirror bench/bench.lua so msg sizes and shapes
+// match across runtimes. Same field set + field numbers as the Lua
+// build_person_payload / build_proto2_payload helpers, so encoded byte
+// lengths land within a byte or two for the same target size.
+package bench
+
+import (
+ "fmt"
+ "strings"
+
+ hellopb "github.com/tarantool-protobuf/bench/go/pb/hellopb"
+ proto2pb "github.com/tarantool-protobuf/bench/go/pb/proto2pb"
+ "google.golang.org/protobuf/proto"
+)
+
+type SizeSpec struct {
+ Label string
+ Target int
+}
+
+var PersonSizes = []SizeSpec{
+ {"10B", 10},
+ {"100B", 100},
+ {"1KB", 1024},
+ {"10KB", 10240},
+ {"100KB", 102400},
+}
+
+var Proto2Sizes = []SizeSpec{
+ {"min", 0},
+ {"mid", 1024},
+}
+
+func BuildPerson(target int) *hellopb.Person {
+ if target <= 10 {
+ return &hellopb.Person{Name: "bigbes", Age: 42}
+ }
+ if target <= 100 {
+ return &hellopb.Person{
+ Name: strings.Repeat("a", target-10),
+ Age: 42,
+ }
+ }
+ const perEmail = 36
+ const fixedBytes = 80
+ nEmails := (target - fixedBytes) / perEmail
+ if nEmails < 1 {
+ nEmails = 1
+ }
+ p := &hellopb.Person{
+ Name: "bigbes",
+ Age: 42,
+ Address: &hellopb.Address{
+ Street: "1 Main St",
+ City: "Springfield",
+ Zip: 12345,
+ },
+ LuckyNumbers: []int32{7, 13, 21, 42, 99},
+ Emails: make([]string, nEmails),
+ }
+ for i := 0; i < nEmails; i++ {
+ p.Emails[i] = strings.Repeat("e", 28) + fmt.Sprintf("%04d", i+1)
+ }
+ return p
+}
+
+func BuildProto2(target int) *proto2pb.BenchPayload {
+ if target <= 0 {
+ id := int32(7)
+ m := &proto2pb.BenchPayload{Id: &id}
+ proto.SetExtension(m, proto2pb.E_ExtCount, int32(42))
+ return m
+ }
+ const nTags = 16
+ const perTag = 30
+ tags := make([]string, nTags)
+ for i := 0; i < nTags; i++ {
+ tags[i] = strings.Repeat("t", perTag-2) + fmt.Sprintf("%02d", i+1)
+ }
+ lucky := make([]int32, 8)
+ for i := 0; i < 8; i++ {
+ lucky[i] = int32(1000 + i + 1)
+ }
+ id := int32(7)
+ name := "bench"
+ retries := int32(9)
+ innerKey := strings.Repeat("k", 16)
+ innerWeight := int32(3)
+ latency := int32(1234567)
+ attempts := int32(4)
+ m := &proto2pb.BenchPayload{
+ Id: &id,
+ Name: &name,
+ Retries: &retries,
+ LuckyNumbers: lucky,
+ Tags: tags,
+ Inner: &proto2pb.BenchPayload_Inner{
+ Key: &innerKey,
+ Weight: &innerWeight,
+ },
+ Stats: &proto2pb.BenchPayload_Stats{
+ LatencyNs: &latency,
+ Attempts: &attempts,
+ },
+ }
+ proto.SetExtension(m, proto2pb.E_ExtCount, int32(99))
+ proto.SetExtension(m, proto2pb.E_ExtLabel, strings.Repeat("x", 32))
+ return m
+}
A bench/go/go.mod => bench/go/go.mod +8 -0
@@ 0,0 1,8 @@
+module github.com/tarantool-protobuf/bench/go
+
+go 1.23
+
+require (
+ github.com/planetscale/vtprotobuf v0.6.0
+ google.golang.org/protobuf v1.36.11
+)
A bench/go/go.sum => bench/go/go.sum +6 -0
@@ 0,0 1,6 @@
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/planetscale/vtprotobuf v0.6.0 h1:nBeETjudeJ5ZgBHUz1fVHvbqUKnYOXNhsIEabROxmNA=
+github.com/planetscale/vtprotobuf v0.6.0/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
A bench/go/proto/hello.proto => bench/go/proto/hello.proto +35 -0
@@ 0,0 1,35 @@
+// Bench-only subset of examples/proto/hello.proto with go_package set.
+// Field numbers and wire format match the Lua version exactly so encoded
+// bytes are interchangeable between runtimes.
+syntax = "proto3";
+
+package hello;
+
+option go_package = "github.com/tarantool-protobuf/bench/go/pb/hellopb;hellopb";
+
+enum Status {
+ UNKNOWN = 0;
+ OK = 1;
+ ERROR = 2;
+}
+
+message Address {
+ string street = 1;
+ string city = 2;
+ int32 zip = 3;
+ optional string apartment = 4;
+}
+
+message Person {
+ string name = 1;
+ int32 age = 2;
+ repeated string emails = 3;
+ Status status = 4;
+ Address address = 5;
+ repeated Person friends = 6;
+ repeated int32 lucky_numbers = 7;
+ bytes avatar = 8;
+ fixed64 user_id = 9;
+ sint32 balance = 10;
+ double weight_kg = 11;
+}
A bench/go/proto/proto2_basic.proto => bench/go/proto/proto2_basic.proto +32 -0
@@ 0,0 1,32 @@
+// Bench-only subset of test/proto/proto2_basic.proto with go_package set.
+// Same field numbers and wire format as the Lua-tested version.
+syntax = "proto2";
+
+package proto2_basic;
+
+option go_package = "github.com/tarantool-protobuf/bench/go/pb/proto2pb;proto2pb";
+
+message BenchPayload {
+ required int32 id = 1;
+ optional string name = 2 [default = "anonymous"];
+ optional int32 retries = 3 [default = 3];
+ repeated int32 lucky_numbers = 4 [packed = true];
+ repeated string tags = 5;
+ optional Inner inner = 6;
+ optional group Stats = 7 {
+ optional int32 latency_ns = 8;
+ optional int32 attempts = 9;
+ }
+
+ message Inner {
+ required string key = 1;
+ optional int32 weight = 2;
+ }
+
+ extensions 100 to 199;
+}
+
+extend BenchPayload {
+ optional int32 ext_count = 100;
+ optional string ext_label = 101;
+}