~bigbes/tarantool

tarantool-protobuf

ref: 035b2adbf9bf91e4cfc031aa1141ab75d0bb30f1 tarantool-protobuf/Justfile -rw-r--r-- 13.1 KiB
035b2adb — Eugene Blikh codegen: CHARS[_len] lookup replaces string.char(_len) at length-prefix sites (2ri) 2 months ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
# Justfile — canonical entry point for build / gen / test / bench / conformance.
#
# `just` is required (https://github.com/casey/just). On macOS:  brew install just
# On Linux:                                                       cargo install just
#
# Quick map of common targets:
#   just                          show available recipes
#   just build                    build both plugins
#   just gen                      regenerate examples/expected/{full,runtime}/*
#   just test                     run the luatest suite
#   just bench                    alloc + throughput per op
#   just conformance              run Google's conformance suite in Docker
#   just examples                 per-example runners (see examples/Justfile)
#
# Sub-recipes live in examples/Justfile (one target per runnable example).

set shell := ["bash", "-cu"]

# ---------------------------------------------------------------------------
# Paths and constants
# ---------------------------------------------------------------------------

plugin            := "protoc-gen-tarantool"
doc_plugin        := "protoc-gen-tarantool-doc"
gen_dir           := "examples/expected"
docs_dir          := "examples/docs"
proto_dir         := "examples/proto"
conformance_proto := "test/conformance/proto"
proto2_test_dir   := "test/proto"
luatest           := ".rocks/bin/luatest"
image             := "tarantool-protobuf-conformance:latest"

# Semicolon-joined LUA_PATH for the luatest suite. Trailing `;;` defers to the
# standard package.path for everything not explicitly listed.
lua_path := "./runtime/?/init.lua;./runtime/?.lua;./" + gen_dir + "/?.lua;./" + gen_dir + "/?/init.lua;./?.lua;./?/init.lua;./test/?.lua;;"

# LUA_CPATH for the optional C runtime (require('pb.c_runtime')). Trailing
# `;;` defers to the standard cpath. The C module is built into
# runtime/pb/c_runtime.{so,dylib} by `just build-c` when PB_ENABLE_C=1 is
# in play; absent without it. See docs/specs/c_accel_build_packaging.md.
#
# `.dylib` is searched before `.so` because `package.searchpath` returns
# the first existing file (not the first loadable one). If both exist
# (e.g., a Linux `.so` left in the working tree by `just conformance-c`,
# which builds inside the bind-mounted container), macOS dlopen would
# fail on the foreign ELF without ever trying the local `.dylib`. Linux
# never produces a `.dylib` so the order is harmless there.
lua_cpath := "./runtime/?.dylib;./runtime/?.so;./runtime/?/init.dylib;./runtime/?/init.so;;"

# ---------------------------------------------------------------------------
# Default
# ---------------------------------------------------------------------------

# Show available recipes.
default:
    @just --list

# Build everything from scratch and run the test suite.
all: build gen test

# ---------------------------------------------------------------------------
# Build
# ---------------------------------------------------------------------------

# Build the Lua codegen plugin (./protoc-gen-tarantool).
build:
    go build -o {{plugin}} ./cmd/protoc-gen-tarantool

# Build the Markdown doc plugin (./protoc-gen-tarantool-doc).
build-doc:
    go build -o {{doc_plugin}} ./cmd/protoc-gen-tarantool-doc

# Build the optional C-acceleration runtime into runtime/pb/c_runtime.{so,dylib}.
# The runtime is opt-in via PB_ENABLE_C=1 — see docs/specs/c_accel_build_packaging.md
# for the full contract.
build-c:
    make -C runtime/pb/c

# Remove built C-runtime artifacts.
clean-c:
    rm -f runtime/pb/c_runtime.so runtime/pb/c_runtime.dylib
    @if [ -d runtime/pb/c ]; then make -C runtime/pb/c clean; fi

# ---------------------------------------------------------------------------
# Codegen
# ---------------------------------------------------------------------------

# Regenerate examples/expected/{full,runtime}/* + conformance protos.
gen: gen-full gen-runtime gen-conformance gen-proto2-tests

# Generate full-mode Lua (inline encode/decode bodies).
gen-full: build
    mkdir -p {{gen_dir}}
    protoc \
        --plugin=./{{plugin}} \
        --tarantool_out={{gen_dir}} \
        --tarantool_opt=mode=full,prefix=full \
        -I {{proto_dir}} -I options \
        {{proto_dir}}/*.proto

# Generate runtime-mode Lua (delegates to pb.encode / pb.decode).
gen-runtime: build
    mkdir -p {{gen_dir}}
    protoc \
        --plugin=./{{plugin}} \
        --tarantool_out={{gen_dir}} \
        --tarantool_opt=mode=runtime,prefix=runtime \
        -I {{proto_dir}} -I options \
        {{proto_dir}}/*.proto

# Generate the Google conformance protos (TestAllTypesProto3) in both modes.
gen-conformance: build
    mkdir -p {{gen_dir}}
    protoc \
        --plugin=./{{plugin}} \
        --tarantool_out={{gen_dir}} \
        --tarantool_opt=mode=full,prefix=full \
        -I {{conformance_proto}} -I options \
        {{conformance_proto}}/*.proto
    protoc \
        --plugin=./{{plugin}} \
        --tarantool_out={{gen_dir}} \
        --tarantool_opt=mode=runtime,prefix=runtime \
        -I {{conformance_proto}} -I options \
        {{conformance_proto}}/*.proto

# Generate the proto2 test fixtures (test/proto/*.proto) in both modes.
# Lives outside examples/ because proto2 is exercised through the luatest
# suite, not the per-example runners.
gen-proto2-tests: build
    mkdir -p {{gen_dir}}
    protoc \
        --plugin=./{{plugin}} \
        --tarantool_out={{gen_dir}} \
        --tarantool_opt=mode=full,prefix=full \
        -I {{proto2_test_dir}} -I options \
        {{proto2_test_dir}}/*.proto
    protoc \
        --plugin=./{{plugin}} \
        --tarantool_out={{gen_dir}} \
        --tarantool_opt=mode=runtime,prefix=runtime \
        -I {{proto2_test_dir}} -I options \
        {{proto2_test_dir}}/*.proto

# Regenerate Markdown reference docs (examples/docs/*.md) — committed output.
gen-docs: build-doc
    mkdir -p {{docs_dir}}
    protoc \
        --plugin=./{{doc_plugin}} \
        --tarantool-doc_out={{docs_dir}} \
        -I {{proto_dir}} -I options \
        {{proto_dir}}/*.proto

# Regenerate test/interop/fixtures/*.bin via mainline `protoc --encode`.
goldens:
    @for f in test/interop/fixtures/*.txtpb; do \
        type=$(awk '/^# type:/ {print $3; exit}' "$f"); \
        out="${f%.txtpb}.bin"; \
        echo "  protoc --encode=$type < $f > $out"; \
        protoc --encode="$type" -I {{proto_dir}} -I options {{proto_dir}}/hello.proto < "$f" > "$out" || exit $?; \
    done

# ---------------------------------------------------------------------------
# Test
# ---------------------------------------------------------------------------

# Run the luatest suite (639 tests, parametrized over both codegen modes).
test: gen
    LUA_PATH="{{lua_path}}" LUA_CPATH="{{lua_cpath}}" {{luatest}} -v test/

# Run a single luatest group or test. Example:
#   just test-one protobuf_test.lua::hello.full.test_packed_repeated_int32
test-one filter: gen
    LUA_PATH="{{lua_path}}" LUA_CPATH="{{lua_cpath}}" {{luatest}} -v test/{{filter}}

# Same as `test`, but with PB_ENABLE_C=1 so pb.encode / pb.decode dispatch
# through the C runtime (runtime/pb/c_runtime.{so,dylib}). Builds the C
# module first. Together with `test`, this is the parity gate — every
# luatest assertion is against a reference output (golden bytes, txtpb,
# conformance result), so if Lua passes and C passes, both equal the
# reference and Lua ≡ C by transitivity. See bd-43t.
test-c: build-c gen
    PB_ENABLE_C=1 LUA_PATH="{{lua_path}}" LUA_CPATH="{{lua_cpath}}" {{luatest}} -v test/

# Parity gate: run the suite under both codecs. Use before pushing C-runtime
# or codec changes.
test-all: test test-c

# ---------------------------------------------------------------------------
# Bench
# ---------------------------------------------------------------------------

# Microbench: alloc + throughput per op across 5 payload sizes, both modes.
bench: gen
    tarantool bench/bench.lua --print

# Same as `bench`, but with PB_ENABLE_C=1 so the runtime-mode results
# exercise the C codec. The runtime-mode column is relabelled to
# `c-runtime` in the output. Full-mode results are unchanged (full mode
# uses inline wire calls, not pb.encode). Allocation baselines (`bench-baseline`
# / `bench-compare`) remain Lua-only — C alloc shape differs by design
# and would noise the gate.
bench-c: build-c gen
    PB_ENABLE_C=1 tarantool bench/bench.lua --print

# Overwrite bench/baseline.json with current alloc-per-op numbers.
bench-baseline: gen
    tarantool bench/bench.lua --baseline

# Fail with exit 1 if any alloc-per-op regressed >5% vs the baseline.
bench-compare: gen
    tarantool bench/bench.lua --compare

# Per-helper microbench for runtime/pb/wire.lua (every primitive).
bench-wire: gen
    tarantool bench/wire_bench.lua

# Shape-variety microbench (scalar-heavy, packed, nested, maps, oneof, WKT).
bench-shapes: gen
    tarantool bench/shapes_bench.lua

# Trace-stability gate: assert hot paths JIT-compile without fatal aborts.
jit-trace: gen
    tarantool bench/jit_trace.lua

# Regenerate Go-side pb.go + vtproto pb.go in bench/go/pb/. Not committed
# (*.pb.go is gitignored repo-wide); run before `bench-go` on a fresh
# checkout. Requires `protoc-gen-go` and `protoc-gen-go-vtproto` on PATH:
#   go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
#   go install github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto@latest
gen-go:
    mkdir -p bench/go/pb/hellopb bench/go/pb/proto2pb
    cd bench/go && PATH="$(go env GOPATH)/bin:$PATH" protoc \
        --go_out=. --go_opt=module=github.com/tarantool-protobuf/bench/go \
        --go-vtproto_out=. --go-vtproto_opt=module=github.com/tarantool-protobuf/bench/go \
        --go-vtproto_opt=features=marshal+unmarshal+size \
        -I proto proto/hello.proto proto/proto2_basic.proto

# Go-side comparison bench: same fixtures + sizes as bench.lua, run against
# google.golang.org/protobuf (apiv2) and planetscale/vtprotobuf. Serial by
# default — Go's testing.B doesn't parallelize unless RunParallel is called.
bench-go: gen-go
    cd bench/go && go test -bench=. -benchmem -run=^$ -benchtime=1s ./...

# ---------------------------------------------------------------------------
# Conformance (Google's protobuf conformance suite, in Docker)
# ---------------------------------------------------------------------------

# Build the conformance Docker image (idempotent).
conformance-build:
    docker build -t {{image}} -f docker/conformance.Dockerfile docker/

# Run the conformance suite with --enforce_recommended (strictest mode).
conformance: conformance-build gen
    docker run --rm -v "$(pwd):/work" -w /work {{image}}

# Same as `conformance`, but exercises the C-acceleration path
# (require('pb.c_runtime')). Builds runtime/pb/c_runtime.so inside the
# container (host .dylib won't load there), then re-runs the suite with
# PB_ENABLE_C=1 so pb/init.lua dispatches to the C codec.
conformance-c: conformance-build gen
    docker run --rm -v "$(pwd):/work" -w /work -e PB_ENABLE_C=1 \
        --entrypoint bash {{image}} -c '\
            set -e; \
            make -C runtime/pb/c clean >/dev/null; \
            make -C runtime/pb/c >/dev/null; \
            conformance_test_runner --enforce_recommended \
                --failure_list test/conformance/known_failures.txt \
                --text_format_failure_list test/conformance/known_failures_text.txt \
                /usr/bin/tarantool cmd/conformance-runner.lua; \
            rc=$?; \
            make -C runtime/pb/c clean >/dev/null; \
            exit $rc'

# Quick pass without --enforce_recommended.
conformance-quick: conformance-build gen
    docker run --rm -v "$(pwd):/work" -w /work --entrypoint conformance_test_runner {{image}} \
        --failure_list test/conformance/known_failures.txt \
        --text_format_failure_list test/conformance/known_failures_text.txt \
        /usr/bin/tarantool cmd/conformance-runner.lua

# Dump failing_tests.txt under test/conformance for triage.
conformance-refresh-failures: conformance-build gen
    docker run --rm -v "$(pwd):/work" -w /work --entrypoint conformance_test_runner {{image}} \
        --enforce_recommended \
        --output_dir /work/test/conformance \
        /usr/bin/tarantool cmd/conformance-runner.lua || true
    @echo "Inspect test/conformance/*failing_tests.txt and update known_failures.txt as needed."

# Open an interactive shell in the conformance image.
conformance-shell: conformance-build
    docker run --rm -it -v "$(pwd):/work" -w /work --entrypoint bash {{image}}

# ---------------------------------------------------------------------------
# Examples — see examples/Justfile for per-example runners
# ---------------------------------------------------------------------------

# Run an example. `just examples` lists per-example recipes.
examples *ARGS:
    just -f examples/Justfile {{ARGS}}

# ---------------------------------------------------------------------------
# Clean
# ---------------------------------------------------------------------------

# Remove built plugin binaries and regenerated outputs.
clean: clean-c
    rm -f {{plugin}} {{doc_plugin}}
    rm -rf {{gen_dir}}

# Remove Tarantool instance state that leaked from running examples.
clean-state:
    rm -f *.snap *.xlog *.vylog *.run *.pid 512.lock
    rm -rf /tmp/tarantool-protobuf-*