From a0005a0576092d45f73bfcc87200ab6a721d5c3a Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Sat, 16 May 2026 23:13:29 +0300 Subject: [PATCH] docs: full reference + how-to set, migrate Makefile to Justfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documentation overhaul that adds the missing user-facing surface: four reference pages (runtime-api, generated-api, cli, grpc-contract), twelve how-tos walking from first-message through custom transports, a troubleshooting page, and a docs/index map. Every how-to references a runnable artifact under examples/, all of them verified end-to-end. Build system migration: the Makefile is gone; the Justfile is now the canonical entry point and absorbs every target. examples/Justfile ships one recipe per runnable example, forwarded via top-level 'just examples '. The 'examples are part of the documented surface' convention is pinned in CLAUDE.md, alongside a dedicated section on updating the conformance harness (PROTOBUF_TAG bumps, libjsoncpp path drift, new test-category wiring). Stale-number sweep across README/PLAN/CLAUDE: fixture count 18→10, test count 613/130→639, wire.lua LOC dropped, M7 marked done. Descriptor-shape block deduplicated against codegen.md as the canonical source. gRPC transports spec status reframed from 'draft / decision deferred' to 'shipped contract; external transports deferred'. .gitignore picks up *.snap / *.xlog / *.vylog / *.run / *.pid / 512.lock so example state can't leak into the working tree. --- .gitignore | 13 +- Justfile | 198 +++++++++++- Makefile | 135 --------- PLAN.md | 189 +++++------- README.md | 47 +-- docs/api-modes.md | 25 +- docs/codegen.md | 24 +- docs/howto/01-first-message.md | 133 ++++++++ docs/howto/02-module-layout.md | 178 +++++++++++ docs/howto/03-grpc-loopback.md | 250 ++++++++++++++++ docs/howto/04-wkt-struct-value.md | 150 ++++++++++ docs/howto/05-wkt-any.md | 156 ++++++++++ docs/howto/06-json-http.md | 160 ++++++++++ docs/howto/07-text-format.md | 117 ++++++++ docs/howto/08-dynamic-schemas.md | 178 +++++++++++ docs/howto/09-lazy-when.md | 142 +++++++++ docs/howto/11-migrate-from-builtin.md | 177 +++++++++++ docs/howto/12-build-integration.md | 212 +++++++++++++ docs/howto/13-custom-transport.md | 207 +++++++++++++ docs/index.md | 88 ++++++ docs/reference/cli.md | 181 +++++++++++ docs/reference/generated-api.md | 253 ++++++++++++++++ docs/reference/grpc-contract.md | 204 +++++++++++++ docs/reference/runtime-api.md | 283 ++++++++++++++++++ docs/specs/grpc_transports.md | 13 +- docs/troubleshooting.md | 276 +++++++++++++++++ examples/Justfile | 57 ++++ examples/dynamic/load_from_space.lua | 105 +++++++ .../full/quickstart/quickstart_pb.lua | 151 ++++++++++ .../runtime/quickstart/quickstart_pb.lua | 65 ++++ examples/grpc/client.lua | 64 ++++ examples/grpc/server.lua | 59 ++++ examples/grpc/transport_netbox_stub.lua | 63 ++++ examples/http/json_api.lua | 59 ++++ examples/proto/quickstart.proto | 25 ++ 35 files changed, 4327 insertions(+), 310 deletions(-) delete mode 100644 Makefile create mode 100644 docs/howto/01-first-message.md create mode 100644 docs/howto/02-module-layout.md create mode 100644 docs/howto/03-grpc-loopback.md create mode 100644 docs/howto/04-wkt-struct-value.md create mode 100644 docs/howto/05-wkt-any.md create mode 100644 docs/howto/06-json-http.md create mode 100644 docs/howto/07-text-format.md create mode 100644 docs/howto/08-dynamic-schemas.md create mode 100644 docs/howto/09-lazy-when.md create mode 100644 docs/howto/11-migrate-from-builtin.md create mode 100644 docs/howto/12-build-integration.md create mode 100644 docs/howto/13-custom-transport.md create mode 100644 docs/index.md create mode 100644 docs/reference/cli.md create mode 100644 docs/reference/generated-api.md create mode 100644 docs/reference/grpc-contract.md create mode 100644 docs/reference/runtime-api.md create mode 100644 docs/troubleshooting.md create mode 100644 examples/Justfile create mode 100644 examples/dynamic/load_from_space.lua create mode 100644 examples/expected/full/quickstart/quickstart_pb.lua create mode 100644 examples/expected/runtime/quickstart/quickstart_pb.lua create mode 100644 examples/grpc/client.lua create mode 100644 examples/grpc/server.lua create mode 100644 examples/grpc/transport_netbox_stub.lua create mode 100644 examples/http/json_api.lua create mode 100644 examples/proto/quickstart.proto diff --git a/.gitignore b/.gitignore index 923e89fe600668c15d3a5b5a584e2d5568e9154f..150e9a9f11da79bfab275f30067e12f310abd7b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /protoc-gen-tarantool +/protoc-gen-tarantool-doc /dist/ /.rocks/ *.pb.go @@ -7,4 +8,14 @@ .vscode/ *.swp *.swo -/protoc-gen-tarantool-doc + +# Tarantool instance state. Generated when running examples or anything +# that calls box.cfg{} from the working directory. Examples under +# examples/dynamic/ point their memtx_dir/wal_dir/log at /tmp; older or +# user-written scripts may leak state here. +*.snap +*.xlog +*.vylog +*.run +*.pid +512.lock diff --git a/Justfile b/Justfile index b58f6dd76ad9812a5d96afaa8e2dd0c78a05fdbd..c5ad99009a4e92c05939c9233d7be19639275343 100644 --- a/Justfile +++ b/Justfile @@ -1,36 +1,184 @@ -# Justfile — task targets that don't fit cleanly into the Makefile. +# Justfile — canonical entry point for build / gen / test / bench / conformance. # -# The Makefile remains canonical for build / gen / test / bench. Use Just -# for orchestration that wraps Docker or other host-level workflows. +# `just` is required (https://github.com/casey/just). On macOS: brew install just +# On Linux: cargo install just +# +# Quick map of common targets: +# just show available recipes +# just build build both plugins +# just gen regenerate examples/expected/{full,runtime}/* +# just test run the luatest suite +# just bench alloc + throughput per op +# just conformance run Google's conformance suite in Docker +# just examples per-example runners (see examples/Justfile) +# +# Sub-recipes live in examples/Justfile (one target per runnable example). set shell := ["bash", "-cu"] -image := "tarantool-protobuf-conformance:latest" +# --------------------------------------------------------------------------- +# Paths and constants +# --------------------------------------------------------------------------- + +plugin := "protoc-gen-tarantool" +doc_plugin := "protoc-gen-tarantool-doc" +gen_dir := "examples/expected" +docs_dir := "examples/docs" +proto_dir := "examples/proto" +conformance_proto := "test/conformance/proto" +luatest := ".rocks/bin/luatest" +image := "tarantool-protobuf-conformance:latest" + +# Semicolon-joined LUA_PATH for the luatest suite. Trailing `;;` defers to the +# standard package.path for everything not explicitly listed. +lua_path := "./runtime/?/init.lua;./runtime/?.lua;./" + gen_dir + "/?.lua;./" + gen_dir + "/?/init.lua;./?.lua;./?/init.lua;./test/?.lua;;" + +# --------------------------------------------------------------------------- +# Default +# --------------------------------------------------------------------------- # Show available recipes. default: @just --list -# Build the conformance image (idempotent — Docker caches layers). +# Build everything from scratch and run the test suite. +all: build gen test + +# --------------------------------------------------------------------------- +# Build +# --------------------------------------------------------------------------- + +# Build the Lua codegen plugin (./protoc-gen-tarantool). +build: + go build -o {{plugin}} ./cmd/protoc-gen-tarantool + +# Build the Markdown doc plugin (./protoc-gen-tarantool-doc). +build-doc: + go build -o {{doc_plugin}} ./cmd/protoc-gen-tarantool-doc + +# --------------------------------------------------------------------------- +# Codegen +# --------------------------------------------------------------------------- + +# Regenerate examples/expected/{full,runtime}/* + conformance protos. +gen: gen-full gen-runtime gen-conformance + +# Generate full-mode Lua (inline encode/decode bodies). +gen-full: build + mkdir -p {{gen_dir}} + protoc \ + --plugin=./{{plugin}} \ + --tarantool_out={{gen_dir}} \ + --tarantool_opt=mode=full,prefix=full \ + -I {{proto_dir}} -I options \ + {{proto_dir}}/*.proto + +# Generate runtime-mode Lua (delegates to pb.encode / pb.decode). +gen-runtime: build + mkdir -p {{gen_dir}} + protoc \ + --plugin=./{{plugin}} \ + --tarantool_out={{gen_dir}} \ + --tarantool_opt=mode=runtime,prefix=runtime \ + -I {{proto_dir}} -I options \ + {{proto_dir}}/*.proto + +# Generate the Google conformance protos (TestAllTypesProto3) in both modes. +gen-conformance: build + mkdir -p {{gen_dir}} + protoc \ + --plugin=./{{plugin}} \ + --tarantool_out={{gen_dir}} \ + --tarantool_opt=mode=full,prefix=full \ + -I {{conformance_proto}} -I options \ + {{conformance_proto}}/*.proto + protoc \ + --plugin=./{{plugin}} \ + --tarantool_out={{gen_dir}} \ + --tarantool_opt=mode=runtime,prefix=runtime \ + -I {{conformance_proto}} -I options \ + {{conformance_proto}}/*.proto + +# Regenerate Markdown reference docs (examples/docs/*.md) — committed output. +gen-docs: build-doc + mkdir -p {{docs_dir}} + protoc \ + --plugin=./{{doc_plugin}} \ + --tarantool-doc_out={{docs_dir}} \ + -I {{proto_dir}} -I options \ + {{proto_dir}}/*.proto + +# Regenerate test/interop/fixtures/*.bin via mainline `protoc --encode`. +goldens: + @for f in test/interop/fixtures/*.txtpb; do \ + type=$(awk '/^# type:/ {print $3; exit}' "$f"); \ + out="${f%.txtpb}.bin"; \ + echo " protoc --encode=$type < $f > $out"; \ + protoc --encode="$type" -I {{proto_dir}} -I options {{proto_dir}}/hello.proto < "$f" > "$out" || exit $?; \ + done + +# --------------------------------------------------------------------------- +# Test +# --------------------------------------------------------------------------- + +# Run the luatest suite (639 tests, parametrized over both codegen modes). +test: gen + LUA_PATH="{{lua_path}}" {{luatest}} -v test/ + +# Run a single luatest group or test. Example: +# just test-one protobuf_test.lua::hello.full.test_packed_repeated_int32 +test-one filter: gen + LUA_PATH="{{lua_path}}" {{luatest}} -v test/{{filter}} + +# --------------------------------------------------------------------------- +# Bench +# --------------------------------------------------------------------------- + +# Microbench: alloc + throughput per op across 5 payload sizes, both modes. +bench: gen + tarantool bench/bench.lua --print + +# Overwrite bench/baseline.json with current alloc-per-op numbers. +bench-baseline: gen + tarantool bench/bench.lua --baseline + +# Fail with exit 1 if any alloc-per-op regressed >5% vs the baseline. +bench-compare: gen + tarantool bench/bench.lua --compare + +# Per-helper microbench for runtime/pb/wire.lua (every primitive). +bench-wire: gen + tarantool bench/wire_bench.lua + +# Shape-variety microbench (scalar-heavy, packed, nested, maps, oneof, WKT). +bench-shapes: gen + tarantool bench/shapes_bench.lua + +# Trace-stability gate: assert hot paths JIT-compile without fatal aborts. +jit-trace: gen + tarantool bench/jit_trace.lua + +# --------------------------------------------------------------------------- +# Conformance (Google's protobuf conformance suite, in Docker) +# --------------------------------------------------------------------------- + +# Build the conformance Docker image (idempotent). conformance-build: docker build -t {{image}} -f docker/conformance.Dockerfile docker/ -# Run the Google conformance suite against cmd/conformance-runner.lua. -conformance: conformance-build - make gen +# Run the conformance suite with --enforce_recommended (strictest mode). +conformance: conformance-build gen docker run --rm -v "$(pwd):/work" -w /work {{image}} -# Same as `conformance`, but skips --enforce_recommended for a quick pass. -conformance-quick: conformance-build - make gen +# Quick pass without --enforce_recommended. +conformance-quick: conformance-build gen docker run --rm -v "$(pwd):/work" -w /work --entrypoint conformance_test_runner {{image}} \ --failure_list test/conformance/known_failures.txt \ --text_format_failure_list test/conformance/known_failures_text.txt \ /usr/bin/tarantool cmd/conformance-runner.lua -# Dump failing_tests.txt under test/conformance for known_failures triage. -conformance-refresh-failures: conformance-build - make gen +# Dump failing_tests.txt under test/conformance for triage. +conformance-refresh-failures: conformance-build gen docker run --rm -v "$(pwd):/work" -w /work --entrypoint conformance_test_runner {{image}} \ --enforce_recommended \ --output_dir /work/test/conformance \ @@ -40,3 +188,25 @@ conformance-refresh-failures: conformance-build # Open an interactive shell in the conformance image. conformance-shell: conformance-build docker run --rm -it -v "$(pwd):/work" -w /work --entrypoint bash {{image}} + +# --------------------------------------------------------------------------- +# Examples — see examples/Justfile for per-example runners +# --------------------------------------------------------------------------- + +# Run an example. `just examples` lists per-example recipes. +examples *ARGS: + just -f examples/Justfile {{ARGS}} + +# --------------------------------------------------------------------------- +# Clean +# --------------------------------------------------------------------------- + +# Remove built plugin binaries and regenerated outputs. +clean: + rm -f {{plugin}} {{doc_plugin}} + rm -rf {{gen_dir}} + +# Remove Tarantool instance state that leaked from running examples. +clean-state: + rm -f *.snap *.xlog *.vylog *.run *.pid 512.lock + rm -rf /tmp/tarantool-protobuf-* diff --git a/Makefile b/Makefile deleted file mode 100644 index 41b498d8eb84c65313abbd8bd329b096b7d64eed..0000000000000000000000000000000000000000 --- a/Makefile +++ /dev/null @@ -1,135 +0,0 @@ -PLUGIN := protoc-gen-tarantool -DOC_PLUGIN := protoc-gen-tarantool-doc -GEN_DIR := examples/expected -DOCS_DIR := examples/docs -PROTO_DIR := examples/proto -CONFORMANCE_PROTO_DIR := test/conformance/proto - -LUATEST := .rocks/bin/luatest -LUA_PATH_PARTS := \ - ./runtime/?/init.lua \ - ./runtime/?.lua \ - ./$(GEN_DIR)/?.lua \ - ./$(GEN_DIR)/?/init.lua \ - ./?.lua \ - ./?/init.lua \ - ./test/?.lua - -# semicolon-joined; trailing ;; lets the standard package.path defaults apply -empty := -space := $(empty) $(empty) -LUA_PATH_JOINED := $(subst $(space),;,$(strip $(LUA_PATH_PARTS)));; - -.PHONY: all build build-doc gen gen-full gen-runtime gen-docs goldens \ - test test-suite bench bench-baseline bench-compare \ - bench-wire bench-shapes jit-trace clean - -all: build gen test - -build: - go build -o $(PLUGIN) ./cmd/protoc-gen-tarantool - -build-doc: - go build -o $(DOC_PLUGIN) ./cmd/protoc-gen-tarantool-doc - -gen: gen-full gen-runtime gen-conformance - -# Render Markdown reference docs for example protos. Output is committed -# so the doc plugin's behavior is visible in PR diffs. -gen-docs: build-doc - mkdir -p $(DOCS_DIR) - protoc \ - --plugin=./$(DOC_PLUGIN) \ - --tarantool-doc_out=$(DOCS_DIR) \ - -I $(PROTO_DIR) -I options \ - $(PROTO_DIR)/*.proto - -gen-full: build - mkdir -p $(GEN_DIR) - protoc \ - --plugin=./$(PLUGIN) \ - --tarantool_out=$(GEN_DIR) \ - --tarantool_opt=mode=full,prefix=full \ - -I $(PROTO_DIR) -I options \ - $(PROTO_DIR)/*.proto - -gen-runtime: build - mkdir -p $(GEN_DIR) - protoc \ - --plugin=./$(PLUGIN) \ - --tarantool_out=$(GEN_DIR) \ - --tarantool_opt=mode=runtime,prefix=runtime \ - -I $(PROTO_DIR) -I options \ - $(PROTO_DIR)/*.proto - -# Conformance + Google test_messages_proto3 are needed by cmd/conformance-runner.lua -# and the conformance self-test. Only the `full` mode is required by the runner. -gen-conformance: build - mkdir -p $(GEN_DIR) - protoc \ - --plugin=./$(PLUGIN) \ - --tarantool_out=$(GEN_DIR) \ - --tarantool_opt=mode=full,prefix=full \ - -I $(CONFORMANCE_PROTO_DIR) -I options \ - $(CONFORMANCE_PROTO_DIR)/*.proto - protoc \ - --plugin=./$(PLUGIN) \ - --tarantool_out=$(GEN_DIR) \ - --tarantool_opt=mode=runtime,prefix=runtime \ - -I $(CONFORMANCE_PROTO_DIR) -I options \ - $(CONFORMANCE_PROTO_DIR)/*.proto - -# Regenerate the interop golden corpus from .txtpb sources using mainline -# protoc. Run only when fixtures change; the generated .bin files are committed. -goldens: - @for f in test/interop/fixtures/*.txtpb; do \ - type=$$(awk '/^# type:/ {print $$3; exit}' $$f); \ - out=$${f%.txtpb}.bin; \ - echo " protoc --encode=$$type < $$f > $$out"; \ - protoc --encode=$$type -I $(PROTO_DIR) -I options $(PROTO_DIR)/hello.proto < $$f > $$out || exit $$?; \ - done - -test: gen - LUA_PATH="$(LUA_PATH_JOINED)" $(LUATEST) -v test/ - -# Microbenchmark: throughput + allocation per op across 5 payload sizes, -# both codegen modes. Throughput numbers print to stderr (informational — -# they vary with CPU load); the JSON document on stdout is the full record. -bench: gen - tarantool bench/bench.lua --print - -# Overwrite bench/baseline.json with current alloc-per-op numbers. Run on -# a quiet machine; allocs are deterministic to ~10 bytes so the file is -# hardware-independent. -bench-baseline: gen - tarantool bench/bench.lua --baseline - -# Fail with exit 1 if any alloc-per-op grows by >5% vs the committed -# baseline. Wire into CI to gate PRs. -bench-compare: gen - tarantool bench/bench.lua --compare - -# Per-helper microbenchmark for the wire layer (encode/decode of every -# primitive + tag + skip + UTF-8). Use when tuning runtime/pb/wire.lua -# to confirm a change moved the helper-level ns/op as expected. -bench-wire: gen - tarantool bench/wire_bench.lua - -# Workload-variety benchmark. Runs encode + decode for several Person -# (and Event / Result) shapes — scalar-heavy, packed-ints, nested -# friends, maps, oneof, WKT — so shape-specific regressions surface -# instead of being averaged out by bench/bench.lua's single shape. -bench-shapes: gen - tarantool bench/shapes_bench.lua - -# Trace-stability gate: assert every hot encode/decode path JIT-compiles -# without fatal aborts (NYI bytecode, blacklisting, persistent type -# instability) in our own source files. Runs as a standalone tarantool -# script — luatest's framework on macOS arm64 exhausts JIT mcode pages -# before tests run, masking the real abort reasons. -jit-trace: gen - tarantool bench/jit_trace.lua - -clean: - rm -f $(PLUGIN) - rm -rf $(GEN_DIR) diff --git a/PLAN.md b/PLAN.md index 2d17a1b9561644c497c74c83322980769822acef..59090b24e2564c73554230e8b45a34f556b14401 100644 --- a/PLAN.md +++ b/PLAN.md @@ -24,19 +24,19 @@ See `README.md` for the user-visible summary. Internally: - **Plugin**: Go, two emission modes (`mode=full` inline + `mode=runtime` descriptor-delegating wrappers). Sibling `protoc-gen-tarantool-doc` emits Markdown reference per `.proto`. -- **Runtime**: pure Lua + LuaJIT FFI. `wire.lua` (~660 LOC) + `codec.lua` +- **Runtime**: pure Lua + LuaJIT FFI. `wire.lua` + `codec.lua` + `lazy.lua` (zero-copy view) + `text.lua` (encode + decode) + `json.lua` (strict proto3 JSON) + `wkt.lua` (all 9 well-known types) + `grpc.lua` (transport + loopback) + `parser.lua` / `dynamic.lua` / `fileset.lua` (three runtime descriptor producers — `.proto` source, AST, FileDescriptorSet bytes). -- **Tests**: 613 luatest assertions across 19 groups, parametrized over +- **Tests**: 639 luatest tests across 15 test files (39 groups), parametrized over both codegen modes where applicable. - **Conformance**: proto3 binary+JSON suite **1493 ✓ / 0 failures**; proto3 text-format suite **416 ✓ / 0 failures** (Google's `conformance_test_runner` v34.1). - **Bench**: `bench/baseline.json` tracks allocation/op (regression - gate at 5%); `make jit-trace` pins LuaJIT trace stability. + gate at 5%); `just jit-trace` pins LuaJIT trace stability. The remaining unfinished bullets in section 3 are mostly M8 (release engineering) and a handful of optional perf items in M6. @@ -213,14 +213,14 @@ fiber and bridges client ↔ handler via `fiber.channel`. All four flavors - [x] Microbenchmarks: encode and decode throughput (MB/s, msgs/s) for messages of 5 sizes (10 B / 100 B / 1 KB / 10 KB / 100 KB). Shipped - as `bench/bench.lua`, run via `make bench`. Throughput is stderr-only + as `bench/bench.lua`, run via `just bench`. Throughput is stderr-only (varies with CPU load); JSON document on stdout. - [x] Allocation profiling — bytes per encode/decode op, measured via GC delta with `collectgarbage('stop')` framing. Committed as - `bench/baseline.json`. Regression gate: `make bench-compare` exits + `bench/baseline.json`. Regression gate: `just bench-compare` exits non-zero if alloc/op grows >5% vs baseline. Allocations are deterministic to ~10 bytes regardless of hardware. -- [x] Trace stability — `make jit-trace` (`bench/jit_trace.lua`) +- [x] Trace stability — `just jit-trace` (`bench/jit_trace.lua`) attaches a `jit.attach('trace')` listener over the hot encode/decode paths and asserts no aborts in our source files fall into the fatal set (NYI bytecode, blacklisting, persistent @@ -277,7 +277,7 @@ fiber and bridges client ↔ handler via `fiber.channel`. All four flavors that also handles sparse reads at parity. Best fit: proxy / router shapes that decode, touch a few fields, and re-encode. -### M7 — Developer ergonomics +### M7 — Developer ergonomics *(done)* - [x] Generated EmmyLua / lua-language-server type annotations so `t:Person_encode({name=...})` autocompletes in editors. @@ -356,8 +356,8 @@ fiber and bridges client ↔ handler via `fiber.channel`. All four flavors message/enum references, and `map` for maps; synthetic map-entry messages are skipped. Leading comments are preserved via SourceCodeInfo (squashed to a single line inside table cells). - Build with `make build-doc`; generate sample docs into - `examples/docs/` with `make gen-docs`. + Build with `just build-doc`; generate sample docs into + `examples/docs/` with `just gen-docs`. [proto3json]: https://protobuf.dev/programming-guides/proto3/#json @@ -428,8 +428,10 @@ Locked: `int64`/`uint64`/`fixed64`/`sfixed64`/`sint64` are LuaJIT - Same convention as Tarantool `msgpackffi`, `net.box`, `box.tuple`. - Compares cleanly against `0` (numeric coercion in LuaJIT). -We will document a `wire.from_string(s)` helper for users who get hex/dec -strings (e.g. from JSON) and need to feed them into encode. +Two coercion helpers — `pb.to_uint64(v)` and `pb.to_int64(v)` (re-exported +from `pb.wire`) — accept Lua number, cdata, or a numeric string and return +the matching 64-bit cdata. Use them at the boundary when values come from +JSON, text format, or net.box arguments where the type isn't already cdata. ### 4.5 Unknown fields *(implemented)* @@ -471,115 +473,80 @@ Server: `M.Greeter_server(impl)` returns a table compatible with the ## 5. Testing strategy -Tests live in `test/`. Layout: - -``` -test/ - unit/ -- pb.wire and pb.codec unit tests (no codegen) - wire_test.lua - codec_test.lua - roundtrip/ -- generated-module round-trip, both modes - scalars_test.lua - repeated_test.lua - nested_test.lua - map_test.lua -- M2 - oneof_test.lua -- M2 - wkt_test.lua -- M3 - conformance/ -- M5: Google conformance harness - interop/ -- M5: cross-impl byte-for-byte equality - fixtures/ -- pre-encoded payloads from Go/Python - bench/ -- M6: microbenchmarks - fuzz/ -- malformed-input + random-input harness -``` - -### 5.1 Unit tests (M1 onwards) - -For each `wire.encode_` / `wire.decode_` pair: -- Round-trip 0, min, max, edge cases (1, -1, NaN, Inf, empty string, - 254/255/256-byte string for varint length-byte boundaries). -- Truncated input → controlled error. -- Spec-conformant byte sequences from the [protobuf encoding doc][encoding]. - -[encoding]: https://protobuf.dev/programming-guides/encoding/ - -### 5.2 Round-trip parity tests (M1+) - -Every test runs against **both** generated modules (full and runtime mode) -using a parameterized luatest group: +Tests live as a flat `test/*.lua` set (15 files, 39 luatest groups, 639 +assertions as of 2026-05-16). Each behavior file is parametrized over +both codegen modes via the pattern: ```lua for _, mode in ipairs({'full', 'runtime'}) do - local g = t.group('roundtrip.' .. mode) - local hello = require('hello.' .. mode .. '.hello_pb') - g.test_address = function() ... end - -- ... + local g = t.group(name .. '.' .. mode) + local hello = require(mode .. '.hello.hello_pb') + g.test_x = function() ... end end ``` -Coverage targets: -- All 15 scalar types, singular and repeated. -- Packed vs non-packed repeated (default + explicit). -- Nested messages, self-references, mutually recursive cycles. -- Cross-file imports (2-file fixture). -- Maps with all valid key types (M2). -- Oneofs with all 3 wire-type families (M2). -- Default-value elision (proto3 semantics). -- Explicit-optional presence (M2). -- Empty messages, single-field messages, 100-field messages. -- Field IDs straddling varint boundaries: 1, 15, 16, 2047, 2048, 2^29 - 1. - -### 5.3 Conformance suite (M5) - -Build a small Lua binary `cmd/conformance-runner.lua` that speaks the -Google protobuf conformance protocol on stdin/stdout. Checked into the -repo so CI runs it against the canonical test corpus. Track pass-rate as -a CI metric. - -### 5.4 Cross-impl interop (M5) - -For each of N reference protos: -1. `protoc --encode=msg < input.txt > golden.bin` (using mainline protoc). -2. Our test asserts `pb.decode(msg, read('golden.bin'))` produces the - expected Lua table. -3. Asserts `pb.encode(msg, expected)` produces bytes that, when decoded - by mainline protoc, yield the same logical message (allow field - reordering since neither order is canonical). - -Generate goldens once via a `make goldens` target; commit them to the -repo. CI just verifies them, never regenerates. - -### 5.5 Fuzz tests (M5/M6) - -- **Malformed-input fuzz**: feed random bytes to every `_decode` function; - assert no crashes, only controlled `error()` calls. -- **Round-trip fuzz**: generate random-but-valid messages (size-bounded), - assert `decode(encode(t)) == t` (value-equal under our equality helper). -- Use Tarantool's `math.random` with a seeded PRNG for reproducibility. - -### 5.6 Performance tests (M6) - -Microbenchmarks measured: -- Encode + decode throughput (msgs/sec, bytes/sec). -- Allocation rate (tables/sec, strings/sec) via `misc.memprof`. -- JIT trace count (no side traces in hot loops). - -Run on a fixed corpus across 5 message sizes; track results in -`bench/baseline.json` and fail PRs that regress >5%. - -### 5.7 Test infrastructure - -- **luatest** as the framework. Already installed via - `tt rocks install luatest` to `.rocks/`. -- Run via `make test` → `.rocks/bin/luatest -v test/`. -- CI: sourcecraft.dev native pipelines (TBD), matrix on Tarantool versions. -- Go side: `go test ./...` for any pure-Go logic in the plugin. -- Go integration test: spawn `protoc` on a fixture, diff generated Lua - against committed expected output. +Categories — see file names under `test/`: + +- **Wire + codec primitives** — exercised indirectly through `protobuf_test` + (scalars, repeated, nested, optional, oneof) and `interop_test` + (the byte-for-byte fixture corpus under `test/interop/fixtures/`, + 10 `.txtpb`/`.bin` pairs from mainline `protoc --encode`). +- **Composites** — `protobuf_test` for map/oneof/explicit-optional. +- **WKT** — `struct_value_test`, `any_fieldmask_test`. +- **Lazy** — `lazy_test` covers MessageView/ArrayView/MapView + the + byte-splice re-encode path. +- **Codec dialects** — `json_test`, `text_test`, `text_decode_test`, + `unknown_test`. +- **Dynamic descriptors** — `dynamic_test` (.proto source via + `pb.parse`), `fileset_test` (FileDescriptorSet via `pb.from_pb`), + with parity assertions against the generated modules. +- **Codegen surface** — `codegen_doc_comments_test`, + `doc_test` (the `protoc-gen-tarantool-doc` plugin), `conformance_test` + (self-test of the runner against crafted requests). +- **gRPC** — `grpc_streaming_test` against the loopback transport. + +Cross-mode parity is pinned by `parity.full_vs_runtime.*` groups +inside each behavior file. + +### Conformance and interop + +The Google protobuf conformance suite is driven by +`cmd/conformance-runner.lua` (stdin/stdout framing) and run via +`just conformance` (Docker-bundled `conformance_test_runner`). Known +failures live in `test/conformance/known_failures.txt` (binary+JSON) +and `test/conformance/known_failures_text.txt` (text-format) — both +empty for proto3 as of 2026-05-16. + +Interop fixtures (`test/interop/fixtures/*.{txtpb,bin}`) are produced +by mainline `protoc --encode` via `just goldens` and asserted in +`interop_test.lua` for round-trip equality across both codegen modes. +Map fixtures use a single key to lock byte-for-byte equality (Lua +`pairs` ordering won't match protoc's text-proto-order output); +multi-key map behavior is covered via decode-then-compare-Lua-table +rather than byte equality. + +### Performance + JIT-trace stability + +`bench/baseline.json` tracks allocation per op (a deterministic +metric; throughput swings 30%+ across machines). `just bench-compare` +fails on >5% alloc regression. `just jit-trace` (`bench/jit_trace.lua`) +asserts the hot encode/decode paths stay on the JIT trace — a +companion safety net for the "no `pairs()` on hot paths" rule. + +### Not (yet) automated + +- Fuzz harnesses (malformed-input + random-valid) — out of band. +- Go-side `go test`. Codegen correctness is asserted end-to-end via + Lua tests against committed expected outputs in + `examples/expected/{full,runtime}/`. +- CI pipeline. Local-only today; sourcecraft.dev wire-up is M8. ## 6. Tooling roadmap -- **Justfile** (or extend Makefile) with targets: `build`, `gen`, - `test`, `bench`, `lint`, `goldens`, `conformance`, `clean`, `release`. +- **Makefile + Justfile** — both shipped. Makefile covers `build`, `gen`, + `test`, `goldens`, `bench`, `bench-baseline`, `bench-compare`, + `jit-trace`, `clean`. Justfile adds `just conformance` (Docker-bundled + conformance runner). - **golangci-lint** for the Go side; **luacheck** for the Lua side. - **gofumpt** + **stylua** for formatting. - **Coverage**: `go test -cover` for plugin; `luacov` for runtime. diff --git a/README.md b/README.md index 89136c13cb37820a5cbf2cd4b75cffdb121303ca..aa93f0f4efcafcf51ddfc9a61a2854564a08a7d6 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ both the binary+JSON and text-format suites passes. | WKT: Struct, Value, ListValue | ✅ | | WKT: Any (opaque + registry pack/unpack) | ✅ | | WKT: FieldMask (strict round-trip) | ✅ | -| Byte-for-byte interop with `protoc` (18 fixtures) | ✅ | +| Byte-for-byte interop with `protoc` (10 fixtures) | ✅ | | **Google conformance suite — proto3 binary+JSON** | **1493 ✓ / 0 failures** | | **Google conformance suite — proto3 text format** | **416 ✓ / 0 failures** | | Runtime `.proto` parsing (`pb.parse`) | ✅ | @@ -49,17 +49,17 @@ both the binary+JSON and text-format suites passes. | proto3 JSON (`pb.json.encode`/`.decode`) | ✅ | | Text format (`pb.text.encode` / `pb.text.decode`) | ✅ | | Unknown-field passthrough (`_unknown_fields`) | ✅ | -| Microbenchmark + alloc regression gate (`make bench`) | ✅ | +| Microbenchmark + alloc regression gate (`just bench`) | ✅ | | proto2 / editions | ❌ deferred (separate slice; see [docs/codegen.md](docs/codegen.md)) | ## Quick start ```bash # 1. Build the plugin and generate the example. -make gen +just gen # 2. Run the round-trip test in Tarantool. -make test +just test ``` The plugin emits one `.lua` file per `.proto`. By default the output path @@ -71,6 +71,9 @@ import "tarantool/tarantool.proto"; option (tarantool.lua_package) = "myapp.proto.foo"; ``` +For a full walk-through that takes a fresh `.proto` to a Tarantool process +encoding and decoding it, see **[docs/howto/01-first-message.md](docs/howto/01-first-message.md)**. + ## Generated API For each message `Foo` the plugin emits: @@ -156,16 +159,20 @@ bench/ per-helper bench + JIT-trace gate + alloc baseline ## Documentation -- **[docs/api-modes.md](docs/api-modes.md)** — when to use `Foo_encode` (full), - `pb.encode(desc, t)` (runtime / reflect), or `pb.decode_lazy(desc, b)` (lazy - view). Measured allocation + throughput trade-offs. -- **[docs/codegen.md](docs/codegen.md)** — plugin internals, descriptor shape - contract, how to add a new wire type or scalar, the LuaJIT hot-path rules - the generated code observes. -- **PLAN.md** — phased roadmap and per-feature design notes. -- **CLAUDE.md** — invariants and conventions enforced across the codebase - (no `pairs()` on hot paths, 64-bit ints as cdata, SoA over AoS for large - index structures, etc.). +Start at **[docs/index.md](docs/index.md)** — the documentation map, +grouped by what you're trying to do (getting started, reference, specs, +internals). The two reference pages worth knowing by name: + +- **[docs/api-modes.md](docs/api-modes.md)** — when to use `Foo_encode` + (full), `pb.encode(desc, t)` (runtime / reflect), or + `pb.decode_lazy(desc, b)` (lazy view). Measured allocation + throughput + trade-offs. +- **[docs/codegen.md](docs/codegen.md)** — plugin internals, descriptor + shape contract, how to add a new wire type or scalar, the LuaJIT + hot-path rules generated code observes. + +Roadmap is in [PLAN.md](PLAN.md); cross-codebase invariants in +[CLAUDE.md](CLAUDE.md). ## Conformance @@ -174,7 +181,7 @@ protocol][gconf] on stdin/stdout. Drive it with the canonical `conformance_test_runner` binary like so: ```bash -make gen +just gen conformance_test_runner --enforce_recommended \ tarantool cmd/conformance-runner.lua ``` @@ -187,7 +194,7 @@ protobuf source and bundles Tarantool. Run the full suite locally with: just conformance ``` -(Mounts the repo into the container — generated Lua from `make gen` on the +(Mounts the repo into the container — generated Lua from `just gen` on the host is what gets tested.) Known failures live in `test/conformance/known_failures.txt` (binary + JSON suite) and `test/conformance/known_failures_text.txt` (text-format suite); both are @@ -210,16 +217,16 @@ binary, JSON, and text-format input/output, including the `JSON_IGNORE_UNKNOWN_PARSING_TEST` category (forwarded as `ignore_unknown_fields=true` to `pb.json.decode`). The self-test in `test/conformance_test.lua` exercises the runner with crafted requests on -every `make test` run. +every `just test` run. [gconf]: https://github.com/protocolbuffers/protobuf/tree/main/conformance ## Benchmarks ```bash -make bench # print throughput + alloc per op (5 sizes × 2 modes) -make bench-baseline # overwrite bench/baseline.json (run on a quiet machine) -make bench-compare # exit 1 if any alloc-per-op regressed >5% vs baseline +just bench # print throughput + alloc per op (5 sizes × 2 modes) +just bench-baseline # overwrite bench/baseline.json (run on a quiet machine) +just bench-compare # exit 1 if any alloc-per-op regressed >5% vs baseline ``` The committed `bench/baseline.json` tracks only allocation per op — that's diff --git a/docs/api-modes.md b/docs/api-modes.md index d6937ff06ad0ab94934f4f9adae7efccd190ad77..ecf54eca704756e7c4be29897e07971ce16ca4ae 100644 --- a/docs/api-modes.md +++ b/docs/api-modes.md @@ -65,7 +65,7 @@ on the hot path. Tag bytes are precomputed Lua string literals. The LuaJIT trace compiler sees a flat sequence of monomorphic wire calls and specializes the whole encode into a single trace. -This is what `make bench` measures by default. The +This is what `just bench` measures by default. The `bench/baseline.json` numbers under the "full" column are this path. ## Runtime (descriptor-driven / reflect) @@ -98,7 +98,10 @@ local desc = set.lookup('hello.Person') local p = pb.decode(desc, bytes) ``` -The descriptor table is the contract: +The descriptor table is the contract — every field-name, presence rule, +and `kind` tag the runtime cares about lives there. **Canonical shape: +[codegen.md → descriptor table](codegen.md#the-descriptor-table--the-contract).** +A minimal `hello.Person` looks like: ```lua { @@ -107,23 +110,17 @@ The descriptor table is the contract: {name='name', id=1, kind='scalar', proto_type='string'}, {name='user_id', id=2, kind='scalar', proto_type='uint64'}, {name='emails', id=3, kind='scalar', proto_type='string', repeated=true}, - {name='status', id=4, kind='enum', enum=}, {name='address', id=5, kind='message', message=}, - {name='ages_by_nickname', id=13, kind='map', - key={kind='scalar', proto_type='string'}, - value={kind='scalar', proto_type='int32'}}, -- modifiers: repeated, packed, oneof, optional }, - field_by_id = {[1]=, ...}, -- filled by pb.finalize_message - field_by_name = {name=, ...}, - oneofs = { = {}}, - reserved_names = {[]=true}, -- for text-format parser } +-- field_by_id / field_by_name / oneofs_list filled by pb.finalize_message ``` -Three producers emit this shape — generated codegen (runtime mode), -`pb.parse` (from .proto source), `pb.from_pb` (from -FileDescriptorSet bytes). All three flow through the same `pb.codec`. +Four producers emit this shape — generated codegen (full or runtime), +`pb.parse` (from .proto source), `pb.from_pb` (from FileDescriptorSet +bytes), and `runtime/pb/wkt.lua` (hand-rolled WKTs). All flow through +the same `pb.codec`. The `pb.finalize_message(desc)` helper builds `field_by_id` / `field_by_name` / `oneofs_list` and attaches per-field `_writer` / @@ -131,7 +128,7 @@ The `pb.finalize_message(desc)` helper builds `field_by_id` / **Cost vs full**: per-field table lookup + kind dispatch. Allocation overhead is one extra `pairs()` walk in encode and a small per-field -function-call cost; on `make bench` numbers the runtime mode is ~5-15% +function-call cost; on `just bench` numbers the runtime mode is ~5-15% slower for encode and within noise for decode (the decoder fast paths are shared via `f._reader` closures). diff --git a/docs/codegen.md b/docs/codegen.md index da5c1b18a4d8a1099d733db1b24bddbd370b5453..9d447e23925e1fc307da0ff452c5a963150f1ebf 100644 --- a/docs/codegen.md +++ b/docs/codegen.md @@ -37,7 +37,7 @@ Passed via `protoc --tarantool_opt==,...`: | Option | Values | Meaning | |---|---|---| | `mode` | `full` (default) or `runtime` | Inline `_encode`/`_decode` vs descriptor-delegating wrappers. See [api-modes.md](api-modes.md). | -| `prefix` | any Lua-require path | Prepended to every generated module's require path **and** its on-disk subpath. The Makefile uses this to generate both modes side-by-side into `examples/expected/{full,runtime}/`. | +| `prefix` | any Lua-require path | Prepended to every generated module's require path **and** its on-disk subpath. The Justfile uses this to generate both modes side-by-side into `examples/expected/{full,runtime}/`. | The `(tarantool.lua_package)` file option (defined in `options/tarantool/tarantool.proto`) overrides the per-file Lua module @@ -137,19 +137,21 @@ The shape consumed by the runtime is: } ``` -**Three producers emit this exact shape.** They live in different +**Four producers emit this exact shape.** They live in different codepaths but consume the same runtime: 1. `cmd/protoc-gen-tarantool/internal/gen` — build-time codegen. 2. `runtime/pb/dynamic.lua` — runtime synthesis from an AST that - `runtime/pb/parser.lua` produces from `.proto` source. + `runtime/pb/parser.lua` produces from `.proto` source + (`pb.parse(text)`). 3. `runtime/pb/fileset.lua` — runtime synthesis from a - `FileDescriptorSet` binary (`protoc --descriptor_set_out=...`). - -Hand-rolled WKT descriptors in `runtime/pb/wkt.lua` use the same shape -but additionally set `desc.encode` / `desc.decode` to take over the wire -path entirely (Timestamp, Duration, Wrappers, Struct, Value, ListValue, -Any, FieldMask, Empty). + `FileDescriptorSet` binary (`pb.from_pb(bytes)`; + produced by `protoc --descriptor_set_out=...`). +4. `runtime/pb/wkt.lua` — hand-rolled WKT descriptors. Same shape but + additionally set `desc.encode` / `desc.decode` (and sometimes + `desc.text`, `desc.json_encode` / `desc.json_decode`) to take over + the wire path entirely. Covers Timestamp, Duration, Wrappers, Struct, + Value, ListValue, Any, FieldMask, Empty. **Pinning this shape is what lets us run every behavior test against both codegen modes.** The test harness in `test/protobuf_test.lua` and @@ -336,8 +338,8 @@ string allocation. `cmd/protoc-gen-tarantool-doc/` is a second Go plugin that consumes the same `CodeGeneratorRequest` and emits one Markdown file per input -`.proto`. Build with `make build-doc`; sample docs land in -`examples/docs/` via `make gen-docs`. It's deliberately separate from +`.proto`. Build with `just build-doc`; sample docs land in +`examples/docs/` via `just gen-docs`. It's deliberately separate from the Lua codegen to keep the codegen plugin small. ## Proto2 deferral diff --git a/docs/howto/01-first-message.md b/docs/howto/01-first-message.md new file mode 100644 index 0000000000000000000000000000000000000000..9fd53b6a734e6e06ae2145a18bfe749a7f4950a3 --- /dev/null +++ b/docs/howto/01-first-message.md @@ -0,0 +1,133 @@ +# How-to: your first message, end-to-end + +Goal: take a `.proto` file from zero to a Tarantool process encoding +and decoding it. + +## What you need + +- `tarantool` (3.x, with LuaJIT 2.1) +- `protoc` (Google's compiler — `brew install protobuf` on macOS, + `apt install protobuf-compiler` on Debian/Ubuntu) +- The `protoc-gen-tarantool` binary on `PATH` — build it with + `just build` from this repo, or `go build -o protoc-gen-tarantool + ./cmd/protoc-gen-tarantool` and copy the result. + +## 1. Write the proto + +The example file lives at `examples/proto/quickstart.proto`: + +```proto +syntax = "proto3"; + +package quickstart; + +enum Role { + ROLE_UNSPECIFIED = 0; + USER = 1; + ADMIN = 2; +} + +message User { + int32 id = 1; + string name = 2; + Role role = 3; + repeated string emails = 4; +} +``` + +## 2. Generate the Lua module + +```bash +protoc --tarantool_out=./out -I. examples/proto/quickstart.proto +``` + +This drops `out/quickstart/quickstart_pb.lua`. The output path mirrors +the proto package (`package quickstart;` → `quickstart/`). + +To change where it lands, see +[how-to: module layout](02-module-layout.md). + +## 3. Round-trip in Tarantool + +The generated module requires `pb` from `runtime/pb/`, so point +`LUA_PATH` at both `runtime/` and the generated `out/` directory: + +```bash +tarantool -e ' +package.path = "./runtime/?/init.lua;./runtime/?.lua;./out/?.lua;./out/?/init.lua;" .. package.path + +local qs = require("quickstart.quickstart_pb") + +-- Encode a Lua table to wire bytes. +local bytes = qs.User_encode({ + id = 7, + name = "Alice", + role = qs.Role.ADMIN, + emails = {"a@x", "b@x"}, +}) +print(#bytes .. " bytes") + +-- Decode back. +local user = qs.User_decode(bytes) +print(user.id, user.name, user.role, user.emails[1], user.emails[2]) + +-- Pretty-print with text format (mainline-protoc compatible). +print(qs.User_text(user)) +' +``` + +Expected output: + +``` +21 bytes +7 Alice 2 a@x b@x +id: 7 +name: "Alice" +role: ADMIN +emails: "a@x" +emails: "b@x" +``` + +## 4. Common issues + +**`module 'pb' not found`** — `LUA_PATH` doesn't include the +`runtime/` directory. The two patterns you need are +`./runtime/?/init.lua` (for `require('pb')` → `runtime/pb/init.lua`) +and `./runtime/?.lua` (for sibling modules like `require('pb.wire')`). + +**`module 'quickstart.quickstart_pb' not found`** — generated path +isn't on `LUA_PATH`. Add `./out/?.lua;./out/?/init.lua;` (or whatever +directory you passed to `--tarantool_out`). + +**`expected cdata int64_t, got number`** — you passed a Lua number +to a `int64` / `uint64` / `fixed64` / `sfixed64` / `sint64` field. +Use `pb.to_uint64(value)` or `pb.to_int64(value)` to coerce: + +```lua +local pb = require("pb") +qs.User_encode({id = pb.to_int64(7)}) +``` + +(The `User.id` example above is `int32`, which accepts a plain Lua +number — only 64-bit-typed fields need the coercion.) + +**`protoc-gen-tarantool: program not found or is not executable`** — +`protoc` couldn't find the plugin on `PATH`. Either copy the binary +to a directory on `PATH`, or invoke `protoc` with an explicit +`--plugin=` flag: + +```bash +protoc --plugin=protoc-gen-tarantool=./protoc-gen-tarantool \ + --tarantool_out=./out ... +``` + +## What's next + +- [Module layout](02-module-layout.md) — control where generated + modules land and what their require paths look like. +- [Reference: generated API](../reference/generated-api.md) — every + symbol the plugin emits per message/enum/service. +- [Reference: runtime API](../reference/runtime-api.md) — what + `require('pb')` gives you beyond `encode` / `decode`. +- [API modes](../api-modes.md) — when to use full vs runtime vs + lazy. diff --git a/docs/howto/02-module-layout.md b/docs/howto/02-module-layout.md new file mode 100644 index 0000000000000000000000000000000000000000..eee71d59a2fb839f4f0cc5a0a60c0b7fad453a55 --- /dev/null +++ b/docs/howto/02-module-layout.md @@ -0,0 +1,178 @@ +# How-to: module layout + +Where generated files land, what `require()` path they get, and how to +control both. The full path-resolution rules live in +[reference/cli.md](../reference/cli.md#path-resolution-rules); this +how-to is the practical "what should I put in my Makefile / Justfile" answer. + +## The defaults + +For a single `.proto` file, with no overrides: + +```proto +// foo.proto +syntax = "proto3"; +package my.app; +message Bar { ... } +``` + +```bash +protoc --tarantool_out=./gen foo.proto +``` + +| Generated file | `require()` path | +|---|---| +| `gen/my/app/foo_pb.lua` | `require('my.app.foo_pb')` | + +The Lua module path is the `package` declaration plus the source file's +basename with `_pb` appended. Subdirectories of the input path do not +appear in the output — only the package and the basename matter. + +If there's no `package`, just the basename: `foo.proto` → +`gen/foo_pb.lua`, `require('foo_pb')`. + +## Three ways to override + +### 1. `prefix=` plugin arg + +Prepend a path to every generated module: + +```bash +protoc --tarantool_out=./gen \ + --tarantool_opt=prefix=apps.myapp \ + foo.proto +``` + +| Without prefix | With `prefix=apps.myapp` | +|---|---| +| `gen/my/app/foo_pb.lua` | `gen/apps/myapp/my/app/foo_pb.lua` | +| `require('my.app.foo_pb')` | `require('apps.myapp.my.app.foo_pb')` | + +Affects every file in the codegen invocation. Useful for vendoring — +"all generated modules belong under `apps.myapp.gen.*`". + +The repo's Justfile uses this trick to produce `full/` and +`runtime/` copies side by side: + +```bash +protoc --tarantool_opt=mode=full,prefix=full ... +protoc --tarantool_opt=mode=runtime,prefix=runtime ... +# => examples/expected/full/... and examples/expected/runtime/... +``` + +### 2. `option (tarantool.lua_package)` — per file + +Override one file's path with a file option: + +```proto +syntax = "proto3"; +package my.app; + +import "tarantool/tarantool.proto"; +option (tarantool.lua_package) = "myapp.proto.foo"; + +message Bar { ... } +``` + +```bash +protoc -Ioptions --tarantool_out=./gen foo.proto +``` + +| Without `lua_package` | With `lua_package = "myapp.proto.foo"` | +|---|---| +| `gen/my/app/foo_pb.lua` | `gen/myapp/proto/foo_pb.lua` | +| `require('my.app.foo_pb')` | `require('myapp.proto.foo_pb')` | + +The `-Ioptions` argument lets `protoc` find the option definition +file at `options/tarantool/tarantool.proto` inside this repo. Copy +that file into your project (or vendor the repo) and use the same +import path. + +If `lua_package` already ends in `_pb`, the plugin doesn't append a +second one — `lua_package = "myapp.foo_pb"` lands at `myapp/foo_pb.lua`, +not `myapp/foo_pb_pb.lua`. + +### 3. Both: `prefix` composes with `lua_package` + +`prefix` is prepended to whatever path the per-file rules produced — +including `lua_package` overrides: + +```bash +protoc -Ioptions \ + --tarantool_out=./gen \ + --tarantool_opt=prefix=vendor \ + foo.proto +``` + +With the `lua_package = "myapp.proto.foo"` option above: + +| Final path | +|---| +| `gen/vendor/myapp/proto/foo_pb.lua` | +| `require('vendor.myapp.proto.foo_pb')` | + +## Side-by-side: three invocations + +Same input proto, three different layouts: + +```bash +# 1. Default — mirrors the package +protoc --tarantool_out=./out1 my/app/foo.proto +# => out1/my/app/foo_pb.lua (require 'my.app.foo_pb') + +# 2. Prefix only — vendor namespace +protoc --tarantool_out=./out2 \ + --tarantool_opt=prefix=apps.myapp \ + my/app/foo.proto +# => out2/apps/myapp/my/app/foo_pb.lua (require 'apps.myapp.my.app.foo_pb') + +# 3. lua_package only (with `option (tarantool.lua_package) = "app.foo";`) +protoc -Ioptions --tarantool_out=./out3 my/app/foo.proto +# => out3/app/foo_pb.lua (require 'app.foo_pb') +``` + +## Wiring `LUA_PATH` + +Two patterns the generated code expects: + +```bash +LUA_PATH="./runtime/?/init.lua;./runtime/?.lua;./gen/?.lua;./gen/?/init.lua;;" +``` + +- `./runtime/?/init.lua` — resolves `require('pb')` to + `runtime/pb/init.lua`. +- `./runtime/?.lua` — resolves sibling modules like + `require('pb.wire')` to `runtime/pb/wire.lua`. +- `./gen/?.lua` and `./gen/?/init.lua` — resolves your generated + modules. Add **both** patterns so flat (`gen/foo_pb.lua`) and + nested (`gen/my/app/foo_pb.lua`) layouts both work. +- The trailing `;;` defers to Lua's built-in path for everything + else (the `box.*` modules, `fiber`, etc.). + +In a `tt`-managed app, drop the same string into the `LUA_PATH` +environment variable in your instance config, or extend +`package.path` from `main.lua` before the first `require`. + +## WKT imports + +`google/protobuf/*.proto` files referenced by your protos are **not** +generated as Lua modules. References to WKT types are rewritten at +codegen time to point at `pb.wkt._descriptor`. The runtime side +ships them in `runtime/pb/wkt.lua`. So: + +```proto +import "google/protobuf/timestamp.proto"; +message Event { + google.protobuf.Timestamp created_at = 1; +} +``` + +Doesn't require a `protoc -I` pointing at the WKT proto path beyond +what your `protoc` already knows about (`protoc` finds them itself). +The plugin will not emit a `google/protobuf/timestamp_pb.lua`. + +## What's next + +- [Reference: CLI](../reference/cli.md) — every flag and option. +- [How-to: build integration](12-build-integration.md) — wiring + this into `make`, `just`, `buf`, CMake. diff --git a/docs/howto/03-grpc-loopback.md b/docs/howto/03-grpc-loopback.md new file mode 100644 index 0000000000000000000000000000000000000000..873dafccb2719bbf4e7d606208a780abeb6b159d --- /dev/null +++ b/docs/howto/03-grpc-loopback.md @@ -0,0 +1,250 @@ +# How-to: gRPC with the loopback transport + +End-to-end Greeter service running entirely in-process: same Tarantool +instance hosts the server, calls it through a generated client. The +loopback transport (`pb.grpc.loopback`) bridges the two via +`fiber.channel` — no network, no HTTP/2 library, no external +dependencies. + +This is the path for tests, same-process apps, and any scenario where +you want gRPC's ergonomics without leaving the process. + +For real (external) transports the contract is documented in +[reference/grpc-contract.md](../reference/grpc-contract.md); a +worked example of writing your own lives in +[how-to: custom transport](13-custom-transport.md). + +## The proto + +We use the `Greeter` service from `examples/proto/hello.proto`. All +four streaming flavors: + +```proto +service Greeter { + rpc SayHello(HelloRequest) returns (HelloReply); + rpc Echo(HelloRequest) returns (HelloRequest); + rpc StreamHellos(HelloRequest) returns (stream HelloReply); + rpc CollectHellos(stream HelloRequest) returns (HelloReply); + rpc Chat(stream HelloRequest) returns (stream HelloReply); +} + +message HelloRequest { string name = 1; } +message HelloReply { string greeting = 1; } +``` + +The repo regenerates this on `just gen`. For a project of your own: +`protoc --tarantool_out=./gen hello.proto`. + +## The server + +Plain Lua table with one function per RPC. Streaming functions +receive a `stream` parameter that exposes `:send`, `:recv`, etc. (see +[grpc-contract.md → stream object — server view](../reference/grpc-contract.md#the-stream-object--server-view)). + +`examples/grpc/server.lua`: + +```lua +local pb = require('pb') +local hello = require('full.hello.hello_pb') + +local impl = { + -- Unary: req in, reply out. + SayHello = function(req, _ctx) + return {greeting = 'Hi ' .. req.name} + end, + + -- Server-stream: push N replies then return. + StreamHellos = function(req, stream, _ctx) + for i = 1, 3 do + stream:send({greeting = ('Hi #%d %s'):format(i, req.name)}) + end + end, + + -- Client-stream: pull until peer closes, return one reply. + CollectHellos = function(stream, _ctx) + local names = {} + while true do + local req, err = stream:recv() + if req == nil then + if err ~= nil then error(err, 0) end + break + end + names[#names + 1] = req.name + end + return {greeting = 'Hi ' .. table.concat(names, ', ')} + end, + + -- Bidi: pull a request, push a reply, repeat until peer closes. + Chat = function(stream, _ctx) + while true do + local req, err = stream:recv() + if req == nil then + if err ~= nil then error(err, 0) end + return + end + stream:send({greeting = 'Echo ' .. req.name}) + end + end, +} + +local server = hello.Greeter_server(impl) +return pb.grpc.loopback(server) +``` + +## The client + +`examples/grpc/client.lua`: + +```lua +local fiber = require('fiber') +local hello = require('full.hello.hello_pb') + +local transport = dofile('examples/grpc/server.lua') -- builds loopback +local client = hello.Greeter_client(transport) + +-- Unary +print(client.SayHello({name = 'Alice'}, {}).greeting) + +-- Server-stream +local s = client.StreamHellos({name = 'Bob'}, {}) +while true do + local msg, err = s:recv() + if msg == nil then break end + print(msg.greeting) +end + +-- Client-stream: send N, close, recv one +local c = client.CollectHellos({}) +c:send({name = 'Alice'}); c:send({name = 'Bob'}); c:send({name = 'Carol'}) +c:close_send() +print(c:recv().greeting) + +-- Bidi: send on one fiber, recv on another +local b = client.Chat({}) +fiber.create(function() + for _, name in ipairs({'X', 'Y', 'Z'}) do b:send({name = name}) end + b:close_send() +end) +while true do + local msg, err = b:recv() + if msg == nil then break end + print(msg.greeting) +end +``` + +Run it: + +```bash +LUA_PATH="./runtime/?/init.lua;./runtime/?.lua;./examples/expected/?.lua;./examples/expected/?/init.lua;;" \ + tarantool examples/grpc/client.lua +``` + +Expected output: + +``` +--- unary --- +Hi Alice +--- server-stream --- +Hi #1 Bob +Hi #2 Bob +Hi #3 Bob +--- client-stream --- +Hi Alice, Bob, Carol +--- bidi --- +Echo X +Echo Y +Echo Z +``` + +## Streaming patterns + +### Bidi from a single fiber + +The example above splits the bidi call across two fibers (one +sends, the main fiber receives). That's the safest pattern — `:send` +blocks when the channel buffer fills, so doing both from one fiber +risks self-deadlock if you're not careful. + +Single-fiber bidi works when send/recv naturally interleave: send, +recv, send, recv, … + +```lua +local b = client.Chat({}) +for _, name in ipairs({'X', 'Y', 'Z'}) do + b:send({name = name}) + print(b:recv().greeting) +end +b:close_send() +``` + +### Cancellation + +Either side can cancel mid-stream: + +```lua +local s = client.StreamHellos({name = 'Bob'}, {}) +local first = s:recv() +s:cancel() -- drop the rest +``` + +Cancellation closes the channel; the server-side `:send` raises +`pb.grpc: stream canceled` on the next attempt. Handlers that don't +need to react gracefully can ignore the raise — the fiber unwinds and +the call ends. + +### Handler errors propagate + +Errors raised in the server's handler surface as the `err` return on +the client's next `:recv()`: + +```lua +StreamHellos = function(req, stream, _ctx) + stream:send({greeting = 'first'}) + error('boom!', 0) +end + +-- Client side: +local s = client.StreamHellos({name = 'x'}, {}) +local r1 = s:recv() -- {greeting = 'first'} +local r2, err = s:recv() -- nil, "boom!" +``` + +The first message goes through; the error surfaces when the channel +drains. + +## The `ctx` argument + +The second-or-third argument to every RPC is `ctx`, an opaque table. +Standard reserved keys (`ctx.deadline`, `ctx.headers`, +`ctx.trace_id`, `ctx.span_id`, `ctx.options`) are documented in +[grpc-contract.md → context](../reference/grpc-contract.md#context-ctx). + +The `loopback` transport doesn't enforce deadlines today — it threads +`ctx` to the handler unchanged. Real external transports that +implement the contract are expected to honor the standard keys. + +## Multiple services on one transport + +```lua +local greeter = hello.Greeter_server(greeter_impl) +local catalog = catalog.Catalog_server(catalog_impl) + +local transport = pb.grpc.multiplex({greeter, catalog}) + +local greeter_client = hello.Greeter_client(transport) +local catalog_client = catalog.Catalog_client(transport) +``` + +`pb.grpc.multiplex` errors on duplicate paths, so two services with +the same package + service name will fail loudly at setup time, not +silently at the first dispatch. + +## What's next + +- [Reference: grpc-contract](../reference/grpc-contract.md) — the + shipped contract, stream objects, helper wrappers. +- [How-to: custom transport](13-custom-transport.md) — when you + need to talk to a real network endpoint. +- [Specs: gRPC transports](../specs/grpc_transports.md) — the + protocol matrix (HTTP/2 gRPC vs Connect vs net.box vs IProto) and + why we recommend each. diff --git a/docs/howto/04-wkt-struct-value.md b/docs/howto/04-wkt-struct-value.md new file mode 100644 index 0000000000000000000000000000000000000000..a58aa61b9cd89b15b3b9054225c13ed7bb2c16ee --- /dev/null +++ b/docs/howto/04-wkt-struct-value.md @@ -0,0 +1,150 @@ +# How-to: round-tripping Struct, Value, and ListValue + +`google.protobuf.Struct`, `Value`, and `ListValue` are designed to +carry arbitrary JSON-shaped data through proto. They're the WKT slice +with the most foot-guns — null sentinel, disambiguation between Struct +vs Value vs ListValue, integer-vs-float — so this how-to walks through +the Lua-side conventions. + +## The shapes + +| Proto type | Lua-side value | Use when | +|---|---|---| +| `Struct` | plain Lua table with string keys | The field is *typed* `Struct`. | +| `ListValue` | plain Lua array (1-based contiguous) | The field is *typed* `ListValue`. | +| `Value` | native Lua of matching shape (scalar / table / array / `pb.NULL`) | The field is `Value` — the codec auto-detects via Lua type. | + +Pick `Struct` or `ListValue` whenever you can — they're unambiguous. +Pick `Value` when the schema needs a "this could be anything" cell. + +## Setting a `Value` field + +```lua +local pb = require('pb') +local hello = require('full.hello.hello_pb') -- has Event.attribute: Value + +-- Scalars round-trip natively: +hello.Event_encode({attribute = 'a string'}) +hello.Event_encode({attribute = 42}) +hello.Event_encode({attribute = true}) +hello.Event_encode({attribute = pb.NULL}) -- JSON null +``` + +For container shapes, the codec needs to know whether you mean a +`Struct` (string-keyed map) or a `ListValue` (1-based array). Two +ways to disambiguate: + +```lua +-- 1. Tag with pb.wkt.struct(t) / pb.wkt.list(t): +hello.Event_encode({attribute = pb.wkt.struct({nested = 'x'})}) +hello.Event_encode({attribute = pb.wkt.list({1, 2, 3})}) + +-- 2. Or pass a table the codec can auto-classify: dictionary-shaped +-- tables become Struct, 1-based contiguous arrays become ListValue. +hello.Event_encode({attribute = {nested = 'x'}}) -- Struct +hello.Event_encode({attribute = {1, 2, 3}}) -- ListValue +``` + +The auto-classifier is good enough for most cases. Use the tags when +the table is ambiguous (empty `{}`, mixed keys, etc.) — explicit beats +the heuristic. + +## Setting a `Struct` field (typed) + +When the schema is `google.protobuf.Struct` directly, the field always +holds a string-keyed table — no disambiguation needed: + +```lua +hello.Event_encode({ + payload = { + greeting = 'hi', + count = 42, + active = true, + when = pb.NULL, + nested = {a = 1, b = 2}, + }, +}) +``` + +Decoded: + +```lua +local d = hello.Event_decode(bytes) +print(d.payload.greeting) -- 'hi' +print(d.payload.count) -- 42 +print(d.payload.when == pb.NULL) -- true +print(d.payload.nested.a) -- 1 +``` + +## Null vs absent + +Two distinct concepts: + +- **Absent** — the field wasn't set. `t.foo` is `nil` (for + `Value` / `Struct`) or `{}` (proto3 default for repeated/map). +- **Null** — the field is explicitly `Value{null_value}` / + `Struct` value of null. Lua-side sentinel is `pb.NULL`. + +```lua +hello.Event_encode({payload = {seen = pb.NULL}}) -- explicit null +hello.Event_encode({payload = {}}) -- empty Struct +hello.Event_encode({}) -- payload absent +``` + +`pb.NULL` is equal to `box.NULL`. Use `pb.NULL` so app code doesn't +need to `require('box')` just for the sentinel. + +## Number gotchas + +Proto3 `Value` is a `double` under the hood — integers and floats +share the wire encoding. The codec preserves Lua integer-vs-float when +encoding (`42` stays integer; `42.5` stays float), but on the +*decode* side everything that travels through `Value` comes back as a +Lua `number`. If you need a 64-bit integer through `Value` you must +either: + +1. Box it as a `Struct` with two fields (`hi` / `lo`), or +2. String-encode it, or +3. Switch to a strongly-typed schema field (`int64`). + +This is a `Value` limitation, not a codec one — JSON has the same +trade-off and `Value` is `Struct`-shaped for JSON-compat reasons. + +## Mixed lists + +`ListValue` carries an array of `Value`s, so mixed-type arrays work: + +```lua +hello.Event_encode({tags = {'a', 'b', 3, true, pb.NULL}}) + +local d = hello.Event_decode(bytes) +-- d.tags = {'a', 'b', 3, true, pb.NULL} +``` + +Each entry passes through the same `Value` auto-classification as +above, so tables inside a `ListValue` need the same `pb.wkt.struct` / +`pb.wkt.list` tagging when ambiguous. + +## Round-tripping through JSON + +The proto3 JSON mapping for `Struct` / `Value` / `ListValue` is +"native JSON" — no envelope, just the shape itself. `pb.json.encode` +emits the table content directly: + +```lua +local pbjson = require('pb').json +print(pbjson.encode(hello.Event_descriptor, { + payload = {greeting = 'hi'}, +})) +-- {"payload":{"greeting":"hi"}} +``` + +This matches mainline protoc and is what makes `Struct` useful as a +JSON-in-proto cell. + +## What's next + +- [How-to: packing and unpacking Any](05-wkt-any.md) — the WKT for + typed message payloads (vs `Struct` which is JSON-typed). +- [Reference: runtime API → WKT](../reference/runtime-api.md#well-known-types--pbwkt) + — every WKT descriptor and helper. diff --git a/docs/howto/05-wkt-any.md b/docs/howto/05-wkt-any.md new file mode 100644 index 0000000000000000000000000000000000000000..50670d638961d064f874054982096863b91d0923 --- /dev/null +++ b/docs/howto/05-wkt-any.md @@ -0,0 +1,156 @@ +# How-to: packing and unpacking `google.protobuf.Any` + +`Any` carries a *typed* message payload as opaque bytes plus a +`type_url`. It's the WKT for "this field holds some other message, +and I don't know which until I look at it." (Compare with `Struct`, +which carries *JSON-shaped* data — see +[how-to: Struct, Value, ListValue](04-wkt-struct-value.md).) + +## The shape + +```lua +-- An Any value is a Lua table of this shape: +{ + type_url = 'type.googleapis.com/hello.Address', + value = '', +} +``` + +You can pass this verbatim into any `Any`-typed field. The runtime +provides two helpers to make pack/unpack ergonomic: +`pb.any.pack(desc, t)` and `pb.any.unpack(any_t, desc?)`. + +## Packing + +```lua +local pb = require('pb') +local hello = require('full.hello.hello_pb') + +local addr = {street = '5th Ave', city = 'NYC', zip = 10001} +local any_t = pb.any.pack(hello.Address_descriptor, addr) + +-- any_t = { +-- type_url = 'type.googleapis.com/hello.Address', +-- value = '', +-- } + +-- Drop it into an Any-typed field on some other message: +local e = hello.Event_encode({title = 'demo', extension = any_t}) +``` + +By default the type URL prefix is `type.googleapis.com` (the gRPC +convention). Override with a third argument: + +```lua +pb.any.pack(hello.Address_descriptor, addr, 'myorg.example.com') +-- type_url = 'myorg.example.com/hello.Address' +``` + +The part *after* the last `/` is the fully-qualified proto name; that's +what `pb.any.unpack` keys off, so the prefix is purely informational +unless your registry is prefix-aware. + +## Unpacking + +Two flavors: + +### Explicit descriptor + +```lua +local d = hello.Event_decode(e) +local addr = pb.any.unpack(d.extension, hello.Address_descriptor) +print(addr.street, addr.city) +``` + +This is the safe path — you've validated the descriptor matches what +you expect. Works without any registry setup. + +### Registry lookup + +If you have many possible types, register them up-front: + +```lua +pb.register(hello.Address_descriptor) +pb.register(hello.Person_descriptor) +-- ...register every descriptor you might unpack... + +local payload = pb.any.unpack(d.extension) -- no second arg +``` + +`pb.any.unpack` reads `type_url`, strips the prefix, and looks up +`` in the registry. If not registered, it errors. + +Registry registration is **not automatic** — generated `_pb.lua` +doesn't call `pb.register` on its descriptors. Decide which types +your app accepts via `Any` and register exactly those. + +## When `type_url` is unknown + +If you receive an `Any` whose type you don't recognize, leave it +opaque. The Lua table form is good enough for passthrough: + +```lua +local function handle(any_t) + if any_t.type_url == 'type.googleapis.com/hello.Address' then + local addr = pb.any.unpack(any_t, hello.Address_descriptor) + return process_address(addr) + elseif any_t.type_url == 'type.googleapis.com/hello.Person' then + local p = pb.any.unpack(any_t, hello.Person_descriptor) + return process_person(p) + else + -- Unknown type — log, drop, or re-emit verbatim: + log.warn('unknown Any type: ' .. any_t.type_url) + return nil + end +end +``` + +Re-encoding the opaque `{type_url, value}` table back into the outer +message is byte-stable — the codec doesn't second-guess the inner +bytes when you didn't unpack them. That makes proxy/router shapes +work without registering every possible type. + +## JSON form + +The proto3 JSON canonical mapping for `Any` is the flat +`{"@type": "...", "fieldA": ..., "fieldB": ...}` envelope when the type +is registered: + +```lua +pb.register(hello.Address_descriptor) +local pbjson = require('pb').json + +print(pbjson.encode(hello.Event_descriptor, { + title = 'demo', + extension = pb.any.pack(hello.Address_descriptor, addr), +})) +-- {"title":"demo","extension":{"@type":"type.googleapis.com/hello.Address","street":"5th Ave","city":"NYC","zip":10001}} +``` + +When the type isn't registered, the JSON encoder falls back to the +opaque form: `{"@type":"...","value":""}`. + +## Security notes + +- An attacker who controls the `type_url` can make your code decode + arbitrary registered types. Limit `pb.register` to the types you + actually want to accept; treat the registry as an allowlist. +- The `value` bytes are *not* validated until you unpack. A + `value`-only check (length, prefix bytes) is cheap and useful + before calling `pb.any.unpack` on untrusted input. +- `type_url`'s prefix is purely informational unless your code + enforces it. If you care that requests come from a specific + authority, check the prefix explicitly: + +```lua +if not any_t.type_url:find('^type.googleapis.com/') then + error('unexpected type_url prefix', 0) +end +``` + +## What's next + +- [Reference: runtime API → Any](../reference/runtime-api.md#any) + — full signatures. +- [How-to: Struct/Value/ListValue](04-wkt-struct-value.md) — the + JSON-shaped sibling to `Any`. diff --git a/docs/howto/06-json-http.md b/docs/howto/06-json-http.md new file mode 100644 index 0000000000000000000000000000000000000000..f8a5626ed870ded3a9c701f3d01479fa3f4fe6c3 --- /dev/null +++ b/docs/howto/06-json-http.md @@ -0,0 +1,160 @@ +# How-to: JSON over `tarantool/http` + +Use `pb.json` to expose proto-defined messages as a JSON HTTP API. +This is the lowest-friction way to give external clients (browsers, +curl, mobile) a typed API without running HTTP/2 termination or a +gRPC proxy. + +The wire is JSON; the *schema* is your `.proto`. Same shape on both +sides; same field names (camelCase per proto3 JSON spec — see +[the proto3 JSON mapping][proto3json] for the canonical rules the +codec follows). + +[proto3json]: https://protobuf.dev/programming-guides/proto3/#json + +## Setup + +Install the `http` rock (`tt rocks install http`) and the standard +runtime/example `LUA_PATH`: + +```bash +LUA_PATH="./runtime/?/init.lua;./runtime/?.lua;./examples/expected/?.lua;./examples/expected/?/init.lua;;" \ + tarantool examples/http/json_api.lua +``` + +## The handler + +`examples/http/json_api.lua`: + +```lua +local pb = require('pb') +local http = require('http.server') +local hello = require('full.hello.hello_pb') + +local httpd = http.new('127.0.0.1', 8080) + +httpd:route({path = '/v1/users', method = 'POST'}, function(req) + local body = req:read_cached() + + -- proto3 JSON → Lua table, validated against the schema. + local ok, person = pcall(pb.json.decode, hello.Person_descriptor, body, { + ignore_unknown_fields = true, + }) + if not ok then + return {status = 400, + headers = {['content-type'] = 'application/json'}, + body = pb.json.encode(hello.HelloReply_descriptor, + {greeting = 'bad request: ' .. person})} + end + + -- Business logic. + person.user_id = pb.to_uint64(42) + + return { + status = 200, + headers = {['content-type'] = 'application/json'}, + body = pb.json.encode(hello.Person_descriptor, person), + } +end) + +httpd:start() +``` + +Call it: + +```bash +curl -X POST http://127.0.0.1:8080/v1/users \ + -H 'Content-Type: application/json' \ + -d '{"name":"Alice","age":30,"emails":["a@x"]}' +``` + +Response: + +```json +{"age":30,"name":"Alice","emails":["a@x"],"userId":"42"} +``` + +Notes on the response: + +- `userId` is camelCase per the proto3 JSON spec, even though the + proto field is `user_id`. +- `userId` is a JSON **string** because the field is `fixed64` and + 64-bit integers don't fit a JSON `number` losslessly. The codec + follows the spec here; `int32` and `uint32` come out as numbers. + +## `ignore_unknown_fields` + +Default is **strict** — `pb.json.decode` errors on unknown keys. Pass +`{ignore_unknown_fields = true}` to accept them silently (matches the +conformance suite's `JSON_IGNORE_UNKNOWN_PARSING_TEST` category and is +the right choice for forward-compat: clients can send fields you +haven't shipped support for yet). + +## Mapping the proto3 JSON rules + +What the codec does, per the spec: + +| Proto type | JSON shape | +|---|---| +| `string` | string | +| `bytes` | base64 string | +| `bool` | bool | +| `int32`, `uint32`, `enum` | number (or enum-name string for known enum values) | +| `int64`, `uint64`, `fixed64`, `sfixed64`, `sint64` | **string** (lossless 64-bit) | +| `float`, `double` | number, or `"NaN"` / `"Infinity"` / `"-Infinity"` | +| `repeated T` | array | +| `map` | object (keys coerced to strings per spec) | +| `Timestamp` | RFC 3339 string (`"2026-05-16T10:30:00Z"`) | +| `Duration` | string with `s` suffix (`"3.5s"`) | +| `Struct` / `Value` / `ListValue` | native JSON of matching shape | +| `Any` (registered) | flat object with `"@type": "..."` | +| `Any` (unregistered) | `{"@type": "...", "value": ""}` | +| `FieldMask` | lowerCamelCase paths joined by `,` | +| `Value` wrappers | unwrapped scalar | +| missing proto3 implicit field | omitted from JSON (proto3 default) | +| missing explicit-optional | omitted; decoded back as `nil` | + +## Errors + +`pb.json.encode` raises on: +- 64-bit fields holding bare Lua numbers (use `pb.to_int64` / + `pb.to_uint64`) +- Invalid UTF-8 in `string` fields +- Out-of-spec `Timestamp` (negative nanos, year > 9999) +- A `Value` holding a Lua type the spec doesn't map (function, + userdata) + +`pb.json.decode` raises on: +- Malformed JSON (parse error) +- Unknown keys, unless `ignore_unknown_fields = true` +- Duplicate keys at the same nesting level (strict-validation pass) +- Numbers that don't fit the target proto type (overflow, + fractional values for integer fields) + +Wrap calls in `pcall` and surface a clean error response to the +client. + +## Production checklist + +- **Content type negotiation.** Real services should support both + `application/json` (this howto) and `application/proto` (raw wire + bytes via `Foo_encode`/`Foo_decode`). Branch on + `req.headers['content-type']`. +- **Schema versioning.** Adding fields is wire-compatible; removing + fields means old clients can still send them, and JSON + `ignore_unknown_fields` lets you ignore the leftovers. Reuse + field numbers only after a `reserved` declaration. +- **Logging.** Don't log raw bodies that may contain credentials or + PII. The codec roundtrips `bytes` fields as base64 — if you log a + decoded message you'll see the base64; if you log the raw body + you'll see whatever JSON was sent. +- **Streaming responses.** `pb.json` is one-shot (encode the whole + message). For NDJSON-style streaming, encode each message + separately and join with `\n`. + +## What's next + +- [Reference: runtime API → JSON](../reference/runtime-api.md#pbjson) + — full `pb.json.encode` / `pb.json.decode` signatures. +- [How-to: Struct, Value, ListValue](04-wkt-struct-value.md) — when + your JSON has dynamic shape inside a fixed schema. diff --git a/docs/howto/07-text-format.md b/docs/howto/07-text-format.md new file mode 100644 index 0000000000000000000000000000000000000000..97e581175b5377696c2109193453f567c663775b --- /dev/null +++ b/docs/howto/07-text-format.md @@ -0,0 +1,117 @@ +# How-to: text format for debugging + +Mainline `protoc --decode` produces a readable, line-oriented text +representation of any proto message. The `pb` runtime emits the +same shape via `pb.text.encode` and parses it via `pb.text.decode`. + +Use it for: + +- **Log lines** — set `single_line = true` and you get a one-liner + that grep / jq won't choke on. +- **Test fixtures** — `.txtpb` files round-trip into bytes via + mainline `protoc --encode` and into Lua tables via + `pb.text.decode`; the conformance suite uses this pattern for the + 10-fixture interop corpus. +- **Debugging** — when a JSON dump elides 64-bit precision or you + want to *see* the field IDs, the text form is more faithful. + +## Encode + +Generated codegen emits a `M._text` wrapper: + +```lua +local hello = require('full.hello.hello_pb') + +print(hello.Person_text({name = 'Alice', age = 30, emails = {'a@x'}})) +-- name: "Alice" +-- age: 30 +-- emails: "a@x" + +-- Compact, one-line form: +print(hello.Person_text({name = 'Alice', age = 30}, {single_line = true})) +-- name: "Alice" age: 30 + +-- Custom indent: +print(hello.Person_text(t, {indent = ' '})) +``` + +Options: + +| Key | Default | Meaning | +|---|---|---| +| `single_line` | `false` | Collapse to a single space-separated line — log-friendly. | +| `indent` | `' '` (two spaces) | Per-nesting-level indent. Ignored when `single_line = true`. | + +## Decode + +There's no per-message `_text_decode` wrapper — call +`pb.text.decode(desc, text)` directly. (The wrapper was deliberately +not generated; it's used in a few places like the conformance +runner, and didn't warrant codegen surface.) + +```lua +local pb = require('pb') + +local t = pb.text.decode(hello.Person_descriptor, [[ + name: "Alice" + age: 30 + emails: "a@x" + emails: "b@x" +]]) + +print(t.name, t.age, t.emails[1]) +``` + +The parser handles every grammar bucket the proto3 conformance text +suite exercises: + +- Integer literals: decimal, `0x` hex, `0`-prefix octal. +- Float specials: `inf` / `infinity` / `nan` (any case), oversize + exponents saturating to ±inf, underflow to ±0. +- C-style escapes (`\n`, `\xFF`, `\377`) and `\u`/`\U` escapes with + adjacent-literal concat. +- Aggregate `{ ... }` and `< ... >` bodies for sub-messages. +- Repeated short-form `field: [a, b, c]`. +- Map entries: `field { key: K value: V }`. +- Inline `Any`: `field [type.googleapis.com/Foo] { ... }`. +- Enums by name (`status: OK`) or number (`status: 1`). +- Reserved-name fields are silently dropped. +- Numeric field IDs tolerated: `42: "value"` works alongside + `name: "value"`. + +## Round-tripping via mainline `protoc` + +The interop suite leans on this: + +```bash +# Encode a .txtpb fixture to wire bytes via mainline protoc. +protoc --encode=hello.Person hello.proto < person.txtpb > person.bin + +# Decode it back via pb. +local bytes = io.open('person.bin', 'rb'):read('*a') +local p = hello.Person_decode(bytes) +print(hello.Person_text(p)) +``` + +The text form is what makes the fixtures human-readable in +`test/interop/fixtures/*.txtpb` while the wire-equality check runs +against the matching `.bin` files. + +## Limitations + +- The text form is mainline-protoc-compatible, not stable across + versions. Don't use it as a persistence format. +- Comments aren't preserved by the parser (they're stripped to + whitespace, like in JSON). +- Float printing uses Lua's `%.*g` formatting — bit-exact + round-trip is guaranteed (the codec picks precision to hit + IEEE-754 exact reconstruction), but adjacent-fixture *byte* + equality vs mainline `protoc` can differ in non-significant + digits. + +## What's next + +- [Reference: runtime API → pb.text](../reference/runtime-api.md#pbtext) + — full signatures. +- [How-to: JSON over HTTP](06-json-http.md) — the structurally- + parsed alternative when humans aren't reading the bytes. diff --git a/docs/howto/08-dynamic-schemas.md b/docs/howto/08-dynamic-schemas.md new file mode 100644 index 0000000000000000000000000000000000000000..12a7252b910c50c5603a5e8bcd23184e96ed1604 --- /dev/null +++ b/docs/howto/08-dynamic-schemas.md @@ -0,0 +1,178 @@ +# How-to: dynamic schemas from a Tarantool space + +When you can't (or don't want to) run `protoc` at build time — schema +registries, multi-tenant apps where each tenant has their own messages, +plugin systems where users upload protos — `pb.parse` and `pb.from_pb` +let you build descriptor modules at runtime from `.proto` source or +binary `FileDescriptorSet` bytes. + +## Two entry points + +| Function | Input | Best for | +|---|---|---| +| `pb.parse(text)` | `.proto` source string | User-uploaded protos, schemas living in a space as text, REPL exploration. | +| `pb.from_pb(bytes)` | Binary `FileDescriptorSet` | `protoc --descriptor_set_out=…` artifacts; gRPC reflection responses; multi-file schemas with imports. | + +Both return modules **shaped exactly like generated `mode=runtime` +output** — same `_descriptor`, `_encode`, `_decode`, +`_decode_lazy`, `_text` surface. Anything that consumes a +generated module consumes these. + +## `pb.parse` — source string in, module out + +```lua +local pb = require('pb') + +local source = [[ +syntax = "proto3"; +package demo; + +message User { + int32 id = 1; + string name = 2; + repeated string tags = 3; +} +]] + +local demo = pb.parse(source) + +local bytes = demo.User_encode({ + id = 42, name = 'Alice', tags = {'admin', 'active'}, +}) +local user = demo.User_decode(bytes) +``` + +The parser handles every proto3 grammar bucket: services, options, +imports, nested messages, oneofs, maps, explicit-optional. WKT imports +(`google/protobuf/*.proto`) resolve to the runtime's `pb.wkt` +descriptors automatically — no extra setup needed. + +## Storing schemas in a space + +The runnable example lives at +`examples/dynamic/load_from_space.lua`. Pattern: + +```lua +local pb = require('pb') + +-- One row per schema, keyed by name + carrying a version for invalidation. +box.schema.space.create('proto_schemas') +box.space.proto_schemas:format({ + {name = 'name', type = 'string'}, + {name = 'version', type = 'unsigned'}, + {name = 'source', type = 'string'}, +}) +box.space.proto_schemas:create_index('pk', {parts = {'name'}}) + +-- Cache parsed modules by (name, version). Bumping the version in the +-- row evicts the cache entry naturally on next get_module. +local cache = {} + +local function get_module(name) + local row = box.space.proto_schemas:get(name) + if row == nil then error('schema not found: ' .. name, 0) end + local key = name .. '@' .. tostring(row.version) + local mod = cache[key] + if mod == nil then + mod = pb.parse(row.source) + cache[key] = mod + end + return mod +end +``` + +Why version-keyed cache: an UPDATE on the row bumps the version, so +the next `get_module` parses the new source and the old module +naturally becomes unreachable. + +## Schema evolution + +Adding a field is wire-compatible. Bytes encoded with the old schema +decode fine with the new one (the new field reads as its proto3 +default; explicit-optional reads as `nil`): + +```lua +local bytes_v1 = old.User_encode({id = 42, name = 'Alice'}) + +box.space.proto_schemas:replace{'demo.user', 2, [[ +syntax = "proto3"; +package demo; +message User { + int32 id = 1; + string name = 2; + repeated string tags = 3; + string email = 4; -- new +} +]]} + +local user_via_v2 = get_module('demo.user').User_decode(bytes_v1) +-- user_via_v2.email == "" (proto3 default for implicit-presence string) +``` + +Removing a field is **not** wire-compatible — the bytes for that field +become "unknown" to the new schema. The runtime preserves them via +`_unknown_fields` so a round-trip back through the same instance +keeps the data, but consumers that only see the new schema can't see +the field. + +Reusing a field number after removal is a category of data corruption +the wire format can't catch. If you remove a field, `reserved 4;` the +number to prevent reuse. + +## `pb.from_pb` — for compiled descriptor sets + +When you have a `FileDescriptorSet` (from `protoc --descriptor_set_out` +or a gRPC reflection response), `pb.from_pb` ingests it: + +```lua +local pb = require('pb') + +-- Build the FileDescriptorSet at deploy time, ship it as a blob. +-- protoc --descriptor_set_out=schemas.pb --include_imports my/app/*.proto +local bytes = io.open('schemas.pb', 'rb'):read('*a') +local set = pb.from_pb(bytes) + +-- Lookup by file name (the original .proto path): +local hello = set.files['hello.proto'] +local user_bytes = hello.User_encode({...}) + +-- Or by fully-qualified message name across all files: +local desc = set.lookup('hello.Person') +local user_bytes = pb.encode(desc, {...}) +``` + +`--include_imports` is important — it embeds every transitively-imported +file in the set. Without it, references to imports won't resolve. + +## When to pick which + +| Use case | Pick | +|---|---| +| Schemas authored by users (admin UI, REST endpoint) | `pb.parse` — they paste `.proto` source. | +| Multi-tenant where each tenant has their own schemas | `pb.parse`, keyed by tenant in a space. | +| Schemas built at deploy time, distributed as a blob | `pb.from_pb` — runs ~10x faster than `pb.parse` on large schemas, and you skip the parser entirely. | +| gRPC server reflection consumer | `pb.from_pb` — the response is a `FileDescriptorSet`. | +| REPL exploration | `pb.parse` — paste a string. | + +For known-at-build-time schemas, generated `_pb.lua` is still the +right choice — same module shape, no startup cost, JIT-friendly +inline encode/decode in `mode=full`. + +## Cost notes + +`pb.parse` does real lexing/parsing work — keep it out of hot paths. +Parse once at boot (or per-schema-update) and cache the resulting +module. Encode/decode against a parsed-descriptor module runs the +same codec as generated `mode=runtime`, ~5-15% slower than full-mode +inline; allocation profile matches. + +`pb.from_pb` skips parsing entirely (it consumes binary protobuf +itself, via the hand-built `descriptor.proto` descriptors in +`runtime/pb/descriptor_pb.lua`). + +## What's next + +- [Reference: runtime API](../reference/runtime-api.md#dynamic-descriptors) + — full `pb.parse` / `pb.from_pb` signatures. +- [How-to: build integration](12-build-integration.md) — generating + `--descriptor_set_out` from your build system. diff --git a/docs/howto/09-lazy-when.md b/docs/howto/09-lazy-when.md new file mode 100644 index 0000000000000000000000000000000000000000..8d6d56fc3d076823b3a0c2e33622e152d3898513 --- /dev/null +++ b/docs/howto/09-lazy-when.md @@ -0,0 +1,142 @@ +# How-to: when to use `decode_lazy` + +The lazy decoder (`pb.decode_lazy` / generated `Foo_decode_lazy`) +trades a higher fixed cost per message for near-zero cost on +subsequent field reads. Pick it when the math works out; stick with +the eager `Foo_decode` otherwise. + +The lazy view's full surface is documented in +[api-modes.md → lazy](../api-modes.md#lazy-zero-copy-view) (`:get`, +`:has`, `:set`, `:iter`, `:totable`, the field-name constants +contract). This page is the *picking* guide. + +## When lazy wins + +### 1. Sparse reads + +If you decode a large message and read only a few fields, the eager +decoder wastes work on everything else. Lazy builds a wire-segment +index (one Lua table + four SoA int arrays) and only materializes +fields you `:get`. + +The cross-over on `hello.Person` (the bench message) is roughly **1 KB**: +below that, eager wins on tiny payloads; above, lazy is competitive +on dense reads and wins on sparse ones. See `bench/baseline.json` for +the exact alloc-per-op numbers eager hits at each size. + +### 2. Mostly-passthrough re-encode (proxy / router shapes) + +Decode → look at a few fields → re-encode. Lazy's byte-splice path +re-emits untouched fields verbatim — only fields you `:set` go through +the encoder. On the 100 KB `hello.Person` workload this puts the +mutate-then-reencode shape at **1.09-1.26× of eager**'s full decode + +encode round-trip. + +```lua +local view = hello.Person_decode_lazy(bytes) +local F = hello.Person_fields + +-- Look at one field, mutate another, re-emit. +local id = view:get(F.user_id) +if id > 0 then + view:set(F.user_id, pb.to_uint64(id + 1)) +end +local new_bytes = view:encode() +``` + +### 3. Iterating large repeated fields without intermediate tables + +`ArrayView:iter()` yields one value at a time without allocating a +flat Lua array first. Useful when the result of the iteration is +something other than "I want all the values in a Lua table" (filter, +fold, find-first). + +```lua +local view = hello.Person_decode_lazy(bytes) +for i, email in view:get(hello.Person_fields.emails):iter() do + if email:find('@example%.com$') then + log.info('found ' .. email) + break + end +end +``` + +## When lazy loses + +### 1. Dense reads + +If you read every field, the index build is wasted work and each +`:get` adds a Lua-call boundary the eager decoder avoided. Eager wins +by ~10-30% on full-field walks. Rule of thumb: if you find yourself +calling `:totable()` to get a plain Lua table out, you wanted the +eager decoder to begin with. + +### 2. Tiny messages + +Below ~1 KB on `hello.Person`, the index build dominates. The +eager decoder also has a flat trace and dispatches into typed +helpers in one shot; lazy has unavoidable overhead from MessageView's +allocation and the SoA index initialization. + +### 3. Code paths that need a plain Lua table + +`:totable()` reverses the win — it forces a full materialize. Any +consumer that wants the result as a plain Lua table (JSON encoding +through `pb.json.encode`, persistence, comparing equal to another +table) is paying eager-decode work plus the index-build overhead. + +## Picking by workload shape + +| Workload | Pick | Reasoning | +|---|---|---| +| RPC handler that reads every field and runs business logic | Eager | Dense reads; index build is overhead. | +| Stream filter that drops 95% of messages after looking at one field | Lazy | Sparse reads; the 95% never paid for the body. | +| Proxy / router (decode, log, mutate one header, re-encode) | Lazy | Byte-splice re-encode skips most of the work. | +| Periodic snapshot that serializes 1k messages to JSON | Eager | `pb.json.encode` materializes everything anyway. | +| Lookup-by-id over a large repeated message | Lazy with `ArrayView:iter()` | Find-first avoids intermediate flat array. | +| Tiny messages (<1 KB), any access pattern | Eager | Index build dominates. | +| Storing decoded results across many requests | Eager | Lazy views hold references to the input bytes; keep allocations short-lived. | + +## Field-name constants — required for lazy + +The lazy view's `:get` / `:has` / `:set` / `:clear` / `:which` take a +field name **string**, not a Lua identifier. Always route through the +generated strict-table constants: + +```lua +-- right +local F = hello.Person_fields +view:get(F.user_id) +-- typo: errors at the read site +-- view:get(F.user_di) -- "unknown field name: 'user_di'" + +-- wrong +view:get('user_id') -- works +view:get('user_di') -- silently returns nil (looks like absent) +``` + +Without the constants, a typo and a legitimately-absent field are +indistinguishable. The strict table catches the typo where it was +written. This is a lazy-view contract — the eager decoder doesn't +need it because misspelled table keys fail visibly in tests. + +See [api-modes.md → field-name constants](../api-modes.md#field-name-constants--required) +for the full reasoning. + +## Measuring + +`just bench` prints alloc per op for full + runtime mode at 5 payload +sizes. `bench/baseline.json` is the committed reference. + +For lazy specifically there's `bench/lazy_bench.lua` (run via the +same `just bench` target) that exercises the sparse-read and +mostly-passthrough-reencode shapes against the eager baseline. Use +this when comparing a hypothetical lazy migration against the +eager-decode allocation budget you already pay. + +## What's next + +- [API modes](../api-modes.md) — the full overview of full / + runtime / lazy. +- [Reference: runtime API → decode_lazy](../reference/runtime-api.md#pbdecode_lazydesc-bytes---messageview) + — signature and link to `MessageView` / `ArrayView` / `MapView`. diff --git a/docs/howto/11-migrate-from-builtin.md b/docs/howto/11-migrate-from-builtin.md new file mode 100644 index 0000000000000000000000000000000000000000..c993a4b9f38f8024b90d095cfac5a2e0298689b4 --- /dev/null +++ b/docs/howto/11-migrate-from-builtin.md @@ -0,0 +1,177 @@ +# How-to: migrating from the built-in `require('protobuf')` + +Tarantool ships an in-tree `protobuf` module. It's small, encode-only, +and predates this project. If you're already using it, this is the +side-by-side guide to switching. + +The relevant differences: + +| | Built-in `protobuf` | `pb` (this project) | +|---|---|---| +| Module name | `require('protobuf')` | `require('pb')` (different name so both can coexist) | +| Schema source | Inline Lua tables (`protobuf.message{...}`) | `.proto` files + `protoc` plugin (or runtime `pb.parse`) | +| Encode | ✅ | ✅ | +| Decode | ❌ | ✅ | +| `map` | ❌ | ✅ | +| `oneof` | ❌ | ✅ | +| `repeated` | ✅ | ✅ | +| WKT (Timestamp, Any, Struct, …) | ❌ | ✅ | +| `optional` (proto3) | ❌ | ✅ with `has_*`/`clear_*` | +| Services (gRPC stubs) | ❌ | ✅ | +| JSON / text format | ❌ | ✅ | +| 64-bit ints as cdata | ✅ | ✅ (same convention) | +| Conformance with mainline `protoc` | partial | proto3-complete (Google's conformance suite, 0 failures) | + +Both can run in the same process — `pb` is named differently +specifically so they don't collide. + +## Translating schemas + +The built-in describes messages as Lua tables. Translate them to `.proto` +files for the codegen path, or to inline `pb.parse(...)` calls for the +runtime path. + +### Built-in + +```lua +local p = require('protobuf') +local proto = p.protocol{ + p.message('User', { + name = {'string', 1}, + age = {'int32', 2}, + emails = {'string', 3, 'repeated'}, + }), + p.enum('Role', { + USER = 0, + ADMIN = 1, + }), +} + +local bytes = proto:encode('User', {name = 'Alice', age = 30, emails = {'a@x'}}) +``` + +### `pb` (codegen path) + +`user.proto`: + +```proto +syntax = "proto3"; +package app; + +enum Role { + USER = 0; + ADMIN = 1; +} + +message User { + string name = 1; + int32 age = 2; + repeated string emails = 3; +} +``` + +`protoc --tarantool_out=./gen user.proto` then: + +```lua +local user = require('app.user_pb') +local bytes = user.User_encode({name = 'Alice', age = 30, emails = {'a@x'}}) +local back = user.User_decode(bytes) -- new capability! +``` + +### `pb` (no-codegen path) + +If you want to keep schemas inline in Lua (matches the built-in's +ergonomics), use `pb.parse`: + +```lua +local pb = require('pb') +local user = pb.parse([[ +syntax = "proto3"; +package app; +message User { + string name = 1; + int32 age = 2; + repeated string emails = 3; +} +]]) + +local bytes = user.User_encode({name = 'Alice', age = 30}) +local back = user.User_decode(bytes) +``` + +See [how-to: dynamic schemas](08-dynamic-schemas.md) for the runtime- +parse approach. + +## API call-site changes + +| Built-in | `pb` | +|---|---| +| `proto:encode('User', t)` | `user.User_encode(t)` | +| (no decode) | `user.User_decode(b)` | +| `proto:encode(msg_name, ...)` returns `string` | `User_encode(t)` returns `string` (same) | +| Lua tables: 1-based arrays for repeated, hash for map (encode only) | Same shape — repeated stays 1-based arrays, maps stay hash tables | +| 64-bit fields: `int64_t` / `uint64_t` cdata | Same | +| `pcall(proto.encode, proto, ...)` for errors | Same: `pcall(user.User_encode, t)` | + +The data shape is the same — translating happens at the schema +declaration, not at the encode call sites. A direct find-replace from +`proto:encode('User', t)` to `user.User_encode(t)` covers most of the +migration. + +## Behavioral differences + +### Defaults + +Proto3 elides default values on encode (`age = 0` is omitted from the +wire, unless the field is explicit-`optional`). The built-in followed +the same proto3 convention — no change. + +### Packed repeated + +Both encode `repeated int32` etc. as packed by default. No change. + +### Unknown fields + +The built-in didn't decode, so this didn't come up. `pb` preserves +unknown fields by default — they decode into `t._unknown_fields` (raw +bytes) and re-emit on encode. If your producer was encoding extra +fields the consumer didn't know about, those now round-trip cleanly +through `pb`. + +### Errors + +Built-in raised on encode of unknown fields (typo in the table key +silently encoded as nothing? or raised? — version-dependent). `pb` +ignores unknown table keys on encode but raises on shape mismatches +(wrong Lua type for the declared proto type, etc.). + +If you relied on encode-side strictness for typo detection, route +field-name accesses through the strict `M.User_fields` table (only +emitted for the lazy view today; eager encoders trust the input +shape). + +## Things you can do now that you couldn't before + +- **Decode.** `User_decode(bytes)` round-trips any input the + built-in produced (and any input mainline `protoc` produced). +- **Maps and oneofs.** Both round-trip. +- **WKT.** `google.protobuf.Timestamp` ↔ Tarantool `datetime`; + `Duration`, wrappers, `Struct`, `Any`, `FieldMask`, `Empty`. +- **Services.** Generated `*_client` / `*_server` factories with a + pluggable transport contract. +- **JSON.** `pb.json.encode` / `pb.json.decode` against the same + descriptor. +- **Text format.** `User_text(t)` for debug printing. +- **Lazy view.** `User_decode_lazy(b)` for proxy/router shapes that + touch a few fields and re-encode. +- **Runtime schemas.** `pb.parse` / `pb.from_pb` for descriptors + built at runtime. + +## What's next + +- [How-to: first message](01-first-message.md) — the green-field + path if you'd rather start fresh. +- [Reference: generated API](../reference/generated-api.md) — + what `User_pb.lua` exposes. +- [How-to: dynamic schemas](08-dynamic-schemas.md) — for the + inline-schema style the built-in encouraged. diff --git a/docs/howto/12-build-integration.md b/docs/howto/12-build-integration.md new file mode 100644 index 0000000000000000000000000000000000000000..c520ce5028772b522e097c5f7acc2d996c52c994 --- /dev/null +++ b/docs/howto/12-build-integration.md @@ -0,0 +1,212 @@ +# How-to: build integration + +Driving `protoc-gen-tarantool` from build systems. The plugin is a +standard `protoc` plugin — anything that invokes `protoc` can invoke +it. This repo's canonical wrapper is the **Justfile**; recipes for +other build tools below. + +For the canonical `protoc` flags, see +[reference/cli.md](../reference/cli.md). + +## `tarantool-protobuf`'s own build (Justfile) + +```bash +just build # build the codegen plugin +just gen # regenerate examples/expected/{full,runtime}/... +just gen-docs # regenerate examples/docs/hello.md +just test # luatest suite +just clean # remove plugin + examples/expected/ +``` + +See `just --list` for the full set. + +## Plain `protoc` + +The minimum: + +```bash +protoc \ + -I. \ + -Ioptions \ + --tarantool_out=./gen \ + path/to/foo.proto path/to/bar.proto +``` + +Assumes `protoc-gen-tarantool` is on `PATH`. If not: + +```bash +protoc \ + --plugin=protoc-gen-tarantool=./bin/protoc-gen-tarantool \ + --tarantool_out=./gen \ + ... +``` + +Pass options via `--tarantool_opt=key=value,key=value`: + +```bash +protoc --tarantool_out=./gen --tarantool_opt=mode=runtime,prefix=vendor ... +``` + +## Makefile (legacy / external projects) + +Example pattern for a downstream project that still uses Make: + +```make +PROTOC ?= protoc +GEN_DIR := gen +PROTO_FILES := $(wildcard proto/**/*.proto) + +.PHONY: gen +gen: protoc-gen-tarantool + $(PROTOC) \ + -I. \ + -Ioptions \ + --tarantool_out=$(GEN_DIR) \ + --tarantool_opt=mode=full \ + $(PROTO_FILES) + +protoc-gen-tarantool: + go build -o $@ ./cmd/protoc-gen-tarantool + +.PHONY: clean +clean: + rm -rf $(GEN_DIR) +``` + +For incremental builds, gate per-file: + +```make +$(GEN_DIR)/%/foo_pb.lua: proto/%/foo.proto protoc-gen-tarantool + $(PROTOC) -I. -Ioptions --tarantool_out=$(GEN_DIR) $< +``` + +## Justfile + +`tarantool-protobuf` itself ships a `Justfile` as the canonical entry +point (`just build`, `just gen`, `just test`, `just bench`, +`just conformance`, `just examples`, …). A downstream user-project +pattern: + +```just +default: + @just --list + +build-plugin: + go build -o ./bin/protoc-gen-tarantool ./vendor/tarantool-protobuf/cmd/protoc-gen-tarantool + +gen: build-plugin + PATH=./bin:$PATH protoc \ + -I. -Ivendor/tarantool-protobuf/options \ + --tarantool_out=./gen \ + proto/**/*.proto + +clean: + rm -rf gen +``` + +## `buf` + +`buf generate` reads a `buf.gen.yaml`: + +```yaml +version: v1 +plugins: + - plugin: tarantool + out: gen + opt: + - mode=full + path: ./bin/protoc-gen-tarantool +``` + +Then `buf generate`. `buf` discovers protos via `buf.yaml` (or +`buf.work.yaml` for monorepos) and runs the plugin for each. + +The plugin doesn't depend on `buf` features beyond what every `protoc` +plugin sees, so `buf generate` is a drop-in for `protoc` if your team +prefers it. + +## CMake + +```cmake +find_package(Protobuf REQUIRED) +find_program(PROTOC_GEN_TARANTOOL protoc-gen-tarantool REQUIRED) + +set(PROTO_FILES proto/foo.proto proto/bar.proto) +set(GEN_DIR ${CMAKE_BINARY_DIR}/gen) +file(MAKE_DIRECTORY ${GEN_DIR}) + +add_custom_command( + OUTPUT ${GEN_DIR}/.stamp + COMMAND ${Protobuf_PROTOC_EXECUTABLE} + --plugin=protoc-gen-tarantool=${PROTOC_GEN_TARANTOOL} + -I${CMAKE_SOURCE_DIR} + -I${CMAKE_SOURCE_DIR}/vendor/tarantool-protobuf/options + --tarantool_out=${GEN_DIR} + ${PROTO_FILES} + COMMAND ${CMAKE_COMMAND} -E touch ${GEN_DIR}/.stamp + DEPENDS ${PROTO_FILES} ${PROTOC_GEN_TARANTOOL} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} +) + +add_custom_target(protos ALL DEPENDS ${GEN_DIR}/.stamp) +``` + +## Generating once, distributing as a binary blob + +For deployments where `protoc` shouldn't run on every Tarantool host, +generate at build time and ship the `.lua` files (or a +`FileDescriptorSet` binary). + +```bash +# At build time, ship pre-generated Lua: +protoc --tarantool_out=./dist/lua ... +tar czf myapp-protos.tar.gz dist/lua + +# Or, ship a binary descriptor set for runtime ingestion: +protoc --descriptor_set_out=./dist/schemas.pb \ + --include_imports \ + proto/**/*.proto +# At Tarantool startup: +local pb = require('pb') +local set = pb.from_pb(io.open('dist/schemas.pb', 'rb'):read('*a')) +``` + +The descriptor-set approach is what makes +[dynamic schemas](08-dynamic-schemas.md) possible without running +`protoc` in production. + +## Vendoring the plugin source + +If you'd rather not depend on a published binary: + +1. Vendor this repo (or just the `cmd/protoc-gen-tarantool` + + `options/` + `runtime/pb/` directories) into your project. +2. Build the plugin as part of your top-level build (`go build` step). +3. Drop the resulting binary on `PATH` for `protoc` to find it. + +The plugin's only runtime dependency is the `runtime/pb/` Lua module +set — that needs to land on `LUA_PATH` in production. See +[how-to: module layout → wiring LUA_PATH](02-module-layout.md#wiring-lua_path). + +## CI: regenerate-and-diff + +A cheap CI guard: regenerate Lua from `.proto` and fail the build if +the working tree changed. + +```yaml +# .github/workflows / sourcecraft pipeline +- run: just gen +- run: git diff --exit-code +``` + +This catches the "proto changed but generated Lua wasn't updated" +class of bug, which `protoc` itself won't notice. + +## What's next + +- [Reference: CLI](../reference/cli.md) — every flag, file option, + path-resolution rule. +- [How-to: dynamic schemas](08-dynamic-schemas.md) — the + no-codegen path. +- [How-to: module layout](02-module-layout.md) — what `prefix`, + `lua_package`, and `LUA_PATH` do. diff --git a/docs/howto/13-custom-transport.md b/docs/howto/13-custom-transport.md new file mode 100644 index 0000000000000000000000000000000000000000..d6e10c87b1b306f8af2cdd2653cdc88aa738331f --- /dev/null +++ b/docs/howto/13-custom-transport.md @@ -0,0 +1,207 @@ +# How-to: writing a custom transport + +A "transport" is any Lua table implementing the four-method contract +documented in +[reference/grpc-contract.md](../reference/grpc-contract.md): +`:unary`, `:server_stream`, `:client_stream`, `:bidi`. Generated +`M._client(transport)` accepts any object matching that +shape — wire protocol, framing, and connection management are +transport-private concerns. + +The shipped reference transports (`pb.grpc.loopback`, +`pb.grpc.multiplex`) handle in-process routing. For anything across a +process boundary, you write a transport. + +This how-to walks through three escalating cases. + +## Case 1: unary-only HTTP/1.1 client + +Outbound calls to a service that speaks Connect-JSON (or HTTP/1.1 +gRPC-Gateway) over a single POST endpoint. Unary is enough for many +use cases. + +```lua +local function http_unary_transport(base_url) + local http = require('http.client').new() + return { + unary = function(_, path, req_bytes, ctx) + local r = http:request('POST', base_url .. path, req_bytes, { + headers = {['content-type'] = 'application/proto'}, + timeout = ctx and ctx.deadline, + }) + if r.status ~= 200 then + error('grpc: HTTP ' .. r.status, 0) + end + return r.body + end, + -- Streaming methods explicitly error rather than silently + -- returning nil — caller learns at setup time, not on first + -- failed message. + server_stream = function() error('streaming not supported', 0) end, + client_stream = function() error('streaming not supported', 0) end, + bidi = function() error('streaming not supported', 0) end, + } +end + +-- Plug into a generated client: +local client = hello.Greeter_client(http_unary_transport('http://api.example.com')) +client.SayHello({name = 'Alice'}, {}) +``` + +The bytes the generated code passes you (`req_bytes`) are already +encoded; you put them on the wire as-is. Same on the way back — +`r.body` is the encoded reply that the generated client will decode. + +## Case 2: net.box tunnel (in-cluster) + +For Tarantool↔Tarantool calls, the simplest pattern is a stored +function that dispatches into a `M._server` and a client +that calls it via `net.box`. The runnable stub: +`examples/grpc/transport_netbox_stub.lua`. + +Server side (one stored function, all services): + +```lua +local function register_server(server, func_name) + func_name = func_name or 'grpc_dispatch' + rawset(_G, func_name, function(path, req_bytes) + local handler = server.methods[path] + if handler == nil then + return {false, 'unknown method: ' .. path} + end + local ok, resp = pcall(handler, req_bytes, {}) + if not ok then return {false, tostring(resp)} end + return {true, resp} + end) +end +``` + +Client side: + +```lua +local function netbox_client(conn, func_name) + func_name = func_name or 'grpc_dispatch' + return { + unary = function(_, path, req_bytes, _ctx) + local r = conn:call(func_name, {path, req_bytes}) + if not r[1] then error('grpc: ' .. tostring(r[2]), 0) end + return r[2] + end, + server_stream = function() error('streaming not supported in stub', 0) end, + client_stream = function() error('streaming not supported in stub', 0) end, + bidi = function() error('streaming not supported in stub', 0) end, + } +end + +local conn = require('net.box').connect('user:pass@localhost:3301') +local client = hello.Greeter_client(netbox_client(conn)) +``` + +This stub punts on streaming. A production net.box transport would +use `box.session.push` for server-stream messages and a stateful +session for client-stream / bidi. + +## Case 3: full streaming over a paired channel + +When you need all four streaming flavors, the pattern is: + +1. Drive a paired client/server stream over your wire (channels, + sockets, HTTP/2 streams — whatever you have). +2. Implement the client-view stream interface + (`:send` / `:close_send` / `:recv` / `:cancel`). +3. Return that stream object from `:server_stream` / `:client_stream` + / `:bidi`. + +`pb.grpc.new_stream_pair(buf_size)` returns paired client/server +views over two `fiber.channel`s. Use it as the in-process half of a +network transport — the fiber that drives the channels writes/reads +from your actual wire. + +Sketch of the I/O fiber pattern: + +```lua +local fiber = require('fiber') + +local function bidi_call(socket, path, ctx) + local client_view, server_view, state = pb.grpc.new_stream_pair() + + -- Reader fiber: bytes from the wire -> server_view (which is the + -- client_view's counterpart for inbound messages). + fiber.create(function() + while true do + local frame, err = socket:read_frame() + if frame == nil then + if err then state.server_err = err end + client_view:close_send() -- closes the reader path + return + end + server_view:send(frame.payload) -- delivered to client_view:recv() + end + end) + + -- Writer fiber: outbound from client_view -> wire. + fiber.create(function() + while true do + local bytes, err = server_view:recv() -- pulls what client sent + if bytes == nil then return end + socket:write_frame({path = path, payload = bytes}) + end + end) + + return client_view +end +``` + +The fiber.channel buffers decouple send/recv timing on each side. +Senders block when the buffer is full; receivers block when it's +empty. Default size is 16 messages — override per call via +`new_stream_pair(buf_size)`. + +## Conventions every transport should honor + +- **Path format.** `/./`. Don't strip the + leading slash — generated code emits it. +- **Error propagation.** Errors flow as `error(...)` calls. Don't + swallow them; transports may wrap with their own message prefix + (`'grpc: ' .. err`) but the generated client expects to see the + error. +- **`ctx` keys.** Honor `ctx.deadline` (cancel on overrun), + `ctx.headers` (transport-specific encoding), `ctx.trace_id` / + `ctx.span_id` (inject as the wire's tracing primitive). See + [grpc-contract.md → context](../reference/grpc-contract.md#context-ctx). +- **Empty `ctx`.** Generated clients pass `ctx = {}` if the caller + didn't supply one. Don't assume keys exist; use `ctx and ctx.foo`. +- **Cancel semantics.** `stream:cancel()` should release resources + promptly. Tolerate it being called twice (e.g. after the stream + already closed). + +## Testing your transport against the loopback + +A useful pattern for transport development is to run the same test +suite against both `pb.grpc.loopback` and your transport. Loopback +gives you the reference behavior; if a test passes against loopback +but fails against yours, the bug is in the transport. + +```lua +local function test_against(transport_factory) + return function() + local server = hello.Greeter_server(impl) + local transport = transport_factory(server) + local client = hello.Greeter_client(transport) + t.assert_equals(client.SayHello({name='X'}, {}).greeting, 'Hi X') + -- ... full suite ... + end +end + +t.group('greeter.loopback').test = test_against(pb.grpc.loopback) +t.group('greeter.mytransport').test = test_against(make_my_transport) +``` + +## What's next + +- [Reference: grpc-contract](../reference/grpc-contract.md) — + contract, stream objects, helper wrappers. +- [Specs: gRPC transports](../specs/grpc_transports.md) — the + protocol matrix and the recommended external transports + (Connect-JSON server, net.box tunnel, HTTP/2 client) we haven't + built yet. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000000000000000000000000000000000000..ae563700127f397acbf71c699d5994b7cd9275e0 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,88 @@ +# Documentation + +The user-facing docs for `tarantool-protobuf`, grouped by what you're +trying to do. The [README](../README.md) has the feature matrix, quick +start, and status numbers; this page is the map for everything else. + +## Getting started + +If you're new to the project, work through the how-tos in order; each +links to the next. + +1. **[Your first message, end-to-end](howto/01-first-message.md)** — + write a proto, run the plugin, encode/decode in Tarantool. +2. **[Module layout](howto/02-module-layout.md)** — `prefix`, + `lua_package`, `LUA_PATH`, three side-by-side `protoc` invocations. +3. **[gRPC with the loopback transport](howto/03-grpc-loopback.md)** — + Greeter end-to-end with all four streaming flavors. +4. **[WKT: Struct / Value / ListValue](howto/04-wkt-struct-value.md)** — + round-tripping JSON-shaped data through proto. +5. **[Packing and unpacking `Any`](howto/05-wkt-any.md)** — typed + message payloads, registry, opaque fallback. +6. **[JSON over `tarantool/http`](howto/06-json-http.md)** — exposing + a proto-defined API as JSON HTTP. +7. **[Text format for debugging](howto/07-text-format.md)** — + mainline-protoc-compatible printer and parser. +8. **[Dynamic schemas from a Tarantool space](howto/08-dynamic-schemas.md)** — + `pb.parse` and `pb.from_pb` for runtime descriptors. +9. **[When to use `decode_lazy`](howto/09-lazy-when.md)** — + picking eager vs lazy by workload shape. +10. **[Migrating from the built-in `protobuf`](howto/11-migrate-from-builtin.md)** — + side-by-side schema and call-site diffs. +11. **[Build integration](howto/12-build-integration.md)** — `protoc`, + Makefile, Justfile, `buf`, CMake. +12. **[Writing a custom transport](howto/13-custom-transport.md)** — + implementing the four-method contract. + +When something doesn't work, **[troubleshooting](troubleshooting.md)** +collects the common errors and their fixes. + +## Reference + +- **[reference/runtime-api.md](reference/runtime-api.md)** — every + export of `require('pb')`: codec, lazy view, dynamic descriptors, + JSON/text/WKT/gRPC, sentinels, codegen helpers. +- **[reference/generated-api.md](reference/generated-api.md)** — + what each `_pb.lua` exposes per message, enum, and service. +- **[reference/cli.md](reference/cli.md)** — driving + `protoc-gen-tarantool` and `protoc-gen-tarantool-doc` from + `protoc`. Flags, file options, path-resolution rules. +- **[reference/grpc-contract.md](reference/grpc-contract.md)** — + the four-method transport interface, stream-object shapes, and + the shipped `loopback` / `multiplex` transports. +- **[api-modes.md](api-modes.md)** — full / runtime / lazy. Same + descriptor, three call shapes. When to pick which, with measured + trade-offs and the field-name-constants contract for lazy views. +- **[codegen.md](codegen.md)** — how `protoc-gen-tarantool` works: + the pipeline, the descriptor-table contract (canonical shape), + inline-mode vs runtime-mode emission, the LuaJIT hot-path rules + generated code observes, and how to add a new scalar type. +- **Auto-generated proto reference** — `examples/docs/hello.md` is + the output of the sibling `protoc-gen-tarantool-doc` plugin + against `examples/proto/hello.proto`. Same plugin can run against + any `.proto` to produce its own per-file Markdown. + +## Specs + +Forward-looking design docs. These describe contracts and trade-offs +for work that's either partly shipped or planned. + +- **[specs/grpc_transports.md](specs/grpc_transports.md)** — the + transport contract (shipped, stable), the protocol matrix + (HTTP/2 gRPC vs Connect vs net.box tunnel vs IProto), recommended + transports to build, and the gRPC status-code mapping. The + loopback and multiplex transports already ship in + `runtime/pb/grpc.lua`; external transports (`http_server`, + `netbox`, `http_client`) are not yet built. +- **[specs/msgpack_encoding.md](specs/msgpack_encoding.md)** — + brainstorm for a sibling MsgPack codec over the same descriptors. + Not implemented; design sketch for picking up later. + +## Internals + +- **[../PLAN.md](../PLAN.md)** — phased roadmap (M1–M8) with + per-milestone status and per-feature design notes. +- **[../CLAUDE.md](../CLAUDE.md)** — invariants and conventions + enforced across the codebase: no `pairs()` on hot paths, 64-bit + ints as cdata, SoA over AoS for large index structures, keep hot + helpers small, WKT routing. diff --git a/docs/reference/cli.md b/docs/reference/cli.md new file mode 100644 index 0000000000000000000000000000000000000000..02bc043a96c794d846a4a29adeb1e1a497a2146e --- /dev/null +++ b/docs/reference/cli.md @@ -0,0 +1,181 @@ +# CLI reference + +How to drive `protoc-gen-tarantool` and the sibling +`protoc-gen-tarantool-doc` from `protoc`. + +## Building the plugins + +```bash +just build # builds ./protoc-gen-tarantool +just build-doc # builds ./protoc-gen-tarantool-doc +``` + +Or directly: + +```bash +go build -o protoc-gen-tarantool ./cmd/protoc-gen-tarantool +go build -o protoc-gen-tarantool-doc ./cmd/protoc-gen-tarantool-doc +``` + +Place both binaries on `PATH` (or pass `--plugin=` to `protoc`) and +they become available as the `tarantool` and `tarantool-doc` outputs. + +## `protoc-gen-tarantool` — the Lua codegen + +### Invocation + +```bash +protoc \ + -I. \ + -Ioptions \ + --tarantool_out= \ + --tarantool_opt==,= \ + path/to/file.proto +``` + +`output_dir` is the root the plugin writes under. Per-file output path +follows the Lua require path (see [`prefix`](#prefix) and +[`(tarantool.lua_package)`](#tarantoollua_package) below). + +`-Ioptions` is needed when any input proto uses the custom +`(tarantool.lua_package)` option — it imports +`tarantool/tarantool.proto` from `options/tarantool/`. + +### `--tarantool_opt` + +Comma-separated `key=value` pairs: + +| Option | Values | Default | Meaning | +|---|---|---|---| +| `mode` | `full` / `runtime` | `full` | `full` inlines `_encode` / `_decode` bodies; `runtime` emits one-line delegations to `pb.encode` / `pb.decode`. See [api-modes.md](../api-modes.md). | +| `prefix` | any Lua require path | empty | Prepended to every generated module's require path and on-disk subpath. | + +### `prefix` + +`prefix=foo.bar` rewrites every generated module name from +`` to `foo.bar.`. Affects both the require path +and the on-disk location. + +| Without prefix | With `prefix=apps.myapp` | +|---|---| +| `hello/hello_pb.lua` | `apps/myapp/hello/hello_pb.lua` | +| `require('hello.hello_pb')` | `require('apps.myapp.hello.hello_pb')` | + +The Justfile uses this to emit `full/` and `runtime/` copies side by +side for the parametrized test suite: + +```bash +protoc --tarantool_opt=mode=full,prefix=full ... # examples/expected/full/... +protoc --tarantool_opt=mode=runtime,prefix=runtime ... # examples/expected/runtime/... +``` + +### `(tarantool.lua_package)` — per-file override + +Defined in `options/tarantool/tarantool.proto`. Overrides the require +path for a single `.proto`: + +```proto +syntax = "proto3"; +package my.app; + +import "tarantool/tarantool.proto"; +option (tarantool.lua_package) = "myapp.proto.foo"; + +message Foo { ... } +``` + +| Without `lua_package` | With `lua_package` | +|---|---| +| `my/app/foo_pb.lua` | `myapp/proto/foo_pb.lua` | +| `require('my.app.foo_pb')` | `require('myapp.proto.foo_pb')` | + +`prefix=` and `(tarantool.lua_package)` compose: `prefix` is prepended +to the final per-file path regardless of which scheme produced it. + +### Path-resolution rules + +For each input `.proto`, the plugin picks a Lua require path in this +order: + +1. **`option (tarantool.lua_package) = "pkg";`** — wins outright. The + plugin writes `pkg` as a `/`-joined path with `_pb.lua` appended + (or just `pkg` if `pkg` already ends in `_pb`). +2. **`package` declaration + filename** — `package my.app;` and a + file `foo.proto` produce `my/app/foo_pb.lua`, + `require('my.app.foo_pb')`. +3. **No package** — the filename alone: `foo.proto` → `foo_pb.lua`, + `require('foo_pb')`. + +WKT proto files (`google/protobuf/*.proto`) are **not** generated as +Lua modules. References to WKT types route to `pb.wkt._descriptor` +at codegen time; the runtime side ships them in `runtime/pb/wkt.lua`. + +### What gets emitted + +See [generated-api.md](generated-api.md) for the per-message / +per-enum / per-service surface. Both modes always emit: + +- `M._descriptor`, `M._fields`, `M._oneofs` (when applicable) +- `M._new`, `M._encode`, `M._decode` +- `M._decode_lazy`, `M._text` (mode-independent wrappers) +- `M._has_` / `M._clear_` (proto3 explicit-optional) +- `M._descriptor`, `M.` (alias for `by_name`) +- `M._service`, `M._client`, `M._server` +- EmmyLua / lua-language-server annotations +- Reserved-name table (`reserved_names`) for the text-format parser + +### Stdin/stdout protocol + +The plugin reads a `CodeGeneratorRequest` from stdin and writes a +`CodeGeneratorResponse` to stdout, per protoc's standard plugin +interface. It advertises `FEATURE_PROTO3_OPTIONAL` so protoc surfaces +explicit-optional fields; without this flag, protoc omits them. + +**Proto2 input is rejected** at the top of `GenerateFile`. See +[codegen.md → proto2 deferral](../codegen.md#proto2-deferral). + +## `protoc-gen-tarantool-doc` — the Markdown reference generator + +Same input as the Lua codegen; emits one Markdown file per input +`.proto` summarizing messages, enums, services. + +```bash +protoc \ + -I. \ + -Ioptions \ + --tarantool-doc_out= \ + path/to/file.proto +``` + +Example: `examples/docs/hello.md` is produced by `just gen-docs` from +`examples/proto/hello.proto`. + +No options today. The output template is minimal: + +- Per message: a table of fields (number, name, type, label, + description from the leading comment). +- Per enum: a table of values. +- Per service: a table of methods with streaming kind. + +Lua call signatures (`Foo_encode`, `Foo_decode_lazy`, etc.) are not +yet in the doc template — they're documented in +[generated-api.md](generated-api.md) for now. + +## Driving from `make` + +The Justfile is the canonical entry point for development: + +```bash +just build # build the codegen plugin +just build-doc # build the doc plugin +just gen # build + regen examples/expected/{full,runtime}/... +just gen-docs # build-doc + regen examples/docs/hello.md +just test # run the luatest suite (parametrized over both modes) +just goldens # regenerate test/interop/fixtures/*.bin via protoc --encode +just bench # alloc + throughput per op +just jit-trace # assert hot paths stay on the JIT trace +``` + +For end-user projects not using this repo's Makefile, see the planned +[howto 12: Build integration](../howto/12-build-integration.md) +(`protoc`, Justfile, `buf`, CMake recipes). diff --git a/docs/reference/generated-api.md b/docs/reference/generated-api.md new file mode 100644 index 0000000000000000000000000000000000000000..eb3c681048499fc237724586f25d52535a90d5fb --- /dev/null +++ b/docs/reference/generated-api.md @@ -0,0 +1,253 @@ +# Generated module API + +What `protoc-gen-tarantool` emits per `.proto` file. The plugin produces +one `_pb.lua` module per input proto; this page is the surface +that module exposes. + +For the *runtime* API (`require('pb')`) see +[runtime-api.md](runtime-api.md). For where each piece comes from see +[../codegen.md](../codegen.md). + +## Module shape + +A file `pkg/foo.proto` with `package pkg;` is generated as +`pkg/foo_pb.lua` and required as `pkg.foo_pb`. Prefix / module-path +overrides are documented in [cli.md](cli.md). + +```lua +local M = require('pkg.foo_pb') +return M +``` + +Every symbol below lives under that returned table. + +## Per-message symbols + +For each message `Foo` in the proto file: + +| Symbol | Type | Notes | +|---|---|---| +| `M.Foo_descriptor` | table | The descriptor consumed by `pb.encode` / `pb.decode` / `pb.decode_lazy`. Shape: [codegen.md → descriptor table](../codegen.md#the-descriptor-table--the-contract). | +| `M.Foo_new(t)` | `t or {}` | Placeholder constructor; currently just `t or {}`. Reserved for future validation/defaulting. | +| `M.Foo_encode(t)` | `(table) -> string` | Table → wire bytes. In full mode the body is inlined; in runtime mode it delegates to `pb.encode(M.Foo_descriptor, t)`. | +| `M.Foo_decode(b)` | `(string) -> table` | Wire bytes → table. Same inline/delegate split as encode. Unknown fields land under `t._unknown_fields`. | +| `M.Foo_decode_lazy(b)` | `(string) -> MessageView` | Zero-copy view. See [api-modes.md → lazy](../api-modes.md#lazy-zero-copy-view). | +| `M.Foo_text(t [, opts])` | `(table [, opts]) -> string` | mainline-protoc text format. `opts.single_line = true` for one-line output, `opts.indent = ''` to override default `' '`. | +| `M.Foo_fields` | strict table | `{field_name = "field_name", ...}` for typo-safe lazy-view access: `view:get(M.Foo_fields.user_id)`. Unknown keys error. | +| `M.Foo_oneofs` | strict table | `{group_name = "group_name", ...}`, only emitted when the message has any oneofs. Used with `view:which(M.Foo_oneofs.outcome)`. | +| `M.Foo_has_(t)` | `(table) -> bool` | Only emitted for proto3 explicit-`optional` fields. Distinguishes "absent" from "set to default". | +| `M.Foo_clear_(t)` | `(table) -> nil` | Same; unsets the field. | + +**Text-format decode** has no per-message wrapper — call +`pb.text.decode(M.Foo_descriptor, text)` directly. It's used in a few +spots (conformance runner, debug tools) and didn't warrant codegen +surface. + +**JSON encode/decode** likewise has no per-message wrapper — call +`pb.json.encode(M.Foo_descriptor, t)` and +`pb.json.decode(M.Foo_descriptor, s [, opts])`. + +### Field types in the table + +| Proto type | Lua type | +|---|---| +| `string` | `string` | +| `bytes` | `string` | +| `bool` | `boolean` | +| `int32`, `sint32`, `sfixed32`, `uint32`, `fixed32`, `enum` | Lua `number` (integer) | +| `int64`, `sint64`, `sfixed64` | `int64_t` cdata | +| `uint64`, `fixed64` | `uint64_t` cdata | +| `float`, `double` | Lua `number` (float) | +| `repeated T` | 1-based contiguous Lua array | +| `map` | Lua table keyed by `K` (cdata keys are dedup'd via pointer-vs-value comparison on decode) | +| nested message | nested Lua table (recursive) | +| `oneof` member | only the active member is present in the table | + +**Absence vs default:** proto3 implicit fields decode as their default +(`0` / `""` / `false` / `{}`); explicit-`optional` fields stay `nil` +when absent. Use the `_has_` helper to disambiguate. + +**`box.NULL`-equivalent:** `pb.NULL` is the canonical null sentinel +for `google.protobuf.Value` null and JSON null. Prefer it over +`box.NULL` so code doesn't depend on `box` being loaded. + +### Unknown fields + +Decoded messages with fields the schema doesn't recognize keep the +raw bytes in `t._unknown_fields` (a single string, in encounter +order). Re-encoding splices them back at the tail. WKT messages and +map entries skip this — they have custom `desc.encode/decode`. + +## Per-enum symbols + +For each enum `Color`: + +| Symbol | Type | Notes | +|---|---|---| +| `M.Color_descriptor` | `{name, by_name, by_value}` | `by_name.RED = 0`, `by_value[0] = 'RED'`. | +| `M.Color` | alias | Shorthand for `M.Color_descriptor.by_name`. Use as `M.Color.RED`. | + +Generated `_pb.lua` reserves only `Color_descriptor` and the alias. +There are no `Color_encode` / `Color_decode` wrappers — enums live +inside other messages, not on the wire by themselves. + +Unknown enum values are passed through as their numeric form. Lua +table fields hold the integer; enum-aware printing/JSON converts back +to the name when the value is known. + +## Per-service symbols + +For each `service Greeter` in the proto file: + +```lua +M.Greeter_service -- descriptor (name, methods, paths) +M.Greeter_client(transport) -- factory: returns {Method = fn(req, ctx)} +M.Greeter_server(impl) -- factory: returns {service, methods} +``` + +### `M._service` + +```lua +{ + name = "hello.Greeter", + full_name = "/hello.Greeter", + methods = { + [] = { + name = "", + full_name = "/hello.Greeter/", + input = M._descriptor, + output = M._descriptor, + client_streaming = , + server_streaming = , + }, + ... + }, +} +``` + +The four streaming flavors fall out of those two booleans: + +| Streaming flavor | `client_streaming` | `server_streaming` | +|---|---|---| +| Unary | absent | absent | +| Server-stream | absent | `true` | +| Client-stream | `true` | absent | +| Bidi | `true` | `true` | + +### `M._client(transport) -> {Method = fn(...)}` + +Pass any object implementing the four-method transport contract from +[grpc-contract.md](grpc-contract.md). The returned table has one +entry per RPC, shaped per its streaming flavor: + +```lua +local client = M.Greeter_client(pb.grpc.loopback(server)) + +-- Unary: encode req, call transport:unary, decode resp. +local reply = client.SayHello({name = 'Alice'}, ctx) + +-- Server-stream: returns a stream; iterate with :recv()/:close(). +local stream = client.StreamHellos({name = 'Alice'}, ctx) +for msg in function() return stream:recv() end do ... end + +-- Client-stream: returns a call; :send(req) / :close_send() / :recv() once. +local call = client.CollectHellos(ctx) +call:send({name = 'Alice'}); call:send({name = 'Bob'}) +call:close_send() +local reply = call:recv() + +-- Bidi: returns a call; :send and :recv interleave freely. +local call = client.Chat(ctx) +fiber.create(function() for msg in ... do call:send(msg) end; call:close_send() end) +for reply in function() return call:recv() end do ... end +``` + +`ctx` is whatever opaque table the transport understands (deadlines, +metadata, etc.). The contract reserves `ctx.deadline`, `ctx.headers`, +`ctx.trace_id`, `ctx.span_id`, `ctx.options`; see +[grpc-contract.md](grpc-contract.md). + +### `M._server(impl) -> {service, methods}` + +`impl` is a table with one Lua function per RPC. Signatures depend on +the streaming flavor: + +```lua +local server = M.Greeter_server({ + -- Unary + SayHello = function(req, ctx) return {greeting = 'Hi ' .. req.name} end, + + -- Server-stream: receive req + a stream to push replies onto + StreamHellos = function(req, stream, ctx) + for i = 1, 3 do stream:send({greeting = 'Hi #' .. i}) end + stream:close() + end, + + -- Client-stream: receive a stream + ctx; collect requests, return one reply + CollectHellos = function(stream, ctx) + local names = {} + for req in function() return stream:recv() end do + names[#names + 1] = req.name + end + return {greeting = 'Hi ' .. table.concat(names, ', ')} + end, + + -- Bidi: receive a stream; send/recv interleave + Chat = function(stream, ctx) + for req in function() return stream:recv() end do + stream:send({greeting = 'Echo ' .. req.name}) + end + stream:close() + end, +}) +``` + +The returned `{service, methods}` plugs into any transport: + +```lua +local transport = pb.grpc.loopback(server) -- in-process +local transport = pb.grpc.multiplex({server, other_server}) +``` + +## File-level boilerplate + +Every generated module starts with: + +```lua +-- Code generated by protoc-gen-tarantool. DO NOT EDIT. +-- source: .proto +-- syntax: proto3 + +local pb = require('pb') +local wire = pb.wire + +local M = {} +-- ... enums, descriptors, fields tables, encoders/decoders ... +return M +``` + +EmmyLua / lua-language-server annotations are emitted alongside each +descriptor: + +```lua +---@class hello.Person +---@field name string +---@field user_id integer +-- ... +``` + +Class identifiers use the full proto name (`hello.Person`, +`hello.Address`) so cross-file references resolve in editors. Lazy +view types (`pb.MessageView`, `pb.ArrayView`, `pb.MapView`) are +declared inline in `runtime/pb/lazy.lua` so the LSP sees them. + +## Two emission modes + +`mode=full` (default) inlines the encode/decode bodies; `mode=runtime` +emits one-line wrappers that delegate to `pb.encode` / `pb.decode`. +**Every other symbol on this page is mode-independent** — descriptor +tables, the lazy wrappers, text/JSON delegation, the +`_fields` / `_oneofs` strict tables, services, EmmyLua annotations +are identical across modes. See +[api-modes.md](../api-modes.md#full-inline-codegen) for the per-mode +trade-offs and how to pick. diff --git a/docs/reference/grpc-contract.md b/docs/reference/grpc-contract.md new file mode 100644 index 0000000000000000000000000000000000000000..e980429a4e091271e527d699a92adaf165884078 --- /dev/null +++ b/docs/reference/grpc-contract.md @@ -0,0 +1,204 @@ +# gRPC transport contract + +The four-method interface every transport implements, and the +stream-object shapes that flow between transport and generated code. + +This page is the **shipped, stable surface** of `runtime/pb/grpc.lua`. +Forward-looking spec work (which external transports to build, +status-code mapping, Connect vs HTTP/2 vs IProto trade-offs) lives in +[../specs/grpc_transports.md](../specs/grpc_transports.md). + +For the *user-facing* shape of `M._client` and +`M._server` see +[generated-api.md → per-service symbols](generated-api.md#per-service-symbols). + +## The transport interface + +A transport is any Lua table implementing these four methods: + +```lua +transport:unary(path, req_bytes, ctx) -> resp_bytes +transport:server_stream(path, req_bytes, ctx) -> stream +transport:client_stream(path, ctx) -> stream +transport:bidi(path, ctx) -> stream +``` + +- `path` is `/./`. Always slash-prefixed; same + format as the gRPC wire-level `:path` pseudo-header. +- `req_bytes` is the encoded request message (already passed through + `M._encode`). +- `resp_bytes` is the encoded reply, returned to the generated client + which decodes it via `M._decode`. +- `ctx` is an opaque table — reserved keys below. + +Errors raised by `error(...)` propagate to the caller verbatim. There +is no built-in retry, fallback, or status-code translation; transports +that need that wrap their core implementation. + +### Why this shape + +It's HTTP/2-shaped on purpose — path, byte-oriented messages, +streaming — but the contract makes **no commitment to a wire +protocol**. Loopback, multiplex, future HTTP/1.1 (Connect-JSON), +net.box-tunnel, and IProto transports all plug into the same four +methods. + +## The stream object — client view + +`server_stream`, `client_stream`, and `bidi` return a stream: + +```lua +stream:send(bytes) -- push a message (client_stream, bidi) +stream:close_send() -- "no more outgoing messages" +stream:recv() -> bytes, err -- pull next reply; (nil, err_or_nil) ends +stream:cancel() -- abort the call, drop pending messages +``` + +For `server_stream`, `:send` and `:close_send` are no-ops — the initial +request travels via the call's `req_bytes` argument. + +Generated client code wraps the raw `bytes`-typed stream with a typed +facade via `pb.grpc.wrap_server_stream` / `pb.grpc.wrap_call`, so user +code sees decoded messages: + +```lua +-- Inside a generated Greeter_client(transport): +StreamHellos = function(req, ctx) + local raw = transport:server_stream("/hello.Greeter/StreamHellos", encode(req), ctx) + return pb.grpc.wrap_server_stream(raw, M.HelloReply_decode) +end +``` + +## The stream object — server view + +Generated server code hands the impl a view that speaks decoded +messages directly: + +```lua +server_view:recv() -> message, err -- pull next request (client_stream, bidi) +server_view:send(message) -- push a reply (server_stream, bidi) +``` + +The handler signals end-of-stream by returning. Errors raised via +`error(...)` propagate to the client as the `err` return of +`stream:recv()`. + +`pb.grpc.wrap_server_view(raw, input_decode, output_encode)` is what +generated server code uses to build the view — `input_decode = nil` for +server_stream (no incoming messages past the request) and +`output_encode = nil` for client_stream (no outgoing messages past +the final reply). + +## Context (`ctx`) + +`ctx` is an opaque table threaded from caller to transport. Standard +keys (none required): + +| Key | Type | Meaning | +|---|---|---| +| `ctx.deadline` | number | fiber-clock timestamp (seconds, double). Transport enforces by canceling on overrun. | +| `ctx.headers` | `{string -> string}` | Flat metadata map. Wire-side translation is transport-specific (HTTP headers, IProto headers, …). | +| `ctx.trace_id`, `ctx.span_id` | string | Optional tracing hooks. Transports inject/extract per W3C `traceparent` for HTTP, custom IProto field for net.box. | +| `ctx.options` | table | Per-call overrides (retry policy, etc.). | + +User code should not put other keys in `ctx` — more standard keys may +be added. + +## Reference transports + +### `pb.grpc.loopback(server)` + +```lua +local server = M.Greeter_server(impl_table) +local transport = pb.grpc.loopback(server) +local client = M.Greeter_client(transport) + +client.SayHello({name = 'Alice'}) +``` + +In-process bridge using `fiber.channel`. Each streaming call spawns a +worker fiber for the handler and pipes messages through a paired +client/server stream view (`pb.grpc.new_stream_pair`). Channels +default to a 16-message buffer; senders block when full, receivers +block when empty. + +Use cases: tests, same-process apps (a Tarantool instance that +implements a service and also calls it locally). + +### `pb.grpc.multiplex({server1, server2, ...})` + +Fans multiple `M._server(impl)` results onto one transport. +Errors on duplicate paths. Useful when one Tarantool instance hosts +several services and you want a single shared transport. + +```lua +local greeter = MService_server(greeter_impl) +local catalog = CatalogService_server(catalog_impl) +local transport = pb.grpc.multiplex({greeter, catalog}) + +local greeter_client = MService_client(transport) +local catalog_client = CatalogService_client(transport) +``` + +### `pb.grpc.new_stream_pair(buf_size)` + +Lower-level: build a paired (client_stream, server_stream, state) over +two `fiber.channel`s. Used internally by `loopback`; exposed for +custom transports that want to reuse the framing without +re-implementing the cancel/close-send machinery. + +```lua +local client, server, state = pb.grpc.new_stream_pair(buf_size_or_nil) +-- run server-side handler on a fiber that consumes `server`, +-- return `client` from your transport's :bidi(...) implementation. +``` + +## Writing a custom transport + +A transport that talks to a real network endpoint implements the same +four methods. The minimal shape for a unary-only transport: + +```lua +local function http_transport(base_url) + local http = require('http.client').new() + return { + unary = function(_, path, req_bytes, ctx) + local r = http:request('POST', base_url .. path, req_bytes, { + headers = {['Content-Type'] = 'application/proto'}, + timeout = ctx and ctx.deadline, + }) + if r.status ~= 200 then + error(('grpc: HTTP ' .. r.status .. ': ' .. r.reason), 0) + end + return r.body + end, + -- Streaming methods: error or wrap as one-shot if not supported. + server_stream = function() error('streaming not supported', 0) end, + client_stream = function() error('streaming not supported', 0) end, + bidi = function() error('streaming not supported', 0) end, + } +end +``` + +For full streaming support, return objects implementing the +client-view stream interface (`:send`, `:close_send`, `:recv`, +`:cancel`). A pair of `fiber.channel`s plus the +`pb.grpc.new_stream_pair` helper covers most in-process needs; +networked transports drive the channels from their I/O callback. + +A worked example (a `net.box` tunnel stub) ships in +`examples/grpc/transport_netbox_stub.lua`; see also +[howto/13-custom-transport.md](../howto/13-custom-transport.md). + +## Helpers used by generated code + +These wrap a transport-level (bytes-typed) stream with a typed +(message-typed) facade. Application code rarely calls them directly; +they're documented here so custom-transport authors know what the +generated client/server code expects on either side. + +| Helper | Used by | Wraps | +|---|---|---| +| `pb.grpc.wrap_server_stream(raw, output_decode)` | generated client (server-stream methods) | adds `:recv -> decoded` to a raw `:recv -> bytes` stream | +| `pb.grpc.wrap_call(raw, input_encode, output_decode)` | generated client (client-stream + bidi) | adds `:send(msg)` / `:recv -> decoded` | +| `pb.grpc.wrap_server_view(raw, input_decode, output_encode)` | generated server | conditionally adds `:send` and `:recv` based on streaming kind | diff --git a/docs/reference/runtime-api.md b/docs/reference/runtime-api.md new file mode 100644 index 0000000000000000000000000000000000000000..14ad8711459b5b7e9b6fe1ff402f42e49f708f03 --- /dev/null +++ b/docs/reference/runtime-api.md @@ -0,0 +1,283 @@ +# `pb` runtime API reference + +Everything `require('pb')` exposes, organized by what you call from +application code. + +For *generated* per-message functions (`M.Foo_encode`, `M.Foo_text`, +etc.) see [generated-api.md](generated-api.md). For the descriptor +table shape see [../codegen.md](../codegen.md#the-descriptor-table--the-contract). + +## At a glance + +```lua +local pb = require('pb') + +local bytes = pb.encode(desc, t) -- table -> wire bytes +local t = pb.decode(desc, bytes) -- wire bytes -> table +local view = pb.decode_lazy(desc, bytes) -- wire bytes -> MessageView + +local mod = pb.parse(proto_source) -- .proto text -> module +local set = pb.from_pb(descset_bytes) -- FileDescriptorSet -> module set + +pb.json.encode(desc, t) -- proto3 JSON encode +pb.json.decode(desc, s [, opts]) -- proto3 JSON decode +pb.text.encode(desc, t [, opts]) -- text format encode +pb.text.decode(desc, s [, opts]) -- text format decode + +pb.NULL -- canonical null sentinel +pb.to_uint64(v) / pb.to_int64(v) -- coerce to 64-bit cdata + +pb.any.pack(desc, t [, prefix]) -- build google.protobuf.Any +pb.any.unpack(any_t [, desc]) -- unpack to {desc, t} +pb.register(desc) / pb.lookup(name) -- type registry for Any + +pb.grpc.loopback(server) -- in-process gRPC transport +pb.grpc.multiplex({srv1, srv2, ...}) -- fan multiple servers + +pb.field_names(t) -- strict field-name table (codegen) +pb.enum(name, {RED=0, ...}) -- enum descriptor (codegen) +pb.finalize_message(desc) -- finalize a hand-rolled descriptor +``` + +## Codec + +### `pb.encode(desc, t) -> string` + +Encode `t` against `desc` to protobuf wire bytes. Same input shape as +generated `M.Foo_encode(t)`; runs through descriptor dispatch instead of +inline code. ~5-15% slower than full-mode `Foo_encode` on the +microbenchmark; same allocation profile. + +### `pb.decode(desc, bytes) -> table` + +Decode wire bytes against `desc`. Returns a plain Lua table whose keys +match field names. Defaults are filled in per proto3 semantics; absent +explicit-optional fields stay `nil`. Unknown fields are concatenated +into `t._unknown_fields` (raw bytes, re-emitted on encode). + +### `pb.decode_lazy(desc, bytes) -> MessageView` + +Build a zero-copy view over the bytes. Nothing past the field index is +decoded until you call `:get`, `:has`, `:iter`, etc. See +[api-modes.md → lazy](../api-modes.md#lazy-zero-copy-view) for the full +surface and when it wins. + +### `pb.lazy` + +The lazy module itself (`pb.lazy.build`, the `MessageView` / +`ArrayView` / `MapView` classes). Most callers use `pb.decode_lazy`; +this is for advanced consumers that need to construct views by hand. + +## Dynamic descriptors + +Two ways to produce a descriptor module at runtime, both yielding the +same shape generated code emits in `mode=runtime`. + +### `pb.parse(source) -> module` + +Parse a `.proto` source string and return a module-shaped table +`{_descriptor = ..., _encode = ..., ...}`. The +parser handles proto3 syntax including services, options, imports, +nested messages, oneofs, maps, explicit-optional. WKT imports +(`google/protobuf/*.proto`) resolve to the `pb.wkt` descriptors +automatically. + +```lua +local hello = pb.parse(io.open('hello.proto'):read('*a')) +local bytes = hello.Person_encode({name = 'Alice'}) +``` + +### `pb.from_pb(descset_bytes) -> {files, order, lookup}` + +Parse a binary `FileDescriptorSet` (the output of +`protoc --descriptor_set_out=...`) and return: + +- `files[name]` — per-file module, indexed by the original `.proto` + filename. +- `order` — array of filenames in dependency order. +- `lookup(fqn) -> descriptor` — find any message or enum by its fully- + qualified name (e.g. `'hello.Person'`). + +```lua +local set = pb.from_pb(io.open('build/all.pb', 'rb'):read('*a')) +local desc = set.lookup('hello.Person') +local bytes = pb.encode(desc, {name = 'Alice'}) +``` + +### `pb.parser`, `pb.dynamic`, `pb.fileset` + +The submodules behind `pb.parse` / `pb.from_pb`. Exposed for callers +that want the AST step (`pb.parser.parse(text) -> ast`) or to build +descriptors by hand (`pb.dynamic.build(ast)`). + +## Codec dialects + +### `pb.json` + +Strict proto3 JSON. See `runtime/pb/json.lua` for the canonical-mapping +details (camelCase field names, base64 for `bytes`, RFC 3339 for +`Timestamp`, etc.). + +- `pb.json.encode(desc, t) -> string` +- `pb.json.decode(desc, s [, opts]) -> table` — + `opts.ignore_unknown_fields = true` accepts JSON with extra keys + (matches the conformance suite's `JSON_IGNORE_UNKNOWN_PARSING_TEST` + category). + +### `pb.text` + +Mainline-protoc text format, both directions. + +- `pb.text.encode(desc, t [, opts]) -> string` — `opts.single_line = + true` collapses to one space-separated line; `opts.indent = + ''` overrides the default two-space indent. +- `pb.text.decode(desc, s [, opts]) -> table` — handles every grammar + bucket the conformance text suite exercises (decimal/hex/octal int + literals, float specials, C-style and `\u`/`\U` escapes, `{}` / + `<>` aggregates, repeated short-form `[a, b, c]`, map entries, the + inline Any form, enum-by-name-or-number, reserved-name drop, and + numeric-field-ID tolerance). + +Both directions support `_unknown_fields` passthrough. + +## Well-known types — `pb.wkt` + +Hand-rolled descriptors for `google.protobuf.*`. Generated code that +references a WKT field routes to these automatically — you only touch +`pb.wkt` directly when packing/unpacking by hand or registering a type +for `Any`. + +| Symbol | Notes | +|---|---| +| `pb.wkt.Timestamp_descriptor` | Lua-side value: `datetime` cdata when in spec, or `{seconds, nanos}` table when out of spec. | +| `pb.wkt.Duration_descriptor` | Lua-side value: `{seconds, nanos}` table. | +| `pb.wkt.Empty_descriptor` | Lua-side value: `{}`. | +| `pb.wkt.Value_descriptor` (Int32, Int64, UInt32, UInt64, Bool, String, Bytes, Float, Double) | Auto-wrap/unwrap: pass the scalar directly, decode returns the scalar. | +| `pb.wkt.Struct_descriptor` | Lua-side value: a table tagged via `pb.wkt.struct(t)` if disambiguation is needed (Struct vs Value vs ListValue). | +| `pb.wkt.Value_descriptor` | Lua-side value: native Lua of matching shape; use `pb.NULL` for null. | +| `pb.wkt.ListValue_descriptor` | Lua-side value: array; use `pb.wkt.list(t)` to disambiguate. | +| `pb.wkt.FieldMask_descriptor` | Lua-side value: `{'foo.bar', 'baz', ...}`. | +| `pb.wkt.Any_descriptor` | Opaque `{type_url, value}` table by default; see [`Any`](#any) below. | +| `pb.wkt.NullValue_descriptor` | Enum with single value `NULL_VALUE = 0`. | + +### Tagging helpers + +```lua +local s = pb.wkt.struct({a = 1, b = 'x'}) -- table tagged as Struct +local l = pb.wkt.list({1, 2, 3}) -- table tagged as ListValue +``` + +Use these when stashing a value into `google.protobuf.Value` (or a +`Struct` field) and the codec can't infer whether you mean a `Struct`, +a `ListValue`, or a primitive map/array. + +### `Any` + +`pb.any.pack(desc, t [, prefix]) -> any_table` + +Encode `t` against `desc`, wrap as +`{type_url = '/', value = }`. Default prefix +is `type.googleapis.com`. + +`pb.any.unpack(any_t [, desc]) -> {desc, t}` or `(t, desc)` + +Decode the `value` bytes against the supplied descriptor, or +auto-resolve via the registry if `desc` is nil. + +`pb.register(desc)` / `pb.lookup(name_or_url) -> desc` + +Type registry. Generated code does **not** auto-register messages — +call `pb.register(M.Foo_descriptor)` once per type you want to round- +trip through `Any` by `type_url` alone. + +## gRPC — `pb.grpc` + +| Symbol | Notes | +|---|---| +| `pb.grpc.loopback(server)` | In-process transport. `server` is the table returned by `M._server(impl)`. Suitable for tests and same-process apps; uses `fiber.channel`. | +| `pb.grpc.multiplex({srv1, srv2, ...})` | Fan multiple `_server` results onto one transport. Errors on duplicate paths. | +| `pb.grpc.new_stream_pair(buf_size)` | Build a paired (client_stream, server_stream) over a `fiber.channel`. Used internally by `loopback`; exposed for custom transports. | +| `pb.grpc.wrap_*` | Helpers that wrap a raw stream/call with input/output codecs. Used by generated client/server code. | + +The transport *contract* (`:unary`, `:server_stream`, `:client_stream`, +`:bidi`) is documented in +[../specs/grpc_transports.md](../specs/grpc_transports.md). Any table +implementing those four methods plugs into a generated client. + +## Sentinels and coercions + +### `pb.NULL` + +Canonical null sentinel used by `google.protobuf.Value` and proto3 +JSON. Equal to `box.NULL` — preferred form is `pb.NULL` so application +code doesn't have to depend on `box` being available. + +### `pb.to_uint64(v) -> uint64_t cdata` +### `pb.to_int64(v) -> int64_t cdata` + +Coerce a Lua number, cdata, or numeric string into the matching +LuaJIT cdata. Use at any boundary where the input type isn't already +cdata (JSON, text format, net.box arguments, user input). The codec +otherwise requires cdata for the five 64-bit-typed fields and will +error on bare Lua numbers past 2^53. + +```lua +local id = pb.to_uint64('18446744073709551615') -- max uint64 +local t = {user_id = id} +``` + +## Codegen helpers (used by generated `_pb.lua`) + +Three helpers application code rarely calls directly — they're how +generated modules and hand-rolled descriptors get built: + +### `pb.field_names(tbl) -> tbl` + +Wrap a `{field_name = field_name, ...}` table with a strict +`__index` / `__newindex` so unknown keys error at the read site. +Generated code emits `M._fields = pb.field_names({...})` for use +with the lazy view (`view:get(F.user_id)`). + +### `pb.enum(name, {NAME = number, ...}) -> enum_descriptor` + +Build an enum descriptor with reversible `by_name` / `by_value` maps: + +```lua +local Color = pb.enum('app.Color', {RED = 0, GREEN = 1, BLUE = 2}) +Color.by_name.RED -- 0 +Color.by_value[1] -- 'GREEN' +``` + +### `pb.finalize_message(desc) -> desc` + +Fill in `field_by_id` / `field_by_name` / `oneofs_list`, mark cdata- +keyed maps for pointer-vs-value dedup on decode, and attach +per-field `_writer` / `_reader` specializations for the hot path. +Call after constructing `desc.fields[]`. Idempotent. + +## Wire-format primitives — `pb.wire` + +Low-level encode/decode for individual wire types. Generated full-mode +code inlines calls to these; application code rarely needs them +directly. The full surface is in `runtime/pb/wire.lua`. Highlights: + +- Wire-type constants: `pb.WIRE_VARINT`, `pb.WIRE_I64`, `pb.WIRE_LEN`, + `pb.WIRE_I32` (also under `pb.wire.WIRE_*`). +- Per-type encode/decode: `pb.wire.encode_int32`, `decode_string`, etc. + for all 15 scalar types. +- Tag handling: `pb.wire.encode_tag(id, wt)`, `decode_tag(buf, pos)`. +- Varint primitives: `encode_varint`, `decode_varint`, the four zigzag + variants. +- UTF-8 validator: `pb.wire.is_valid_utf8(s)` — ICU-backed, matches + every proto3 UTF-8 rejection rule. + +Adding a new scalar means touching `wire.lua` (primitives + +`TYPE_INFO`), `types.go` (Kind mapping), and `inline.go` (emission). +See [codegen.md → adding a new wire type](../codegen.md#plugin-source-layout). + +## Codec internals — `pb.codec` + +Exposed for generated inline code that wants to share helpers (e.g. +`pb.codec.merge_message` for sub-message merging on repeated decode). +Not intended as a stable application-facing surface; reach for the +high-level `pb.encode` / `pb.decode` instead. diff --git a/docs/specs/grpc_transports.md b/docs/specs/grpc_transports.md index a38cdc501e5d1a656eab77c348572f69d6bf4e08..4f055b0227572680e7ca61e313a2b1f77351b0af 100644 --- a/docs/specs/grpc_transports.md +++ b/docs/specs/grpc_transports.md @@ -1,9 +1,14 @@ # Spec: gRPC transports for Tarantool -Status: **draft / decision deferred**. The transport *contract* is -shipped and stable (see [`runtime/pb/grpc.lua`](../../runtime/pb/grpc.lua)). -What's open is **which concrete transports we recommend and/or ship, -and how a user picks between them**. +Status: **shipped contract + loopback/multiplex; external transports +deferred**. The transport *contract* is locked in +[`runtime/pb/grpc.lua`](../../runtime/pb/grpc.lua) — generated +`M._client(transport)` / `M._server(impl)` and the +four-method `transport:unary` / `:server_stream` / `:client_stream` / +`:bidi` interface are stable, with `loopback` and `multiplex` +reference transports in the runtime. What this spec covers is the +forward-looking work: **which concrete external transports we'd build +or recommend, and how a user picks between them**. This spec maps the protocol landscape, says where Tarantool fits, and flags what we'd build vs. recommend an external library for. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000000000000000000000000000000000000..0602a721636678b0b7373e885937770631d50e3d --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,276 @@ +# Troubleshooting + +Errors you might hit and how to fix them. Organized by where they +surface (build, require, encode, decode, runtime). + +## Build / codegen + +### `protoc-gen-tarantool: program not found or is not executable` + +`protoc` couldn't find the plugin on `PATH`. Either: + +```bash +# Put the binary on PATH: +export PATH=$PWD:$PATH + +# Or invoke protoc with an explicit plugin path: +protoc --plugin=protoc-gen-tarantool=./protoc-gen-tarantool ... +``` + +### `imported "tarantool/tarantool.proto" but couldn't find it` + +You're using `option (tarantool.lua_package) = ...` but `protoc` +doesn't see the option definition file. Add an `-I` for the +`options/` directory of this repo: + +```bash +protoc -I. -Ioptions --tarantool_out=... ... +``` + +If you've vendored the repo elsewhere, point to its `options/` +subdirectory. + +### `proto2 is not supported` + +The plugin rejects `syntax = "proto2";` files. Proto2 is a separate +slice — see [codegen.md → proto2 deferral](codegen.md#proto2-deferral) +for what it would take to add. For now, convert to proto3 or skip +those files. + +## Require / load + +### `module 'pb' not found` + +`LUA_PATH` is missing the `runtime/` directory. The two patterns you +need: + +```bash +LUA_PATH="./runtime/?/init.lua;./runtime/?.lua;...;;" +``` + +- `./runtime/?/init.lua` resolves `require('pb')` to + `runtime/pb/init.lua`. +- `./runtime/?.lua` resolves `require('pb.wire')` to + `runtime/pb/wire.lua` (and the other sibling modules). + +### `module 'app.foo_pb' not found` + +Generated module isn't on `LUA_PATH`. Add the generated-output +directory: + +```bash +LUA_PATH="...;./gen/?.lua;./gen/?/init.lua;...;;" +``` + +Include both patterns — `gen/?.lua` for top-level (`gen/foo_pb.lua`) +and `gen/?/init.lua` for nested layouts. + +### `attempt to call nil` on a WKT field + +The generated code references `pb.wkt._*` for WKT fields. If +the `pb` module hasn't loaded, the reference resolves to nil and the +call fails. Make sure `require('pb')` happens before (or as part of) +the generated module load — generated `_pb.lua` does this at the top +(`local pb = require('pb')`), so this only surfaces when something +has overridden `package.loaded.pb` or replaced the runtime. + +### Tarantool's built-in `protobuf` vs this project's `pb` + +If your code does `require('protobuf')` and gets the encode-only +built-in's API, that's because the project module is named `pb`, +not `protobuf`. They're deliberately different to coexist; see +[how-to: migrate from builtin](howto/11-migrate-from-builtin.md). + +## Encode + +### `expected cdata int64_t, got number` + +You passed a Lua number to a 64-bit-typed field (`int64`, `uint64`, +`sint64`, `fixed64`, `sfixed64`). The codec requires cdata for +those — Lua numbers lose precision past 2^53. + +```lua +local pb = require('pb') +hello.Person_encode({user_id = pb.to_uint64(42)}) +-- or +hello.Person_encode({user_id = require('ffi').cast('uint64_t', 42)}) +``` + +`pb.to_uint64` / `pb.to_int64` accept Lua numbers, cdata, or numeric +strings — use them at any boundary (JSON parsing, user input, +net.box arguments) where the input might not already be cdata. + +### `expected table for hello.Foo, got string` + +You passed wire bytes to `Foo_encode` instead of a Lua table. +`Foo_encode(t) -> bytes`, `Foo_decode(bytes) -> t`. Easy to swap. + +### `invalid UTF-8 in string field` + +Proto3 `string` fields require valid UTF-8 (the spec). Use `bytes` +for arbitrary binary, or sanitize/encode the input before assigning +to a `string` field. + +### Map encode order differs from mainline `protoc` + +Lua's `pairs()` iteration order over a hash table isn't stable +across Lua versions — it differs from mainline `protoc`'s +text-proto-order output. **Two-key+ maps are byte-equal *by +coincidence*** when the hash happens to iterate in the right order. + +Test impact: use single-key map fixtures for byte-for-byte interop +assertions; cover multi-key behavior with decode-then-compare-table +assertions where iteration order doesn't matter. The existing +`person_map` fixture in `test/interop/fixtures/` follows this rule. + +## Decode + +### `truncated input` / `unexpected end of input` + +The wire bytes are short — either the producer cut off mid-message or +your `bytes` variable doesn't hold what you think it holds. Check the +length first; mainline-protoc messages start with field-tag bytes, +not a length prefix. + +If you're reading from a length-delimited stream (gRPC frame, custom +framing), strip the framing before decoding. + +### `unknown wire type N` + +The bytes aren't proto3 wire format — likely you're decoding JSON, +msgpack, or some other format against a proto descriptor. The four +valid wire types are 0 (varint), 1 (i64), 2 (LEN), 5 (i32); 3 and 4 +(SGROUP/EGROUP) are proto2-only and tolerated on the skip path. + +### `enum value N out of range` + +Decoded enum value isn't in the descriptor. Proto3 enums are *open* — +unknown values pass through as their integer form. The decoder +doesn't raise on unknown values; if you're seeing this error, it's +likely from a strict consumer (JSON decode in strict mode, custom +validator) rather than the wire-level decoder. + +## Lazy view + +### `view:get('field_name')` returns nil when the field IS set + +Field-name typo, or you passed a literal string instead of the +strict constants table: + +```lua +-- Right +local F = hello.Person_fields +view:get(F.user_id) + +-- Wrong +view:get('user_id') -- works +view:get('user_di') -- silently nil (typo!) + +-- F.user_di errors at the call site: +-- "unknown field name: 'user_di'" +``` + +See [api-modes.md → field-name constants](api-modes.md#field-name-constants--required) +for the reasoning. + +### `MessageView holds a reference to the input bytes` + +Lazy views don't copy the wire bytes — they index into them. If you +mutate the underlying string or let it get GC'd, the view's reads +are undefined. + +In practice strings are immutable in Lua, so the only way to hit this +is to drop the *only* reference to the string while the view is +still in use. Keep the bytes around as long as the view is. + +## gRPC + +### Stream `:send` raises `pb.grpc: stream canceled` + +The peer (or this side) called `:cancel()`. Treat as end-of-stream; +the call is over. + +### Stream `:send` raises `pb.grpc: send after close_send` + +You called `:send` after `:close_send` on the same stream. Once +close_send fires, no more outgoing messages. + +### `unknown unary method "/pkg.Service/Method"` + +The path isn't in the server's `methods` map. Check that: + +- The service / method name matches between client and server + generated code. +- For `pb.grpc.multiplex`, every server was passed. +- The path uses the `/pkg.Service/Method` form, not just + `Method` or `Service.Method`. + +### Loopback hangs on a streaming call + +Single-fiber bidi can self-deadlock — if your send buffer fills +faster than the server pulls, `:send` blocks on the channel, and the +server fiber can't drain because the main fiber owns the runtime. +Split sends to a separate fiber, or interleave send/recv in lockstep. +See [how-to: gRPC loopback → bidi from a single fiber](howto/03-grpc-loopback.md#bidi-from-a-single-fiber). + +## Performance + +### `pairs()` warning from `bench/jit_trace.lua` + +The hot encode/decode paths must use `ipairs` / `for i=1,#t do`, not +`pairs`. `pairs()` over a hash compiles to bytecode `ISNEXT`, which +is NYI in Tarantool's LuaJIT 2.1 fork — the trace aborts. Map fields +are the one allowed exception; the gate pins the limitation. + +See [codegen.md → hot-path rules](codegen.md#the-hot-path-rules-the-generated-code-observes). + +### Encode is much slower than the bench reports + +Two common causes: + +1. **Tarantool starts with `jit.off`** by default in some contexts. + Confirm: `print(jit.status())` should print `true ...`. If false, + `jit.on()` enables it. +2. **JIT mcode disabled on macOS arm64.** `luatest` blocks JIT mcode + on that platform; benchmarks should run via `just bench`, not via + `luatest`. Tests of perf characteristics live in `bench/`, not + `test/`. + +## Conformance + +### `conformance_test_runner` reports skips on every proto2 test + +Expected. The plugin rejects proto2; the conformance runner skips +those test categories. The runner's CLI prints +`skipped : skipped` for each one. The total skip count +(currently 1313 binary + 18 text-format) matches the +`TestAllTypesProto2` bucket. See +[README § Conformance](../README.md#conformance) for the exact +numbers. + +## Other + +### Worktree-based dev: `.rocks` is missing + +After `git worktree add`, the new worktree doesn't share `.rocks/` +with the main checkout. Symlink it: + +```bash +ln -s ../path/to/main/.rocks .rocks +``` + +Otherwise `just test` can't find `luatest`. + +### Need to log a 64-bit cdata as a string + +`tostring(cdata_uint64)` gives `12345ULL` (LuaJIT formatting). +For clean output, use `tostring():gsub('ULL$', '')` or print the +numeric form via `tonumber()` (with the usual 2^53 precision +caveat — use only for display, not for math). + +### Where do generated modules log from? + +They don't. The codegen produces no `log.*` calls; if you see logs, +they came from your app or from `pb`'s runtime modules (none of +which log on the happy path). Error conditions surface as `error()`, +not log lines. diff --git a/examples/Justfile b/examples/Justfile new file mode 100644 index 0000000000000000000000000000000000000000..ab6bc00f30d6db5053266ffa17c8743032a75914 --- /dev/null +++ b/examples/Justfile @@ -0,0 +1,57 @@ +# examples/Justfile — one target per runnable example. +# +# Forwarded from the top-level Justfile via `just examples `. +# Recipes assume the parent repo's `just gen` has run so the +# examples/expected/* modules exist. +# +# Run individual examples: +# just examples quickstart # encode/decode round-trip +# just examples grpc-loopback # Greeter end-to-end, all 4 streaming flavors +# just examples dynamic-schemas # pb.parse on a schema stored in a Tarantool space +# just examples json-http # JSON-over-HTTP server (needs `tt rocks install http`) +# +# Run all non-interactive examples in sequence: +# just examples all + +set shell := ["bash", "-cu"] + +# Run from the repo root so the LUA_PATH globs land correctly. +repo := justfile_directory() / ".." + +# Shared LUA_PATH: runtime/ + generated examples/expected/. +lua_path := "./runtime/?/init.lua;./runtime/?.lua;./examples/expected/?.lua;./examples/expected/?/init.lua;;" + +# Show available example recipes. +default: + @just --list --justfile {{justfile()}} + +# Run every non-interactive example in sequence. +all: quickstart grpc-loopback dynamic-schemas + +# Encode/decode a quickstart.User round-trip. Drives the first-message howto. +quickstart: + @cd {{repo}} && LUA_PATH="{{lua_path}}" tarantool -e ' \ + local qs = require("full.quickstart.quickstart_pb") \ + local pb = require("pb") \ + local b = qs.User_encode({id = 7, name = "Alice", role = qs.Role.ADMIN, emails = {"a@x", "b@x"}}) \ + print("encoded " .. #b .. " bytes") \ + local u = qs.User_decode(b) \ + print(u.id, u.name, u.role, u.emails[1], u.emails[2]) \ + print(qs.User_text(u)) \ + ' + +# Greeter end-to-end via pb.grpc.loopback (all 4 streaming flavors). +grpc-loopback: + @cd {{repo}} && LUA_PATH="{{lua_path}}" tarantool examples/grpc/client.lua + +# pb.parse on schema text stored in a Tarantool space (cache + evolution). +dynamic-schemas: + @cd {{repo}} && LUA_PATH="{{lua_path}}" tarantool examples/dynamic/load_from_space.lua + +# Interactive JSON-over-HTTP server on :8080 (needs `tt rocks install http`). +json-http: + @cd {{repo}} && LUA_PATH="{{lua_path}}" tarantool examples/http/json_api.lua + +# Print the net.box transport stub (skeleton, not runnable on its own). +netbox-stub: + @cd {{repo}} && cat examples/grpc/transport_netbox_stub.lua diff --git a/examples/dynamic/load_from_space.lua b/examples/dynamic/load_from_space.lua new file mode 100644 index 0000000000000000000000000000000000000000..1dff822e005f55ff1056304281c314a24836bc04 --- /dev/null +++ b/examples/dynamic/load_from_space.lua @@ -0,0 +1,105 @@ +-- Load .proto schemas from a Tarantool space at runtime. +-- +-- Pattern: store the .proto source text in a space, parse it on demand, +-- cache the resulting module. Schema updates land in the space; readers +-- pick them up after a cache invalidation. +-- +-- Run with: +-- LUA_PATH="./runtime/?/init.lua;./runtime/?.lua;;" tarantool examples/dynamic/load_from_space.lua + +-- Ephemeral instance; state in /tmp so the example doesn't litter the +-- working directory with .snap / .xlog files. +local data_dir = '/tmp/tarantool-protobuf-dynamic-example' +os.execute('mkdir -p ' .. data_dir) +box.cfg{ + listen = nil, + memtx_dir = data_dir, + wal_dir = data_dir, + log = data_dir .. '/tarantool.log', +} + +local pb = require('pb') + +-- 1. Bootstrap a space that holds proto schemas keyed by name. +box.once('init_schemas', function() + box.schema.space.create('proto_schemas') + box.space.proto_schemas:format({ + {name = 'name', type = 'string'}, + {name = 'version', type = 'unsigned'}, + {name = 'source', type = 'string'}, + }) + box.space.proto_schemas:create_index('pk', {parts = {'name'}}) +end) + +-- 2. Insert a schema. In production this lands via your app's +-- admin / migration path; here we just write it inline. +box.space.proto_schemas:replace{'demo.user.v1', 1, [[ +syntax = "proto3"; +package demo.user.v1; + +message User { + int32 id = 1; + string name = 2; + repeated string tags = 3; +} +]]} + +-- 3. Cache parsed modules by (schema_name, version). Invalidate by +-- bumping the version row when you update the source. +local cache = {} + +local function get_module(name) + local row = box.space.proto_schemas:get(name) + if row == nil then + error(("schema %q not found"):format(name), 0) + end + local key = name .. '@' .. tostring(row.version) + local mod = cache[key] + if mod == nil then + mod = pb.parse(row.source) + cache[key] = mod + end + return mod +end + +-- 4. Use it like any generated module. +local demo = get_module('demo.user.v1') + +local bytes = demo.User_encode({ + id = 42, + name = 'Alice', + tags = {'admin', 'active'}, +}) +print(('encoded %d bytes'):format(#bytes)) + +local user = demo.User_decode(bytes) +print(('decoded: id=%d name=%s tags=%s,%s'):format( + user.id, user.name, user.tags[1], user.tags[2])) + +-- 5. Schema upgrade: bump version + source, next get_module rebuilds. +box.space.proto_schemas:replace{'demo.user.v1', 2, [[ +syntax = "proto3"; +package demo.user.v1; + +message User { + int32 id = 1; + string name = 2; + repeated string tags = 3; + string email = 4; +} +]]} + +local demo_v2 = get_module('demo.user.v1') + +-- v1-encoded bytes still decode (new field absent). +local user_v1 = demo_v2.User_decode(bytes) +print(('v2 reads v1 bytes: email=%s'):format(tostring(user_v1.email))) + +-- v2-encoded bytes carry the new field. +local bytes_v2 = demo_v2.User_encode({ + id = 42, name = 'Alice', tags = {'admin'}, email = 'alice@x', +}) +print(('v2 encoded %d bytes; email round-trips: %s') + :format(#bytes_v2, demo_v2.User_decode(bytes_v2).email)) + +os.exit(0) diff --git a/examples/expected/full/quickstart/quickstart_pb.lua b/examples/expected/full/quickstart/quickstart_pb.lua new file mode 100644 index 0000000000000000000000000000000000000000..2b92d04e07dd948a2a17268d19ddcfa2191e7633 --- /dev/null +++ b/examples/expected/full/quickstart/quickstart_pb.lua @@ -0,0 +1,151 @@ +-- Code generated by protoc-gen-tarantool. DO NOT EDIT. +-- source: quickstart.proto +-- syntax: proto3 +-- package: quickstart + +local pb = require("pb") +local wire = pb.wire + +local M = {} + +-- Enum: quickstart.Role +M.Role_descriptor = pb.enum("quickstart.Role", { + ROLE_UNSPECIFIED = 0, + USER = 1, + ADMIN = 2, +}) +M.Role = M.Role_descriptor.by_name + +-- Pre-declare message descriptors so cross-references resolve. +M.User_descriptor = {name = "quickstart.User"} + +-- Message: quickstart.User +M.User_descriptor.fields = { + {name="id", id=1, kind='scalar', proto_type="int32"}, + {name="name", id=2, kind='scalar', proto_type="string"}, + {name="role", id=3, kind='enum', enum=M.Role_descriptor}, + {name="emails", id=4, kind='scalar', proto_type="string", repeated=true}, +} +pb.finalize_message(M.User_descriptor) +M.User_fields = pb.field_names({ + id = "id", + name = "name", + role = "role", + emails = "emails", +}) + +-- EmmyLua / lua-language-server type annotations. +-- These are comments — no runtime effect. They give editors +-- autocomplete and type-checking for the generated wrappers. +---@alias quickstart.Role integer + +---@class quickstart.User +---@field id integer +---@field name string +---@field role quickstart.Role +---@field emails string[] + +---@param t? quickstart.User +---@return quickstart.User +function M.User_new(t) return t or {} end + +---@param t quickstart.User +---@return string +function M.User_encode(t) + if type(t) ~= 'table' then + error("expected table for quickstart.User, got " .. type(t), 0) + end + local out, n = {}, 0 + local v + -- field 1: id + v = t.id + if v ~= nil and v ~= 0 then + n = n + 1; out[n] = "\x08" + n = n + 1; out[n] = wire.encode_int32(v) + end + -- field 2: name + v = t.name + if v ~= nil and v ~= '' then + n = n + 1; out[n] = "\x12" + n = n + 1; out[n] = wire.encode_varint(#v) + n = n + 1; out[n] = v + end + -- field 3: role + v = t.role + if v ~= nil then + local nv = v + if type(v) == 'string' then + nv = M.Role[v] + if nv == nil then error("unknown enum value '" .. v .. "' for quickstart.Role", 0) end + end + if nv ~= 0 then + n = n + 1; out[n] = "\x18" + n = n + 1; out[n] = wire.encode_int32(nv) + end + end + -- field 4: emails + v = t.emails + if v ~= nil and #v > 0 then + local _tag = "\x22" + for _i = 1, #v do + local _b = v[_i] + n = n + 1; out[n] = _tag + n = n + 1; out[n] = wire.encode_varint(#_b) + n = n + 1; out[n] = _b + end + end + local _uf = t._unknown_fields + if _uf ~= nil and _uf ~= '' then n = n + 1; out[n] = _uf end + return table.concat(out) +end + +---@param b string +---@return quickstart.User +function M.User_decode(buf) + if type(buf) ~= 'string' then + error("expected string for quickstart.User decode, got " .. type(buf), 0) + end + local result = {} + local pos, len = 1, #buf + local _uf + while pos <= len do + local _tag_start = pos + local id, wt + id, wt, pos = wire.decode_tag(buf, pos) + if id == 1 then + local val + val, pos = wire.decode_int32(buf, pos) + result.id = val + elseif id == 2 then + local val + val, pos = wire.decode_string(buf, pos) + result.name = val + elseif id == 3 then + local u + u, pos = wire.decode_varint(buf, pos) + result.role = wire.varint_to_int32(u) + elseif id == 4 then + local list = result.emails + if list == nil then list = {}; result.emails = list end + local val + val, pos = wire.decode_string(buf, pos) + list[#list + 1] = val + else + pos = wire.skip_field(buf, pos, wt, id) + if _uf == nil then _uf = {} end + _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) + end + end + if _uf ~= nil then result._unknown_fields = table.concat(_uf) end + return result +end + +---@param b string +---@return pb.MessageView +function M.User_decode_lazy(b) return pb.decode_lazy(M.User_descriptor, b) end +---@param t quickstart.User +---@param opts? {single_line: boolean?, indent: string?} +---@return string +function M.User_text(t, opts) return pb.text.encode(M.User_descriptor, t, opts) end + +return M diff --git a/examples/expected/runtime/quickstart/quickstart_pb.lua b/examples/expected/runtime/quickstart/quickstart_pb.lua new file mode 100644 index 0000000000000000000000000000000000000000..f9c91070cdec615b9c07b23528a992acb1572588 --- /dev/null +++ b/examples/expected/runtime/quickstart/quickstart_pb.lua @@ -0,0 +1,65 @@ +-- Code generated by protoc-gen-tarantool. DO NOT EDIT. +-- source: quickstart.proto +-- syntax: proto3 +-- package: quickstart + +local pb = require("pb") +local wire = pb.wire + +local M = {} + +-- Enum: quickstart.Role +M.Role_descriptor = pb.enum("quickstart.Role", { + ROLE_UNSPECIFIED = 0, + USER = 1, + ADMIN = 2, +}) +M.Role = M.Role_descriptor.by_name + +-- Pre-declare message descriptors so cross-references resolve. +M.User_descriptor = {name = "quickstart.User"} + +-- Message: quickstart.User +M.User_descriptor.fields = { + {name="id", id=1, kind='scalar', proto_type="int32"}, + {name="name", id=2, kind='scalar', proto_type="string"}, + {name="role", id=3, kind='enum', enum=M.Role_descriptor}, + {name="emails", id=4, kind='scalar', proto_type="string", repeated=true}, +} +pb.finalize_message(M.User_descriptor) +M.User_fields = pb.field_names({ + id = "id", + name = "name", + role = "role", + emails = "emails", +}) + +-- EmmyLua / lua-language-server type annotations. +-- These are comments — no runtime effect. They give editors +-- autocomplete and type-checking for the generated wrappers. +---@alias quickstart.Role integer + +---@class quickstart.User +---@field id integer +---@field name string +---@field role quickstart.Role +---@field emails string[] + +---@param t? quickstart.User +---@return quickstart.User +function M.User_new(t) return t or {} end +---@param t quickstart.User +---@return string +function M.User_encode(t) return pb.encode(M.User_descriptor, t) end +---@param b string +---@return quickstart.User +function M.User_decode(b) return pb.decode(M.User_descriptor, b) end +---@param b string +---@return pb.MessageView +function M.User_decode_lazy(b) return pb.decode_lazy(M.User_descriptor, b) end +---@param t quickstart.User +---@param opts? {single_line: boolean?, indent: string?} +---@return string +function M.User_text(t, opts) return pb.text.encode(M.User_descriptor, t, opts) end + +return M diff --git a/examples/grpc/client.lua b/examples/grpc/client.lua new file mode 100644 index 0000000000000000000000000000000000000000..4c9164ac20ed7c82f608534eec13d7bc1d79b417 --- /dev/null +++ b/examples/grpc/client.lua @@ -0,0 +1,64 @@ +-- Greeter client example. Loads the server-side from server.lua, builds +-- a loopback transport, and exercises every streaming flavor. +-- +-- Run with: +-- LUA_PATH="./runtime/?/init.lua;./runtime/?.lua;./examples/expected/?.lua;./examples/expected/?/init.lua;;" \ +-- tarantool examples/grpc/client.lua + +local fiber = require('fiber') +local hello = require('full.hello.hello_pb') + +-- Builds a transport — see examples/grpc/server.lua. +local transport = dofile('examples/grpc/server.lua') + +local client = hello.Greeter_client(transport) + +-- 1. Unary +print('--- unary ---') +local reply = client.SayHello({name = 'Alice'}, {}) +print(reply.greeting) + +-- 2. Server-stream +print('--- server-stream ---') +local s = client.StreamHellos({name = 'Bob'}, {}) +while true do + local msg, err = s:recv() + if msg == nil then + if err ~= nil then error(err, 0) end + break + end + print(msg.greeting) +end + +-- 3. Client-stream +print('--- client-stream ---') +local c = client.CollectHellos({}) +c:send({name = 'Alice'}) +c:send({name = 'Bob'}) +c:send({name = 'Carol'}) +c:close_send() +local final = c:recv() +print(final.greeting) + +-- 4. Bidi +print('--- bidi ---') +local b = client.Chat({}) + +-- Producer fiber pushes a few messages then closes its side. +fiber.create(function() + for _, name in ipairs({'X', 'Y', 'Z'}) do + b:send({name = name}) + end + b:close_send() +end) + +while true do + local msg, err = b:recv() + if msg == nil then + if err ~= nil then error(err, 0) end + break + end + print(msg.greeting) +end + +os.exit(0) diff --git a/examples/grpc/server.lua b/examples/grpc/server.lua new file mode 100644 index 0000000000000000000000000000000000000000..2b7b1abd491a22c798de3f676ee527f27f9888ac --- /dev/null +++ b/examples/grpc/server.lua @@ -0,0 +1,59 @@ +-- Standalone Greeter server example. +-- +-- Run via the matching client: +-- LUA_PATH="./runtime/?/init.lua;./runtime/?.lua;./examples/expected/?.lua;./examples/expected/?/init.lua;;" \ +-- tarantool examples/grpc/client.lua +-- +-- This file is required by client.lua, not run directly. It returns a +-- transport that the client wraps with the generated stub. + +local pb = require('pb') +local hello = require('full.hello.hello_pb') + +local impl = { + -- Unary + SayHello = function(req, _ctx) + return {greeting = 'Hi ' .. req.name} + end, + + -- Unary: echo the request as the reply (any HelloRequest works). + Echo = function(req, _ctx) + return req + end, + + -- Server-stream: push N replies, then close by returning. + StreamHellos = function(req, stream, _ctx) + for i = 1, 3 do + stream:send({greeting = ('Hi #%d %s'):format(i, req.name)}) + end + end, + + -- Client-stream: collect N requests into one reply. + CollectHellos = function(stream, _ctx) + local names = {} + while true do + local req, err = stream:recv() + if req == nil then + if err ~= nil then error(err, 0) end + break + end + names[#names + 1] = req.name + end + return {greeting = 'Hi ' .. table.concat(names, ', ')} + end, + + -- Bidi: echo each incoming, close when peer closes. + Chat = function(stream, _ctx) + while true do + local req, err = stream:recv() + if req == nil then + if err ~= nil then error(err, 0) end + return + end + stream:send({greeting = 'Echo ' .. req.name}) + end + end, +} + +local server = hello.Greeter_server(impl) +return pb.grpc.loopback(server) diff --git a/examples/grpc/transport_netbox_stub.lua b/examples/grpc/transport_netbox_stub.lua new file mode 100644 index 0000000000000000000000000000000000000000..ac0f2ae0c7360480db066830af1442deff790c2d --- /dev/null +++ b/examples/grpc/transport_netbox_stub.lua @@ -0,0 +1,63 @@ +-- net.box gRPC tunnel — STUB / illustrative. +-- +-- This file shows the *shape* of a custom transport. It's not a +-- production net.box transport; the real one needs error mapping, +-- deadline enforcement, metadata round-tripping, and proper +-- streaming. This stub is intentionally minimal — under 80 lines so +-- you can read it top-to-bottom. +-- +-- Pattern: +-- client side: transport:unary(path, req_bytes, ctx) +-- -> conn:call('grpc_dispatch', {path, req_bytes}) +-- -> {ok, resp_bytes} | {err, msg} +-- server side: function grpc_dispatch(path, req_bytes) +-- -> route to the right M._server method +-- -> return {true, resp_bytes} | {false, err_msg} + +local fiber = require('fiber') + +local M = {} + +-- Server-side: register a stored function that dispatches into a +-- gRPC server table (the result of M._server(impl)). +function M.register_server(server, func_name) + func_name = func_name or 'grpc_dispatch' + + -- box.session.push is the streaming primitive in net.box; this + -- stub doesn't use it. Streaming would need separate functions + -- or a stateful session. + rawset(_G, func_name, function(path, req_bytes) + local handler = server.methods[path] + if handler == nil then + return {false, 'unknown method: ' .. path} + end + local ok, resp = pcall(handler, req_bytes, {}) + if not ok then return {false, tostring(resp)} end + return {true, resp} + end) +end + +-- Client-side: returns a transport object implementing the contract. +-- conn is a net.box connection (require('net.box').connect(...)). +function M.client(conn, func_name) + func_name = func_name or 'grpc_dispatch' + + return { + unary = function(_, path, req_bytes, _ctx) + local result = conn:call(func_name, {path, req_bytes}) + if not result[1] then + error('grpc: ' .. tostring(result[2]), 0) + end + return result[2] + end, + + -- Streaming methods: error explicitly. A real transport + -- would set up box.session.push or a dedicated stream + -- function on the server side. + server_stream = function() error('streaming not supported in stub', 0) end, + client_stream = function() error('streaming not supported in stub', 0) end, + bidi = function() error('streaming not supported in stub', 0) end, + } +end + +return M diff --git a/examples/http/json_api.lua b/examples/http/json_api.lua new file mode 100644 index 0000000000000000000000000000000000000000..dea607b286a28cd6247ee42ff0e165dabc033910 --- /dev/null +++ b/examples/http/json_api.lua @@ -0,0 +1,59 @@ +-- JSON-over-HTTP server example using tarantool/http. +-- +-- Pattern: +-- POST /v1/users +-- Content-Type: application/json +-- Body: proto3 JSON for hello.Person +-- Response: +-- 200 OK +-- Content-Type: application/json +-- Body: same shape, plus the generated user_id +-- +-- Run with: +-- LUA_PATH="./runtime/?/init.lua;./runtime/?.lua;./examples/expected/?.lua;./examples/expected/?/init.lua;;" \ +-- tarantool examples/http/json_api.lua +-- +-- Then: +-- curl -X POST http://127.0.0.1:8080/v1/users \ +-- -H 'Content-Type: application/json' \ +-- -d '{"name":"Alice","age":30,"emails":["a@x"]}' + +local pb = require('pb') +local hello = require('full.hello.hello_pb') + +-- tarantool-http is installed via `tt rocks install http`; without it, +-- swap in your project's preferred HTTP framework. +local httpd_ok, http = pcall(require, 'http.server') +if not httpd_ok then + error('this example needs the `http.server` rock: tt rocks install http') +end + +local httpd = http.new('127.0.0.1', 8080) + +httpd:route({path = '/v1/users', method = 'POST'}, function(req) + -- Parse the request body as proto3 JSON. + local body = req:read_cached() + local ok, person = pcall(pb.json.decode, hello.Person_descriptor, body, { + -- Accept (and ignore) keys the schema doesn't know — handy for + -- forward-compat clients that send extras. + ignore_unknown_fields = true, + }) + if not ok then + return {status = 400, headers = {['content-type'] = 'application/json'}, + body = pb.json.encode(hello.HelloReply_descriptor, + {greeting = 'bad request: ' .. person})} + end + + -- (Pretend) business logic: assign a user_id. + person.user_id = pb.to_uint64(42) + + return { + status = 200, + headers = {['content-type'] = 'application/json'}, + body = pb.json.encode(hello.Person_descriptor, person), + } +end) + +httpd:start() +print('listening on http://127.0.0.1:8080') +require('fiber').sleep(math.huge) -- block; Ctrl-C to stop diff --git a/examples/proto/quickstart.proto b/examples/proto/quickstart.proto new file mode 100644 index 0000000000000000000000000000000000000000..39edab5b1cd40504a7039aec9d22787fe9eeb40f --- /dev/null +++ b/examples/proto/quickstart.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package quickstart; + +// One message, one enum. Enough to exercise the round-trip path. +// +// Generate with: +// protoc --tarantool_out=. examples/proto/quickstart.proto +// +// Then in Tarantool: +// local qs = require('quickstart.quickstart_pb') +// local bytes = qs.User_encode({id = 7, name = 'Alice', role = qs.Role.ADMIN}) + +enum Role { + ROLE_UNSPECIFIED = 0; + USER = 1; + ADMIN = 2; +} + +message User { + int32 id = 1; + string name = 2; + Role role = 3; + repeated string emails = 4; +}