From 6e7835a29a3678aa86a49dfff76c55b18d091755 Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Sat, 23 May 2026 13:25:00 +0300 Subject: [PATCH] c_runtime: encode/decode singular sub-messages (ra6 3d) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor encode_lua/decode_lua into encode_body/decode_body so the field-walk loop is callable recursively, then dispatch the MESSAGE kind into a per-side sub-handler. Repeated and map fields still skip at the field-walk level — 3e (jc9) and 3h (asz) land them next. encode_submessage_field force-establishes the parent enc_buf's heap_idx via a no-op ebuf_grow before recursing. Without that the final ebuf_reserve on the parent could land its new userdata above sub-encode's leaked stack slots, making the closing lua_settop drop the parent's heap. decode_submessage_field bounds the inner read by temporarily shrinking c->len to the sub-message end offset; the wire-prim helpers already bounds-check against c->len, so a malformed inner payload can't over-read into the outer message's bytes. New fixture test/proto/c_nested.proto carries a 5-level singular chain (L1->L2->L3->L4->L5) for the depth test. The two existing "skip message" tests are renamed to "skip repeated and map" — sub- messages now encode and decode end-to-end. bd-hwe --- .beads/interactions.jsonl | 1 + .beads/issues.jsonl | 2 +- .../expected/full/c_nested/c_nested_pb.lua | 625 ++++++++++++++++++ .../expected/runtime/c_nested/c_nested_pb.lua | 187 ++++++ runtime/pb/c/c_runtime.c | 274 ++++++-- test/c_runtime_decode_test.lua | 75 ++- test/c_runtime_encode_test.lua | 85 ++- test/proto/c_nested.proto | 13 + 8 files changed, 1177 insertions(+), 85 deletions(-) create mode 100644 examples/expected/full/c_nested/c_nested_pb.lua create mode 100644 examples/expected/runtime/c_nested/c_nested_pb.lua create mode 100644 test/proto/c_nested.proto diff --git a/.beads/interactions.jsonl b/.beads/interactions.jsonl index fcfa5db925d65d97189990d8844d61d9473dab30..7fad4382bc1b635cd058d9450bc3f888af4aec95 100644 --- a/.beads/interactions.jsonl +++ b/.beads/interactions.jsonl @@ -21,3 +21,4 @@ {"id":"int-b6e91276","kind":"field_change","created_at":"2026-05-19T04:46:39.175032Z","actor":"Eugene Blikh","issue_id":"tarantool-protobuf-mq7","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Descriptor -> C plan compiler landed in runtime/pb/c/c_runtime.c. Compiles a finalized Lua descriptor into an opaque pb_plan userdata with: per-field records (field_number, wire_type, kind, repeated/packed/optional, pre-encoded tag bytes, sub_plan_idx, oneof_idx, enum_ref), oneof grouping (parallel array with member indices + field oneof_idx back-pointers), WKT override pointers (has_override + override_encode_ref/decode_ref skipping field-walk), extension range hooks (proto2 scaffolding), field-name luaL_ref cache, sub-plan table for cycle-safe recursion. Self-references (Person.friends -> Person) handled by stashing c_plan on desc BEFORE recursing, so resolve_sub_plan returns the same userdata. Idempotent: re-compiling returns cached plan. Build via 'just build-c'. Smoke test at test/c_runtime_plan_test.lua: 26 assertions across both codegen modes (full + runtime) verify module surface, scalar/enum/message/map/repeated/packed shapes, sub-plan resolution, self-reference cycle break, oneof grouping (Result.outcome), idempotency, WKT override detection (Timestamp). Full suite: 771/771 with PB_ENABLE_C=1, 745+26 skipped without (silent fallback verified). Unblocks 3b/3c/3l/3k (y1n, mz6, awv, rmf)."}} {"id":"int-59f687bd","kind":"field_change","created_at":"2026-05-23T09:41:30.983997Z","actor":"Eugene Blikh","issue_id":"tarantool-protobuf-y1n","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"C-side singular-scalar encode landed in runtime/pb/c/c_runtime.c. New entry pb.c_runtime.encode(plan, msg) -> string. Buffer strategy: 4KB stack scratch promoted to lua_newuserdata on overflow — mid-encode luaL_error doesn't leak (userdata GC'd at unwind). Covered kinds: int32/64, uint32/64, sint32/64, bool, fixed32/64, sfixed32/64, float, double, enum (number/string lookup via field->enum_ref's by_name), string, bytes. Repeated/map/message silently skipped (3d/3e scope). Proto3 zero-suppression mirrors mode=full exactly: empty strings, zero ints/fixed, +0.0 double skipped; -0.0 emitted via type-pun byte equality (matches Lua's 1/v == -math.huge guard); proto3-optional fields bypass suppression. WKT override plans rejected (3k/bd-rmf scope). New test test/c_runtime_encode_test.lua: 16 cases per mode (full + runtime) = 32 tests covering acceptance (Person {name='x', age=42, balance=-7, user_id=0xDEADBEEFCAFEBABEULL, weight_kg=3.14} byte-equal to mode=full), per-kind sweep, edge cases (proto3-optional empty string, -0.0 double, enum-string lookup + unknown-name error, cdata uint64 fixed64, heap-buffer grow at 8KB string, repeated/message-skip silent). Full suite: 806/806 with PB_ENABLE_C=1, 748/748 + 58 skipped without (silent fallback intact). Unblocks 3d/3e/3g/3i/3j."}} {"id":"int-32f1e096","kind":"field_change","created_at":"2026-05-23T10:00:25.652311Z","actor":"Eugene Blikh","issue_id":"tarantool-protobuf-mz6","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"C-side singular scalar/enum/string/bytes decode. pb.c_runtime.decode(plan, bytes) -> table mirrors mode=full output: int64 family pushes cdata via luaL_pushint64/pushuint64, others Lua number/string/boolean. Linear field_number scan over plan->fields per tag; repeated/map/message and unknown tags skipped by wire type — 3d/3e/3i extend later. 26 tests pass (full + runtime modes); full suite 832/832 with PB_ENABLE_C=1, 748/748+84 skipped without."}} +{"id":"int-634d0065","kind":"field_change","created_at":"2026-05-23T10:24:19.417709Z","actor":"Eugene Blikh","issue_id":"tarantool-protobuf-hwe","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"ra6 3d: sub-message encode/decode (recursion) — refactored encode_lua/decode_lua into reusable encode_body/decode_body, added singular sub-message encode (force-promoted parent heap_idx before recursion to make lua_settop safe) and decode (temporarily shrunk c->len for bounded inner read). 22 new tests cover round-trip, empty sub-message presence, proto3-optional fields, 5-level depth (new test/proto/c_nested.proto), parent-buffer regrow path, truncated-input errors. Full suite 854/854 with PB_ENABLE_C=1."}} diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 375093c5d93e937fde45dff4fdef33285564f553..8ccd24b1350418d670617ed8f6ac78edc406c3f8 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,6 +1,6 @@ {"_type":"issue","id":"tarantool-protobuf-exy","title":"ra6 3f: repeated strings + repeated messages (cached stack-idx)","description":"Repeated string fields and repeated message fields. Encode: iterate the Lua array via cached stack idx, write each element (string field or recursive submsg encode). Decode: first hit lazy-creates the result array and stashes its stack index in a per-field slot (sized by plan-\u003en_fields, cap 16 for the spike fixed-size); subsequent hits lua_rawseti directly without re-lookup; one lua_setfield at decode_message exit attaches the array to the result table. This is the SPIKE-VALIDATED PATTERN — naive lazy-getfield is 2x slower at 100KB. Depends on 3d (needs sub-message machinery for repeated messages). Acceptance: Person.emails (repeated string) and Person.friends (repeated Person, self-reference) both round-trip byte-equal to mode=full at 1KB, 10KB, 100KB sizes.","status":"open","priority":1,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:52Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:20:52Z","dependencies":[{"issue_id":"tarantool-protobuf-exy","depends_on_id":"tarantool-protobuf-hwe","type":"blocks","created_at":"2026-05-18T23:22:18Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-jc9","title":"ra6 3e: repeated/packed scalar encode/decode","description":"Repeated scalar fields: encode-time, walk the Lua array via cached stack idx; decode-time, lazy-create the array on first hit and append via lua_rawseti with a cached stack idx (NOT lazy lua_getfield per element — that's 2x slower at 100KB per the spike). Packed repeated: length-prefixed varint blob; tight loop in C. Covers int32/sint32/uint32/int64/uint64/fixed32/fixed64/double/bool/enum. Depends on 3b + 3c (need the scalar primitives). Acceptance: Person.lucky_numbers (packed int32) round-trips byte-equal; a fixture with both packed and unpacked repeated scalars round-trips against mode=full at 10/100/1000-element counts.","status":"open","priority":1,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:33Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:20:33Z","dependencies":[{"issue_id":"tarantool-protobuf-jc9","depends_on_id":"tarantool-protobuf-mz6","type":"blocks","created_at":"2026-05-18T23:22:07Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-jc9","depends_on_id":"tarantool-protobuf-y1n","type":"blocks","created_at":"2026-05-18T23:22:07Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":2,"dependent_count":2,"comment_count":0} -{"_type":"issue","id":"tarantool-protobuf-hwe","title":"ra6 3d: sub-message encode/decode (recursion)","description":"Nested-message support. Encode: open a sub-buffer, recurse with the submsg plan, prefix parent buffer with tag + length. Decode: bound the byte range, recurse, lua_setfield the resulting table. Sub-buffer stack-backed (4KB) and malloc-promoted, same pattern as the outer buffer. Recursion is by C function call into the same encode/decode routine with a different plan, so depth is bounded by C stack. Depends on 3b + 3c. Acceptance: Person.address (1-level) round-trips; deeper nesting via a 5-level test fixture round-trips byte-equal to mode=full.","status":"open","priority":1,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:24Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:20:24Z","dependencies":[{"issue_id":"tarantool-protobuf-hwe","depends_on_id":"tarantool-protobuf-mz6","type":"blocks","created_at":"2026-05-18T23:22:06Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-hwe","depends_on_id":"tarantool-protobuf-y1n","type":"blocks","created_at":"2026-05-18T23:22:05Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":2,"dependent_count":4,"comment_count":0} +{"_type":"issue","id":"tarantool-protobuf-hwe","title":"ra6 3d: sub-message encode/decode (recursion)","description":"Nested-message support. Encode: open a sub-buffer, recurse with the submsg plan, prefix parent buffer with tag + length. Decode: bound the byte range, recurse, lua_setfield the resulting table. Sub-buffer stack-backed (4KB) and malloc-promoted, same pattern as the outer buffer. Recursion is by C function call into the same encode/decode routine with a different plan, so depth is bounded by C stack. Depends on 3b + 3c. Acceptance: Person.address (1-level) round-trips; deeper nesting via a 5-level test fixture round-trips byte-equal to mode=full.","status":"closed","priority":1,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:24Z","created_by":"Eugene Blikh","updated_at":"2026-05-23T10:24:19Z","started_at":"2026-05-23T10:11:41Z","closed_at":"2026-05-23T10:24:19Z","close_reason":"ra6 3d: sub-message encode/decode (recursion) — refactored encode_lua/decode_lua into reusable encode_body/decode_body, added singular sub-message encode (force-promoted parent heap_idx before recursion to make lua_settop safe) and decode (temporarily shrunk c-\u003elen for bounded inner read). 22 new tests cover round-trip, empty sub-message presence, proto3-optional fields, 5-level depth (new test/proto/c_nested.proto), parent-buffer regrow path, truncated-input errors. Full suite 854/854 with PB_ENABLE_C=1.","dependencies":[{"issue_id":"tarantool-protobuf-hwe","depends_on_id":"tarantool-protobuf-mz6","type":"blocks","created_at":"2026-05-18T23:22:06Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-hwe","depends_on_id":"tarantool-protobuf-y1n","type":"blocks","created_at":"2026-05-18T23:22:05Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":2,"dependent_count":4,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-awv","title":"ra6 3l: 64-bit cdata fidelity","description":"int64/uint64/fixed64/sfixed64 must round-trip as LuaJIT cdata in both encode and decode — never narrowed to a Lua double. Same convention as msgpackffi, net.box, box.tuple, the built-in protobuf. Use luaT_pushuint64/luaT_pushint64 from Tarantool's module.h for the C side. Independent transverse concern — can land in parallel with 3b/3c since it's a per-kind fix rather than a new pipeline stage. Acceptance: a value \u003e 2^53 round-trips byte-equal and remains cdata after decode; luatest test/protobuf_test.lua int64-cdata cases pass with PB_ENABLE_C=1.","status":"open","priority":1,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:10Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:20:10Z","dependencies":[{"issue_id":"tarantool-protobuf-awv","depends_on_id":"tarantool-protobuf-mq7","type":"blocks","created_at":"2026-05-18T23:21:55Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-mz6","title":"ra6 3c: decode scalars","description":"C-side decode for scalar wire types. Pre-sized result table via lua_createtable(0, n_fields) using descriptor stats; per-field plan-driven dispatch on wire type and kind; cached luaL_ref-based lua_setfield. Wire→table output must equal mode=full Lua output for the same input bytes. Depends on 3a. Acceptance: Person decode round-trip works for the bytes produced by 3b; the decoded table is shape-identical to pure-Lua decode (verified by running test/interop tests with PB_ENABLE_C=1).","status":"closed","priority":1,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:06Z","created_by":"Eugene Blikh","updated_at":"2026-05-23T10:00:26Z","started_at":"2026-05-23T09:54:01Z","closed_at":"2026-05-23T10:00:26Z","close_reason":"C-side singular scalar/enum/string/bytes decode. pb.c_runtime.decode(plan, bytes) -\u003e table mirrors mode=full output: int64 family pushes cdata via luaL_pushint64/pushuint64, others Lua number/string/boolean. Linear field_number scan over plan-\u003efields per tag; repeated/map/message and unknown tags skipped by wire type — 3d/3e/3i extend later. 26 tests pass (full + runtime modes); full suite 832/832 with PB_ENABLE_C=1, 748/748+84 skipped without.","dependencies":[{"issue_id":"tarantool-protobuf-mz6","depends_on_id":"tarantool-protobuf-mq7","type":"blocks","created_at":"2026-05-18T23:21:54Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":6,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-y1n","title":"ra6 3b: encode scalars (varint, fixed32/64, double, bool, enum, sint)","description":"C-side encode for scalar wire types into the message's output buffer. Reads field values via cached luaL_ref + lua_rawgeti, dispatches on plan field kind, writes varint/fixed32/fixed64/double/bool/enum/sint into the 4KB stack-backed buffer with malloc-promotion on overflow. Wire-byte output must be byte-equal to mode=full Lua output. Depends on 3a (needs the plan). Acceptance: Person encode round-trip works for {name='x', age=42, balance=-7, user_id=0xDEADBEEFCAFEBABEULL, weight_kg=3.14} with byte-equal output to mode=full pure Lua.","status":"closed","priority":1,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:03Z","created_by":"Eugene Blikh","updated_at":"2026-05-23T09:41:31Z","started_at":"2026-05-23T09:32:14Z","closed_at":"2026-05-23T09:41:31Z","close_reason":"C-side singular-scalar encode landed in runtime/pb/c/c_runtime.c. New entry pb.c_runtime.encode(plan, msg) -\u003e string. Buffer strategy: 4KB stack scratch promoted to lua_newuserdata on overflow — mid-encode luaL_error doesn't leak (userdata GC'd at unwind). Covered kinds: int32/64, uint32/64, sint32/64, bool, fixed32/64, sfixed32/64, float, double, enum (number/string lookup via field-\u003eenum_ref's by_name), string, bytes. Repeated/map/message silently skipped (3d/3e scope). Proto3 zero-suppression mirrors mode=full exactly: empty strings, zero ints/fixed, +0.0 double skipped; -0.0 emitted via type-pun byte equality (matches Lua's 1/v == -math.huge guard); proto3-optional fields bypass suppression. WKT override plans rejected (3k/bd-rmf scope). New test test/c_runtime_encode_test.lua: 16 cases per mode (full + runtime) = 32 tests covering acceptance (Person {name='x', age=42, balance=-7, user_id=0xDEADBEEFCAFEBABEULL, weight_kg=3.14} byte-equal to mode=full), per-kind sweep, edge cases (proto3-optional empty string, -0.0 double, enum-string lookup + unknown-name error, cdata uint64 fixed64, heap-buffer grow at 8KB string, repeated/message-skip silent). Full suite: 806/806 with PB_ENABLE_C=1, 748/748 + 58 skipped without (silent fallback intact). Unblocks 3d/3e/3g/3i/3j.","dependencies":[{"issue_id":"tarantool-protobuf-y1n","depends_on_id":"tarantool-protobuf-mq7","type":"blocks","created_at":"2026-05-18T23:21:53Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":6,"comment_count":0} diff --git a/examples/expected/full/c_nested/c_nested_pb.lua b/examples/expected/full/c_nested/c_nested_pb.lua new file mode 100644 index 0000000000000000000000000000000000000000..4637c8578e5d41831a2294af490a968791d4867b --- /dev/null +++ b/examples/expected/full/c_nested/c_nested_pb.lua @@ -0,0 +1,625 @@ +-- Code generated by protoc-gen-tarantool. DO NOT EDIT. +-- source: c_nested.proto +-- syntax: proto3 +-- package: c_nested + +local pb = require("pb") +local wire = pb.wire +local string_byte = string.byte +local band = bit.band +local rshift = bit.rshift + +local M = {} + +M.options = {go_package = "tarantoolpb_synthetic/c_nested"} + +-- Pre-declare message descriptors so cross-references resolve. +M.L1_descriptor = {name = "c_nested.L1"} +M.L2_descriptor = {name = "c_nested.L2"} +M.L3_descriptor = {name = "c_nested.L3"} +M.L4_descriptor = {name = "c_nested.L4"} +M.L5_descriptor = {name = "c_nested.L5"} + +-- Message: c_nested.L1 +M.L1_descriptor.fields = { + {name="next", id=1, kind='message', message=M.L2_descriptor}, + {name="v", id=2, kind='scalar', proto_type="int32"}, +} +pb.finalize_message(M.L1_descriptor) +M.L1_fields = pb.field_names({ + next = "next", + v = "v", +}) + +-- Message: c_nested.L2 +M.L2_descriptor.fields = { + {name="next", id=1, kind='message', message=M.L3_descriptor}, + {name="v", id=2, kind='scalar', proto_type="int32"}, +} +pb.finalize_message(M.L2_descriptor) +M.L2_fields = pb.field_names({ + next = "next", + v = "v", +}) + +-- Message: c_nested.L3 +M.L3_descriptor.fields = { + {name="next", id=1, kind='message', message=M.L4_descriptor}, + {name="v", id=2, kind='scalar', proto_type="int32"}, +} +pb.finalize_message(M.L3_descriptor) +M.L3_fields = pb.field_names({ + next = "next", + v = "v", +}) + +-- Message: c_nested.L4 +M.L4_descriptor.fields = { + {name="next", id=1, kind='message', message=M.L5_descriptor}, + {name="v", id=2, kind='scalar', proto_type="int32"}, +} +pb.finalize_message(M.L4_descriptor) +M.L4_fields = pb.field_names({ + next = "next", + v = "v", +}) + +-- Message: c_nested.L5 +M.L5_descriptor.fields = { + {name="v", id=2, kind='scalar', proto_type="int32"}, +} +pb.finalize_message(M.L5_descriptor) +M.L5_fields = pb.field_names({ + v = "v", +}) + +-- EmmyLua / lua-language-server type annotations. +-- These are comments — no runtime effect. They give editors +-- autocomplete and type-checking for the generated wrappers. +--- 5-level chain of singular sub-messages. Used by the bd-hwe (ra6 3d) +--- C-runtime tests to verify recursion across non-trivial depth. Each +--- level carries an int32 so the encoded body at each level is +--- non-empty (catches off-by-one bugs in the length-prefix path). +---@class c_nested.L1 +---@field next c_nested.L2 +---@field v integer + +---@class c_nested.L2 +---@field next c_nested.L3 +---@field v integer + +---@class c_nested.L3 +---@field next c_nested.L4 +---@field v integer + +---@class c_nested.L4 +---@field next c_nested.L5 +---@field v integer + +---@class c_nested.L5 +---@field v integer + +---@param t? c_nested.L1 +---@return c_nested.L1 +function M.L1_new(t) return t or {} end + +---@param t c_nested.L1 +---@return string +function M.L1_encode(t) + if type(t) ~= 'table' then + error("expected table for c_nested.L1, got " .. type(t), 0) + end + local out, n = {}, 0 + local v + -- field 1: next + v = t.next + if v ~= nil or type(v) == 'cdata' then + local _b = M.L2_encode(v) + n = n + 1; out[n] = "\x0a" + local _len = #_b + if _len < 128 then + n = n + 1; out[n] = string.char(_len) + else + n = n + 1; out[n] = wire.encode_varint(_len) + end + n = n + 1; out[n] = _b + end + -- field 2: v + v = t.v + if v ~= nil and v ~= 0 then + n = n + 1; out[n] = "\x10" + n = n + 1; out[n] = wire.encode_int32(v) + end + local _exts = t._extensions + if _exts ~= nil then + local _elist = M.L1_descriptor.extensions_list + if _elist ~= nil then + for _i = 1, #_elist do + local _ext = _elist[_i] + local _ev = _exts[_ext.full_name] + if _ev ~= nil then + pb.codec.encode_field(_ext, _ev, out, true) + end + end + n = #out + end + end + local _uf = t._unknown_fields + if _uf ~= nil and _uf ~= '' then n = n + 1; out[n] = _uf end + return table.concat(out) +end + +---@param b string +---@return c_nested.L1 +function M.L1_decode(buf) + if type(buf) ~= 'string' then + error("expected string for c_nested.L1 decode, got " .. type(buf), 0) + end + local result = {} + local pos, len = 1, #buf + local _uf + while pos <= len do + local _tag_start = pos + local id, wt + local _b = string_byte(buf, pos) + if _b ~= nil and _b < 0x80 then + wt = band(_b, 7) + if wt >= 6 then error("illegal wire type " .. wt, 0) end + id = rshift(_b, 3) + if id == 0 then error("illegal field number 0", 0) end + pos = pos + 1 + else + id, wt, pos = wire.decode_tag(buf, pos) + end + if id == 1 then + local payload + payload, pos = wire.decode_len(buf, pos) + local prev = result.next + if prev == nil then + result.next = M.L2_decode(payload) + else + pb.codec.merge_message(M.L2_descriptor, prev, M.L2_decode(payload)) + end + elseif id == 2 then + local val + val, pos = wire.decode_int32(buf, pos) + result.v = val + else + local _ebid = M.L1_descriptor.extensions_by_id + local _ext = _ebid and _ebid[id] or nil + if _ext ~= nil then + pos = pb.codec.decode_extension(_ext, buf, pos, wt, result) + else + pos = wire.skip_field(buf, pos, wt, id) + if _uf == nil then _uf = {} end + _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) + end + end + end + if _uf ~= nil then result._unknown_fields = table.concat(_uf) end + return result +end + +---@param b string +---@return pb.MessageView +function M.L1_decode_lazy(b) return pb.decode_lazy(M.L1_descriptor, b) end +---@param t c_nested.L1 +---@param opts? {single_line: boolean?, indent: string?} +---@return string +function M.L1_text(t, opts) return pb.text.encode(M.L1_descriptor, t, opts) end + +---@param t? c_nested.L2 +---@return c_nested.L2 +function M.L2_new(t) return t or {} end + +---@param t c_nested.L2 +---@return string +function M.L2_encode(t) + if type(t) ~= 'table' then + error("expected table for c_nested.L2, got " .. type(t), 0) + end + local out, n = {}, 0 + local v + -- field 1: next + v = t.next + if v ~= nil or type(v) == 'cdata' then + local _b = M.L3_encode(v) + n = n + 1; out[n] = "\x0a" + local _len = #_b + if _len < 128 then + n = n + 1; out[n] = string.char(_len) + else + n = n + 1; out[n] = wire.encode_varint(_len) + end + n = n + 1; out[n] = _b + end + -- field 2: v + v = t.v + if v ~= nil and v ~= 0 then + n = n + 1; out[n] = "\x10" + n = n + 1; out[n] = wire.encode_int32(v) + end + local _exts = t._extensions + if _exts ~= nil then + local _elist = M.L2_descriptor.extensions_list + if _elist ~= nil then + for _i = 1, #_elist do + local _ext = _elist[_i] + local _ev = _exts[_ext.full_name] + if _ev ~= nil then + pb.codec.encode_field(_ext, _ev, out, true) + end + end + n = #out + end + end + local _uf = t._unknown_fields + if _uf ~= nil and _uf ~= '' then n = n + 1; out[n] = _uf end + return table.concat(out) +end + +---@param b string +---@return c_nested.L2 +function M.L2_decode(buf) + if type(buf) ~= 'string' then + error("expected string for c_nested.L2 decode, got " .. type(buf), 0) + end + local result = {} + local pos, len = 1, #buf + local _uf + while pos <= len do + local _tag_start = pos + local id, wt + local _b = string_byte(buf, pos) + if _b ~= nil and _b < 0x80 then + wt = band(_b, 7) + if wt >= 6 then error("illegal wire type " .. wt, 0) end + id = rshift(_b, 3) + if id == 0 then error("illegal field number 0", 0) end + pos = pos + 1 + else + id, wt, pos = wire.decode_tag(buf, pos) + end + if id == 1 then + local payload + payload, pos = wire.decode_len(buf, pos) + local prev = result.next + if prev == nil then + result.next = M.L3_decode(payload) + else + pb.codec.merge_message(M.L3_descriptor, prev, M.L3_decode(payload)) + end + elseif id == 2 then + local val + val, pos = wire.decode_int32(buf, pos) + result.v = val + else + local _ebid = M.L2_descriptor.extensions_by_id + local _ext = _ebid and _ebid[id] or nil + if _ext ~= nil then + pos = pb.codec.decode_extension(_ext, buf, pos, wt, result) + else + pos = wire.skip_field(buf, pos, wt, id) + if _uf == nil then _uf = {} end + _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) + end + end + end + if _uf ~= nil then result._unknown_fields = table.concat(_uf) end + return result +end + +---@param b string +---@return pb.MessageView +function M.L2_decode_lazy(b) return pb.decode_lazy(M.L2_descriptor, b) end +---@param t c_nested.L2 +---@param opts? {single_line: boolean?, indent: string?} +---@return string +function M.L2_text(t, opts) return pb.text.encode(M.L2_descriptor, t, opts) end + +---@param t? c_nested.L3 +---@return c_nested.L3 +function M.L3_new(t) return t or {} end + +---@param t c_nested.L3 +---@return string +function M.L3_encode(t) + if type(t) ~= 'table' then + error("expected table for c_nested.L3, got " .. type(t), 0) + end + local out, n = {}, 0 + local v + -- field 1: next + v = t.next + if v ~= nil or type(v) == 'cdata' then + local _b = M.L4_encode(v) + n = n + 1; out[n] = "\x0a" + local _len = #_b + if _len < 128 then + n = n + 1; out[n] = string.char(_len) + else + n = n + 1; out[n] = wire.encode_varint(_len) + end + n = n + 1; out[n] = _b + end + -- field 2: v + v = t.v + if v ~= nil and v ~= 0 then + n = n + 1; out[n] = "\x10" + n = n + 1; out[n] = wire.encode_int32(v) + end + local _exts = t._extensions + if _exts ~= nil then + local _elist = M.L3_descriptor.extensions_list + if _elist ~= nil then + for _i = 1, #_elist do + local _ext = _elist[_i] + local _ev = _exts[_ext.full_name] + if _ev ~= nil then + pb.codec.encode_field(_ext, _ev, out, true) + end + end + n = #out + end + end + local _uf = t._unknown_fields + if _uf ~= nil and _uf ~= '' then n = n + 1; out[n] = _uf end + return table.concat(out) +end + +---@param b string +---@return c_nested.L3 +function M.L3_decode(buf) + if type(buf) ~= 'string' then + error("expected string for c_nested.L3 decode, got " .. type(buf), 0) + end + local result = {} + local pos, len = 1, #buf + local _uf + while pos <= len do + local _tag_start = pos + local id, wt + local _b = string_byte(buf, pos) + if _b ~= nil and _b < 0x80 then + wt = band(_b, 7) + if wt >= 6 then error("illegal wire type " .. wt, 0) end + id = rshift(_b, 3) + if id == 0 then error("illegal field number 0", 0) end + pos = pos + 1 + else + id, wt, pos = wire.decode_tag(buf, pos) + end + if id == 1 then + local payload + payload, pos = wire.decode_len(buf, pos) + local prev = result.next + if prev == nil then + result.next = M.L4_decode(payload) + else + pb.codec.merge_message(M.L4_descriptor, prev, M.L4_decode(payload)) + end + elseif id == 2 then + local val + val, pos = wire.decode_int32(buf, pos) + result.v = val + else + local _ebid = M.L3_descriptor.extensions_by_id + local _ext = _ebid and _ebid[id] or nil + if _ext ~= nil then + pos = pb.codec.decode_extension(_ext, buf, pos, wt, result) + else + pos = wire.skip_field(buf, pos, wt, id) + if _uf == nil then _uf = {} end + _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) + end + end + end + if _uf ~= nil then result._unknown_fields = table.concat(_uf) end + return result +end + +---@param b string +---@return pb.MessageView +function M.L3_decode_lazy(b) return pb.decode_lazy(M.L3_descriptor, b) end +---@param t c_nested.L3 +---@param opts? {single_line: boolean?, indent: string?} +---@return string +function M.L3_text(t, opts) return pb.text.encode(M.L3_descriptor, t, opts) end + +---@param t? c_nested.L4 +---@return c_nested.L4 +function M.L4_new(t) return t or {} end + +---@param t c_nested.L4 +---@return string +function M.L4_encode(t) + if type(t) ~= 'table' then + error("expected table for c_nested.L4, got " .. type(t), 0) + end + local out, n = {}, 0 + local v + -- field 1: next + v = t.next + if v ~= nil or type(v) == 'cdata' then + local _b = M.L5_encode(v) + n = n + 1; out[n] = "\x0a" + local _len = #_b + if _len < 128 then + n = n + 1; out[n] = string.char(_len) + else + n = n + 1; out[n] = wire.encode_varint(_len) + end + n = n + 1; out[n] = _b + end + -- field 2: v + v = t.v + if v ~= nil and v ~= 0 then + n = n + 1; out[n] = "\x10" + n = n + 1; out[n] = wire.encode_int32(v) + end + local _exts = t._extensions + if _exts ~= nil then + local _elist = M.L4_descriptor.extensions_list + if _elist ~= nil then + for _i = 1, #_elist do + local _ext = _elist[_i] + local _ev = _exts[_ext.full_name] + if _ev ~= nil then + pb.codec.encode_field(_ext, _ev, out, true) + end + end + n = #out + end + end + local _uf = t._unknown_fields + if _uf ~= nil and _uf ~= '' then n = n + 1; out[n] = _uf end + return table.concat(out) +end + +---@param b string +---@return c_nested.L4 +function M.L4_decode(buf) + if type(buf) ~= 'string' then + error("expected string for c_nested.L4 decode, got " .. type(buf), 0) + end + local result = {} + local pos, len = 1, #buf + local _uf + while pos <= len do + local _tag_start = pos + local id, wt + local _b = string_byte(buf, pos) + if _b ~= nil and _b < 0x80 then + wt = band(_b, 7) + if wt >= 6 then error("illegal wire type " .. wt, 0) end + id = rshift(_b, 3) + if id == 0 then error("illegal field number 0", 0) end + pos = pos + 1 + else + id, wt, pos = wire.decode_tag(buf, pos) + end + if id == 1 then + local payload + payload, pos = wire.decode_len(buf, pos) + local prev = result.next + if prev == nil then + result.next = M.L5_decode(payload) + else + pb.codec.merge_message(M.L5_descriptor, prev, M.L5_decode(payload)) + end + elseif id == 2 then + local val + val, pos = wire.decode_int32(buf, pos) + result.v = val + else + local _ebid = M.L4_descriptor.extensions_by_id + local _ext = _ebid and _ebid[id] or nil + if _ext ~= nil then + pos = pb.codec.decode_extension(_ext, buf, pos, wt, result) + else + pos = wire.skip_field(buf, pos, wt, id) + if _uf == nil then _uf = {} end + _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) + end + end + end + if _uf ~= nil then result._unknown_fields = table.concat(_uf) end + return result +end + +---@param b string +---@return pb.MessageView +function M.L4_decode_lazy(b) return pb.decode_lazy(M.L4_descriptor, b) end +---@param t c_nested.L4 +---@param opts? {single_line: boolean?, indent: string?} +---@return string +function M.L4_text(t, opts) return pb.text.encode(M.L4_descriptor, t, opts) end + +---@param t? c_nested.L5 +---@return c_nested.L5 +function M.L5_new(t) return t or {} end + +---@param t c_nested.L5 +---@return string +function M.L5_encode(t) + if type(t) ~= 'table' then + error("expected table for c_nested.L5, got " .. type(t), 0) + end + local out, n = {}, 0 + local v + -- field 2: v + v = t.v + if v ~= nil and v ~= 0 then + n = n + 1; out[n] = "\x10" + n = n + 1; out[n] = wire.encode_int32(v) + end + local _exts = t._extensions + if _exts ~= nil then + local _elist = M.L5_descriptor.extensions_list + if _elist ~= nil then + for _i = 1, #_elist do + local _ext = _elist[_i] + local _ev = _exts[_ext.full_name] + if _ev ~= nil then + pb.codec.encode_field(_ext, _ev, out, true) + end + end + n = #out + end + end + local _uf = t._unknown_fields + if _uf ~= nil and _uf ~= '' then n = n + 1; out[n] = _uf end + return table.concat(out) +end + +---@param b string +---@return c_nested.L5 +function M.L5_decode(buf) + if type(buf) ~= 'string' then + error("expected string for c_nested.L5 decode, got " .. type(buf), 0) + end + local result = {} + local pos, len = 1, #buf + local _uf + while pos <= len do + local _tag_start = pos + local id, wt + local _b = string_byte(buf, pos) + if _b ~= nil and _b < 0x80 then + wt = band(_b, 7) + if wt >= 6 then error("illegal wire type " .. wt, 0) end + id = rshift(_b, 3) + if id == 0 then error("illegal field number 0", 0) end + pos = pos + 1 + else + id, wt, pos = wire.decode_tag(buf, pos) + end + if id == 2 then + local val + val, pos = wire.decode_int32(buf, pos) + result.v = val + else + local _ebid = M.L5_descriptor.extensions_by_id + local _ext = _ebid and _ebid[id] or nil + if _ext ~= nil then + pos = pb.codec.decode_extension(_ext, buf, pos, wt, result) + else + pos = wire.skip_field(buf, pos, wt, id) + if _uf == nil then _uf = {} end + _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) + end + end + end + if _uf ~= nil then result._unknown_fields = table.concat(_uf) end + return result +end + +---@param b string +---@return pb.MessageView +function M.L5_decode_lazy(b) return pb.decode_lazy(M.L5_descriptor, b) end +---@param t c_nested.L5 +---@param opts? {single_line: boolean?, indent: string?} +---@return string +function M.L5_text(t, opts) return pb.text.encode(M.L5_descriptor, t, opts) end + +return M diff --git a/examples/expected/runtime/c_nested/c_nested_pb.lua b/examples/expected/runtime/c_nested/c_nested_pb.lua new file mode 100644 index 0000000000000000000000000000000000000000..17bece7ae171c163c06cf38c278a1ca1e1b57491 --- /dev/null +++ b/examples/expected/runtime/c_nested/c_nested_pb.lua @@ -0,0 +1,187 @@ +-- Code generated by protoc-gen-tarantool. DO NOT EDIT. +-- source: c_nested.proto +-- syntax: proto3 +-- package: c_nested + +local pb = require("pb") +local wire = pb.wire +local string_byte = string.byte +local band = bit.band +local rshift = bit.rshift + +local M = {} + +M.options = {go_package = "tarantoolpb_synthetic/c_nested"} + +-- Pre-declare message descriptors so cross-references resolve. +M.L1_descriptor = {name = "c_nested.L1"} +M.L2_descriptor = {name = "c_nested.L2"} +M.L3_descriptor = {name = "c_nested.L3"} +M.L4_descriptor = {name = "c_nested.L4"} +M.L5_descriptor = {name = "c_nested.L5"} + +-- Message: c_nested.L1 +M.L1_descriptor.fields = { + {name="next", id=1, kind='message', message=M.L2_descriptor}, + {name="v", id=2, kind='scalar', proto_type="int32"}, +} +pb.finalize_message(M.L1_descriptor) +M.L1_fields = pb.field_names({ + next = "next", + v = "v", +}) + +-- Message: c_nested.L2 +M.L2_descriptor.fields = { + {name="next", id=1, kind='message', message=M.L3_descriptor}, + {name="v", id=2, kind='scalar', proto_type="int32"}, +} +pb.finalize_message(M.L2_descriptor) +M.L2_fields = pb.field_names({ + next = "next", + v = "v", +}) + +-- Message: c_nested.L3 +M.L3_descriptor.fields = { + {name="next", id=1, kind='message', message=M.L4_descriptor}, + {name="v", id=2, kind='scalar', proto_type="int32"}, +} +pb.finalize_message(M.L3_descriptor) +M.L3_fields = pb.field_names({ + next = "next", + v = "v", +}) + +-- Message: c_nested.L4 +M.L4_descriptor.fields = { + {name="next", id=1, kind='message', message=M.L5_descriptor}, + {name="v", id=2, kind='scalar', proto_type="int32"}, +} +pb.finalize_message(M.L4_descriptor) +M.L4_fields = pb.field_names({ + next = "next", + v = "v", +}) + +-- Message: c_nested.L5 +M.L5_descriptor.fields = { + {name="v", id=2, kind='scalar', proto_type="int32"}, +} +pb.finalize_message(M.L5_descriptor) +M.L5_fields = pb.field_names({ + v = "v", +}) + +-- EmmyLua / lua-language-server type annotations. +-- These are comments — no runtime effect. They give editors +-- autocomplete and type-checking for the generated wrappers. +--- 5-level chain of singular sub-messages. Used by the bd-hwe (ra6 3d) +--- C-runtime tests to verify recursion across non-trivial depth. Each +--- level carries an int32 so the encoded body at each level is +--- non-empty (catches off-by-one bugs in the length-prefix path). +---@class c_nested.L1 +---@field next c_nested.L2 +---@field v integer + +---@class c_nested.L2 +---@field next c_nested.L3 +---@field v integer + +---@class c_nested.L3 +---@field next c_nested.L4 +---@field v integer + +---@class c_nested.L4 +---@field next c_nested.L5 +---@field v integer + +---@class c_nested.L5 +---@field v integer + +---@param t? c_nested.L1 +---@return c_nested.L1 +function M.L1_new(t) return t or {} end +---@param t c_nested.L1 +---@return string +function M.L1_encode(t) return pb.encode(M.L1_descriptor, t) end +---@param b string +---@return c_nested.L1 +function M.L1_decode(b) return pb.decode(M.L1_descriptor, b) end +---@param b string +---@return pb.MessageView +function M.L1_decode_lazy(b) return pb.decode_lazy(M.L1_descriptor, b) end +---@param t c_nested.L1 +---@param opts? {single_line: boolean?, indent: string?} +---@return string +function M.L1_text(t, opts) return pb.text.encode(M.L1_descriptor, t, opts) end + +---@param t? c_nested.L2 +---@return c_nested.L2 +function M.L2_new(t) return t or {} end +---@param t c_nested.L2 +---@return string +function M.L2_encode(t) return pb.encode(M.L2_descriptor, t) end +---@param b string +---@return c_nested.L2 +function M.L2_decode(b) return pb.decode(M.L2_descriptor, b) end +---@param b string +---@return pb.MessageView +function M.L2_decode_lazy(b) return pb.decode_lazy(M.L2_descriptor, b) end +---@param t c_nested.L2 +---@param opts? {single_line: boolean?, indent: string?} +---@return string +function M.L2_text(t, opts) return pb.text.encode(M.L2_descriptor, t, opts) end + +---@param t? c_nested.L3 +---@return c_nested.L3 +function M.L3_new(t) return t or {} end +---@param t c_nested.L3 +---@return string +function M.L3_encode(t) return pb.encode(M.L3_descriptor, t) end +---@param b string +---@return c_nested.L3 +function M.L3_decode(b) return pb.decode(M.L3_descriptor, b) end +---@param b string +---@return pb.MessageView +function M.L3_decode_lazy(b) return pb.decode_lazy(M.L3_descriptor, b) end +---@param t c_nested.L3 +---@param opts? {single_line: boolean?, indent: string?} +---@return string +function M.L3_text(t, opts) return pb.text.encode(M.L3_descriptor, t, opts) end + +---@param t? c_nested.L4 +---@return c_nested.L4 +function M.L4_new(t) return t or {} end +---@param t c_nested.L4 +---@return string +function M.L4_encode(t) return pb.encode(M.L4_descriptor, t) end +---@param b string +---@return c_nested.L4 +function M.L4_decode(b) return pb.decode(M.L4_descriptor, b) end +---@param b string +---@return pb.MessageView +function M.L4_decode_lazy(b) return pb.decode_lazy(M.L4_descriptor, b) end +---@param t c_nested.L4 +---@param opts? {single_line: boolean?, indent: string?} +---@return string +function M.L4_text(t, opts) return pb.text.encode(M.L4_descriptor, t, opts) end + +---@param t? c_nested.L5 +---@return c_nested.L5 +function M.L5_new(t) return t or {} end +---@param t c_nested.L5 +---@return string +function M.L5_encode(t) return pb.encode(M.L5_descriptor, t) end +---@param b string +---@return c_nested.L5 +function M.L5_decode(b) return pb.decode(M.L5_descriptor, b) end +---@param b string +---@return pb.MessageView +function M.L5_decode_lazy(b) return pb.decode_lazy(M.L5_descriptor, b) end +---@param t c_nested.L5 +---@param opts? {single_line: boolean?, indent: string?} +---@return string +function M.L5_text(t, opts) return pb.text.encode(M.L5_descriptor, t, opts) end + +return M diff --git a/runtime/pb/c/c_runtime.c b/runtime/pb/c/c_runtime.c index ddd282786c8051da37e068113c3847a4ad0fbb29..f82aa520416edbd4a6c6d816268c335567dfbbdd 100644 --- a/runtime/pb/c/c_runtime.c +++ b/runtime/pb/c/c_runtime.c @@ -775,16 +775,23 @@ plan_sub_plan(lua_State *L) } /* ---------------------------------------------------------------- * - * Encode (bd-y1n / ra6 3b): singular scalar fields. * + * Encode (bd-y1n / ra6 3b, bd-hwe / ra6 3d). * * * - * Repeated/packed, map, and message-typed fields are skipped — * - * those land in 3d (sub-message) and 3e (repeated/packed). * + * Singular scalars (3b) and singular sub-messages (3d). Repeated/ * + * packed and map fields are silently skipped — 3e / 3h land them. * * * * Buffer strategy: a 4KB stack-backed scratch buffer that promotes * * to a Lua userdata (GC'd automatically) on overflow. Using * * `lua_newuserdata` for heap growth means a luaL_error mid-encode * * doesn't leak — the userdata is still on the stack at the unwind * * point and gets collected normally. * + * * + * Recursion shape: encode_body is called once per message; sub- * + * message fields recurse by allocating a fresh enc_buf on the C * + * stack and re-entering encode_body with the sub-plan. The parent's * + * heap_idx is force-established before recursing so it survives * + * the lua_settop cleanup at sub-encode exit (see * + * encode_submessage_field for the contract). * * ---------------------------------------------------------------- */ #define ENC_STACK_BUF 4096 @@ -1093,38 +1100,83 @@ encode_one_field(lua_State *L, enc_buf *b, pb_plan_field *f, int val_idx) } } -static int -encode_lua(lua_State *L) +/* Forward decls for the recursive encode pair. */ +static void encode_body(lua_State *L, enc_buf *b, pb_plan *plan, int msg_idx); + +/* Encode one singular sub-message field into the parent buffer `b`. + * Lifecycle / stack-management contract: + * - Parent's heap is force-established (one 8KB grow) BEFORE recursing + * so b->heap_idx is below the saved_top. This way the final + * ebuf_reserve on the parent can only either re-use b->heap_idx via + * lua_replace (no new stack slot) or — if no further grow is + * needed — leave the stack alone. Either way, lua_settop(L, + * saved_top) at the end is safe. + * - sub-buf is a fresh stack-backed enc_buf; its potential heap + * userdata is on the Lua stack above saved_top and is dropped by + * the lua_settop. */ +static void +encode_submessage_field(lua_State *L, enc_buf *b, pb_plan *plan, + pb_plan_field *f, int val_idx) { - pb_plan *plan = (pb_plan *)luaL_checkudata(L, 1, PB_PLAN_MT); - luaL_checktype(L, 2, LUA_TTABLE); - int msg_idx = 2; + if (lua_type(L, val_idx) != LUA_TTABLE) + luaL_error(L, "message field requires a table value"); + if (plan->sub_plans_ref == LUA_NOREF) + luaL_error(L, "plan '%s' has no sub-plans table", + plan->name != NULL ? plan->name : "?"); + + /* Force parent's heap to exist before the sub-encode allocates. */ + if (b->heap_idx == 0) + ebuf_grow(L, b, 1); + + val_idx = abs_idx(L, val_idx); + int saved_top = lua_gettop(L); + + lua_rawgeti(L, LUA_REGISTRYINDEX, plan->sub_plans_ref); + lua_rawgeti(L, -1, f->sub_plan_idx); + pb_plan *subplan = (pb_plan *)lua_touserdata(L, -1); + if (subplan == NULL) + luaL_error(L, "sub-plan at index %d is not a userdata", + f->sub_plan_idx); + + enc_buf sub; + ebuf_init(&sub); + encode_body(L, &sub, subplan, val_idx); + + /* Write tag + length-varint + body into parent. Parent regrowth + * here goes through lua_replace at b->heap_idx (safely below + * saved_top) — no stack-frame disruption. */ + ebuf_reserve(L, b, f->tag_len + 10 + sub.used); + ebuf_put_tag(b, f); + ebuf_put_varint(b, (uint64_t)sub.used); + if (sub.used > 0) + ebuf_put_bytes(b, ebuf_base(&sub), sub.used); + + lua_settop(L, saved_top); +} + +static void +encode_body(lua_State *L, enc_buf *b, pb_plan *plan, int msg_idx) +{ + msg_idx = abs_idx(L, msg_idx); if (plan->has_override) { - return luaL_error(L, + luaL_error(L, "C encode on plan with override is unsupported (bd-rmf)"); } - /* Pin the field-names table on the stack for the loop's rawgeti. */ - if (plan->field_names_ref == LUA_NOREF || plan->n_fields == 0) { - lua_pushliteral(L, ""); - return 1; - } + if (plan->field_names_ref == LUA_NOREF || plan->n_fields == 0) + return; + lua_rawgeti(L, LUA_REGISTRYINDEX, plan->field_names_ref); int names_idx = lua_gettop(L); - enc_buf b; - ebuf_init(&b); - for (int i = 0; i < plan->n_fields; i++) { pb_plan_field *f = &plan->fields[i]; - /* 3b scope: singular scalar/enum/string/bytes. Repeated, map, - * and message-typed fields are silently skipped — they land - * in 3d/3e and will overwrite this short-circuit. */ + /* Scope for now: singular scalar/enum/string/bytes/message. + * Repeated and map are 3e/3h territory — still skipped. */ if (f->repeated) continue; - if (f->kind == PB_KIND_MAP || f->kind == PB_KIND_MESSAGE) - continue; + if (f->kind == PB_KIND_MAP) continue; lua_rawgeti(L, names_idx, i + 1); /* push field name */ lua_rawget(L, msg_idx); /* push msg[name] */ @@ -1135,27 +1187,55 @@ encode_lua(lua_State *L) continue; } - encode_one_field(L, &b, f, val_idx); + if (f->kind == PB_KIND_MESSAGE) { + encode_submessage_field(L, b, plan, f, val_idx); + } else { + encode_one_field(L, b, f, val_idx); + } lua_pop(L, 1); } + lua_pop(L, 1); /* names table */ +} + +static int +encode_lua(lua_State *L) +{ + pb_plan *plan = (pb_plan *)luaL_checkudata(L, 1, PB_PLAN_MT); + luaL_checktype(L, 2, LUA_TTABLE); + + if (plan->has_override) { + return luaL_error(L, + "C encode on plan with override is unsupported (bd-rmf)"); + } + + enc_buf b; + ebuf_init(&b); + encode_body(L, &b, plan, 2); + lua_pushlstring(L, (const char *)ebuf_base(&b), b.used); return 1; } /* ---------------------------------------------------------------- * - * Decode (bd-mz6 / ra6 3c): singular scalar fields. * + * Decode (bd-mz6 / ra6 3c, bd-hwe / ra6 3d). * * * - * Symmetric scope with 3b. Repeated/map/message-typed fields are * - * skipped over by wire-type — 3d/3e/3i will overwrite this with * - * real handling. Unknown tags are skipped silently too; bd-wyp * - * will add round-trip capture. * + * Singular scalars (3c) and singular sub-messages (3d). Repeated * + * and map fields are skipped by wire type — 3e / 3h will land * + * them. Unknown tags are skipped silently too; bd-wyp will add * + * round-trip capture. * * * * Result-table shape must match mode=full pure-Lua decode. That * * pins per-kind value types: int64/uint64/sint64/fixed64/sfixed64 * * push Tarantool int64_t/uint64_t cdata via luaL_pushint64 / * * luaL_pushuint64; everything else pushes a Lua number, boolean, * * or string. See runtime/pb/wire.lua decoder comments. * + * * + * Nested-message decode bounds the byte range by temporarily * + * shrinking c->len to the sub-message's end offset; the wire-prim * + * helpers (dec_varint / dec_fixed*) already bounds-check against * + * c->len, so this keeps a malformed inner payload from over- * + * reading into the outer message's bytes. * * ---------------------------------------------------------------- */ typedef struct dec_ctx { @@ -1348,37 +1428,74 @@ dec_push_one(dec_ctx *c, pb_plan_field *f) } } -static int -decode_lua(lua_State *L) +/* Forward decl for the recursive decode. */ +static void decode_body(dec_ctx *c, pb_plan *plan, int result_idx); +static void decode_submessage_field(dec_ctx *c, pb_plan_field *f, + int sub_plans_idx); + +/* Decode one singular sub-message field. On entry, `c->pos` points at + * the length-varint byte; on exit, `c->pos == c->pos + plen`. Pushes + * the decoded sub-table onto the Lua stack. */ +static void +decode_submessage_field(dec_ctx *c, pb_plan_field *f, int sub_plans_idx) { - pb_plan *plan = (pb_plan *)luaL_checkudata(L, 1, PB_PLAN_MT); - size_t buf_len; - const char *buf = luaL_checklstring(L, 2, &buf_len); + lua_State *L = c->L; + uint64_t plen = dec_varint(c); + if (c->len - c->pos < plen) + luaL_error(L, "truncated nested message at offset %d", + (int)c->pos); + + lua_rawgeti(L, sub_plans_idx, f->sub_plan_idx); + pb_plan *subplan = (pb_plan *)lua_touserdata(L, -1); + if (subplan == NULL) + luaL_error(L, "sub-plan at index %d is not a userdata", + f->sub_plan_idx); + lua_pop(L, 1); + lua_createtable(L, 0, subplan->n_fields); + int sub_result_idx = lua_gettop(L); + + /* Temporarily shrink c->len so the inner decode loop terminates at + * the sub-message boundary and so inner wire-prim reads cannot + * spill past it. */ + size_t saved_len = c->len; + c->len = c->pos + (size_t)plen; + decode_body(c, subplan, sub_result_idx); + if (c->pos != c->len) + luaL_error(L, + "nested message body underflow at offset %d (expected %d)", + (int)c->pos, (int)c->len); + c->len = saved_len; +} + +static void +decode_body(dec_ctx *c, pb_plan *plan, int result_idx) +{ + lua_State *L = c->L; if (plan->has_override) { - return luaL_error(L, + luaL_error(L, "C decode on plan with override is unsupported (bd-rmf)"); } - /* Pre-size the result table: at most n_fields singular entries. */ - lua_createtable(L, 0, plan->n_fields); - int result_idx = lua_gettop(L); + result_idx = abs_idx(L, result_idx); - /* Pin field-names so the loop can rawgeti by 1-based field index. */ - if (plan->field_names_ref != LUA_NOREF) + /* Pin field-names + sub-plans so the loop can rawgeti by index. */ + int names_idx, sub_plans_idx; + if (plan->field_names_ref != LUA_NOREF) { lua_rawgeti(L, LUA_REGISTRYINDEX, plan->field_names_ref); - else + } else { lua_pushnil(L); - int names_idx = lua_gettop(L); - - dec_ctx c; - c.L = L; - c.buf = (const uint8_t *)buf; - c.len = buf_len; - c.pos = 0; + } + names_idx = lua_gettop(L); + if (plan->sub_plans_ref != LUA_NOREF) { + lua_rawgeti(L, LUA_REGISTRYINDEX, plan->sub_plans_ref); + } else { + lua_pushnil(L); + } + sub_plans_idx = lua_gettop(L); - while (c.pos < c.len) { - uint64_t tag = dec_varint(&c); + while (c->pos < c->len) { + uint64_t tag = dec_varint(c); uint32_t field_number = (uint32_t)(tag >> 3); uint8_t wt = (uint8_t)(tag & 0x07); @@ -1392,29 +1509,56 @@ decode_lua(lua_State *L) } } - /* Out-of-scope shapes for 3c: repeated, map, message. Skip - * by wire type so 3d/3e/3i can replace this branch later. */ - if (f == NULL - || f->repeated - || f->kind == PB_KIND_MAP - || f->kind == PB_KIND_MESSAGE) { - dec_skip(&c, wt); + /* Out-of-scope shapes for current bd-hwe: repeated, map. + * 3e / 3h will replace this skip. */ + if (f == NULL || f->repeated || f->kind == PB_KIND_MAP) { + dec_skip(c, wt); + continue; + } + + if (f->kind == PB_KIND_MESSAGE) { + decode_submessage_field(c, f, sub_plans_idx); + /* stack: ..., names, sub_plans, sub_result */ + int field_idx_1based = (int)(f - plan->fields) + 1; + lua_rawgeti(L, names_idx, field_idx_1based); + lua_insert(L, -2); /* name, sub_result */ + lua_rawset(L, result_idx); /* result[name] = sub_result */ continue; } - dec_push_one(&c, f); /* stack: ..., names, value */ + dec_push_one(c, f); /* stack: ..., names, sub_plans, value */ - /* result[name] = value. Field index within plan = pointer - * arithmetic against plan->fields[0]; +1 for the 1-based - * field_names_ref convention used elsewhere. */ int field_idx_1based = (int)(f - plan->fields) + 1; - lua_rawgeti(L, names_idx, field_idx_1based); /* push name */ - lua_insert(L, -2); /* name, value */ - lua_rawset(L, result_idx); /* result[name] = value */ + lua_rawgeti(L, names_idx, field_idx_1based); + lua_insert(L, -2); /* name, value */ + lua_rawset(L, result_idx); /* result[name] = value */ } - /* Drop names ref; leave result on top. */ - lua_pop(L, 1); + lua_pop(L, 2); /* sub_plans, names */ +} + +static int +decode_lua(lua_State *L) +{ + pb_plan *plan = (pb_plan *)luaL_checkudata(L, 1, PB_PLAN_MT); + size_t buf_len; + const char *buf = luaL_checklstring(L, 2, &buf_len); + + if (plan->has_override) { + return luaL_error(L, + "C decode on plan with override is unsupported (bd-rmf)"); + } + + lua_createtable(L, 0, plan->n_fields); + int result_idx = lua_gettop(L); + + dec_ctx c; + c.L = L; + c.buf = (const uint8_t *)buf; + c.len = buf_len; + c.pos = 0; + + decode_body(&c, plan, result_idx); return 1; } diff --git a/test/c_runtime_decode_test.lua b/test/c_runtime_decode_test.lua index 999b8d28a2183eb248294925dec3bb629d92c0c3..5a04bade2cce8794dd0ad73f56165ccc03375a66 100644 --- a/test/c_runtime_decode_test.lua +++ b/test/c_runtime_decode_test.lua @@ -129,23 +129,76 @@ for _, mode in ipairs({'full', 'runtime'}) do t.assert_equals(c_decoded.avatar, '\x00\x01\xff\xfe') end - function g.test_skips_repeated_and_message_tags() - -- 3c scope: repeated/map/message tags are skipped over by wire - -- type rather than decoded (3d/3e/3i extend this). The result - -- table contains only the singular scalar fields, matching what - -- a 3b-encoded input would have produced anyway. + function g.test_skips_repeated_and_map_tags() + -- 3d scope: singular message decoded into a sub-table. + -- Repeated and map remain skipped (3e/3h replace this). local plan = c_runtime.compile_plan(hello.Person_descriptor) - -- Build bytes via the full Lua codec, then decode through C. local rich = { name = 'x', - emails = {'a@b'}, -- repeated string - address = {street = 'Main'}, -- message - lucky_numbers = {1, 2, 3}, -- repeated packed - ages_by_nickname = {alice = 30}, -- map + emails = {'a@b'}, -- repeated string (skip) + address = {street = 'Main'}, -- message (decode) + lucky_numbers = {1, 2, 3}, -- repeated packed (skip) + ages_by_nickname = {alice = 30}, -- map (skip) } local bytes = full_hello.Person_encode(rich) local c_decoded = c_runtime.decode(plan, bytes) - t.assert_equals(c_decoded, {name = 'x'}) + t.assert_equals(c_decoded, {name = 'x', address = {street = 'Main'}}) + end + + -- ---------- Sub-message decode (bd-hwe / ra6 3d) ---------- + + function g.test_singular_submessage_round_trip() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = { + name = 'x', + address = {street = 'Main', city = 'Springfield', zip = 100}, + } + local bytes = full_hello.Person_encode(msg) + local c_decoded = c_runtime.decode(plan, bytes) + local lua_decoded = full_hello.Person_decode(bytes) + t.assert_equals(c_decoded, lua_decoded) + t.assert_equals(c_decoded.address.zip, 100) + end + + function g.test_empty_submessage_decodes_to_empty_table() + -- The Lua codec emits tag + length(0) for an empty sub-message + -- table; the C decoder must surface it as {} (not nil). + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local bytes = full_hello.Person_encode({address = {}}) + local c_decoded = c_runtime.decode(plan, bytes) + t.assert_equals(type(c_decoded.address), 'table') + t.assert_equals(c_decoded.address, {}) + end + + function g.test_submessage_with_proto3_optional() + -- Address.apartment='' must survive the round trip. + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local bytes = full_hello.Person_encode({address = {apartment = ''}}) + local c_decoded = c_runtime.decode(plan, bytes) + t.assert_equals(c_decoded.address.apartment, '') + end + + function g.test_nested_5_levels() + local cn = require(mode .. '.c_nested.c_nested_pb') + local full_cn = require('full.c_nested.c_nested_pb') + local msg = { + v = 1, + next = {v = 2, next = {v = 3, next = {v = 4, + next = {v = 5}}}}, + } + local plan = c_runtime.compile_plan(cn.L1_descriptor) + local bytes = full_cn.L1_encode(msg) + t.assert_equals(c_runtime.decode(plan, bytes), + full_cn.L1_decode(bytes)) + end + + function g.test_truncated_submessage_errors() + -- Tag for address (field 5, wire 2 = 0x2a), length-prefix=10, + -- but actual body shorter than advertised. + local plan = c_runtime.compile_plan(hello.Person_descriptor) + t.assert_error_msg_contains('truncated', function() + c_runtime.decode(plan, '\x2a\x0a\x0a\x03') + end) end function g.test_skips_unknown_field_tags() diff --git a/test/c_runtime_encode_test.lua b/test/c_runtime_encode_test.lua index 669817033581357d9762e0b99c84fb738f89eab1..fc844e0c9442591cfb64fb0db8efa8efa0bbdb99 100644 --- a/test/c_runtime_encode_test.lua +++ b/test/c_runtime_encode_test.lua @@ -144,21 +144,90 @@ for _, mode in ipairs({'full', 'runtime'}) do full_hello.Person_encode(msg)) end - function g.test_skips_repeated_and_message_fields() - -- 3b scope: repeated/map/message silently skipped (3d/3e will - -- replace this). Singular fields still encode. + function g.test_skips_repeated_and_map_fields() + -- 3d scope: singular message now encoded. Repeated and map + -- are still silently skipped — 3e/3h will replace this. local plan = c_runtime.compile_plan(hello.Person_descriptor) local msg = { name = 'x', - emails = {'a@b'}, -- repeated string - address = {street = 'Main'}, -- message - lucky_numbers = {1, 2, 3}, -- repeated packed - ages_by_nickname = {alice = 30}, -- map + emails = {'a@b'}, -- repeated string (skip) + address = {street = 'Main'}, -- message (encode) + lucky_numbers = {1, 2, 3}, -- repeated packed (skip) + ages_by_nickname = {alice = 30}, -- map (skip) } - local expected = full_hello.Person_encode({name = 'x'}) + local expected = full_hello.Person_encode( + {name = 'x', address = {street = 'Main'}}) t.assert_equals(c_runtime.encode(plan, msg), expected) end + -- ---------- Sub-message encode (bd-hwe / ra6 3d) ---------- + + function g.test_singular_submessage_round_trip() + -- Person.address (one-level singular sub-message) round-trip. + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = { + name = 'x', + address = {street = 'Main', city = 'Springfield', zip = 100}, + } + t.assert_equals(c_runtime.encode(plan, msg), + full_hello.Person_encode(msg)) + end + + function g.test_empty_submessage_emits_tag_with_zero_length() + -- Singular sub-message with an empty table — proto3 presence + -- semantics require emitting tag + length(0). The Lua codec + -- does this; the C codec must match. + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = {name = 'x', address = {}} + t.assert_equals(c_runtime.encode(plan, msg), + full_hello.Person_encode(msg)) + end + + function g.test_submessage_with_proto3_optional_field() + -- Address.apartment is proto3-optional. Encode through the + -- sub-message must surface the empty string just like the + -- top-level Address codec does. + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = {address = {apartment = ''}} + t.assert_equals(c_runtime.encode(plan, msg), + full_hello.Person_encode(msg)) + end + + function g.test_submessage_non_table_value_errors() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + t.assert_error_msg_contains('table', function() + c_runtime.encode(plan, {address = 'not a table'}) + end) + end + + function g.test_nested_5_levels() + -- 5-level chain of singular sub-messages. Exercises the + -- recursion depth and the cycle-breaking c_plan stash. + local cn = require(mode .. '.c_nested.c_nested_pb') + local full_cn = require('full.c_nested.c_nested_pb') + local msg = { + v = 1, + next = {v = 2, next = {v = 3, next = {v = 4, + next = {v = 5}}}}, + } + local plan = c_runtime.compile_plan(cn.L1_descriptor) + t.assert_equals(c_runtime.encode(plan, msg), + full_cn.L1_encode(msg)) + end + + function g.test_large_submessage_triggers_parent_grow() + -- Parent buffer starts at 4KB stack-backed; force a parent + -- regrow path during the post-recursion ebuf_reserve by + -- packing a >4KB string into the sub-message. + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = { + name = 'x', + address = {street = string.rep('s', 8192)}, + } + t.assert_equals(c_runtime.encode(plan, msg), + full_hello.Person_encode(msg)) + end + function g.test_long_string_grows_buffer() -- Stack buffer is 4KB; force the heap-promotion path with a -- string that pushes past it. diff --git a/test/proto/c_nested.proto b/test/proto/c_nested.proto new file mode 100644 index 0000000000000000000000000000000000000000..4e5f5563ceae68118024d6a8dcb1ee2866ae3b89 --- /dev/null +++ b/test/proto/c_nested.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package c_nested; + +// 5-level chain of singular sub-messages. Used by the bd-hwe (ra6 3d) +// C-runtime tests to verify recursion across non-trivial depth. Each +// level carries an int32 so the encoded body at each level is +// non-empty (catches off-by-one bugs in the length-prefix path). +message L1 { L2 next = 1; int32 v = 2; } +message L2 { L3 next = 1; int32 v = 2; } +message L3 { L4 next = 1; int32 v = 2; } +message L4 { L5 next = 1; int32 v = 2; } +message L5 { int32 v = 2; }