M .beads/issues.jsonl => .beads/issues.jsonl +1 -1
@@ 17,7 17,7 @@
{"_type":"issue","id":"tarantool-protobuf-74c","title":"LSP: annotate codec/grpc/json/wkt with EmmyLua types","description":"Phase 2 of the LSP / LLM affordance work (phase 1 landed in commit 8721548 — .luarc.json, runtime/pb/_types.lua, init.lua + lazy.lua annotations).\n\nAdd ---@param / ---@return annotations to the remaining public-surface modules so editor hover and LLM context show types beyond just init.lua and the lazy views:\n\n- runtime/pb/codec.lua — encode(desc, t) / decode(desc, b); compile_writers / compile_readers (called from pb.finalize_message)\n- runtime/pb/grpc.lua — loopback / multiplex transport factories; the service client (factory(transport) -\u003e client_methods) and server (impl -\u003e {service, methods}) shapes\n- runtime/pb/json.lua — pb.json.encode(desc, t, opts) / pb.json.decode(desc, s, opts); opts shape (preserve_proto_field_names, emit_unpopulated, ...)\n- runtime/pb/wkt.lua — register(full_name, desc), lookup, any_pack/any_unpack, NULL sentinel\n\nFoundational @class declarations (pb.Descriptor, pb.Field, pb.GrpcTransport, pb.Json, pb.Wkt, pb.Module) already live in runtime/pb/_types.lua — extend as needed for opts shapes / map\u003cK,V\u003e generics.\n\nPure metadata change — no runtime behavior. Verify by running just test (must stay at 748 passing) and by opening a generated _pb.lua in an LSP-aware editor and confirming hover on require('pb').encode / .grpc.loopback / .json.encode shows the right signatures.","acceptance_criteria":"just test still green (748+ tests); hover in lua-language-server-aware editor shows typed signatures for pb.encode, pb.decode, grpc.loopback, json.encode, json.decode, wkt.register","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-23T09:29:55Z","created_by":"Eugene Blikh","updated_at":"2026-05-23T09:29:55Z","labels":["docs","lsp"],"dependency_count":0,"dependent_count":1,"comment_count":0}
{"_type":"issue","id":"tarantool-protobuf-rmf","title":"ra6 3k: WKT override-hook passthrough","description":"If a plan's descriptor has desc.encode and desc.decode set (the WKT pattern from runtime/pb/wkt.lua), the C runtime must call those Lua functions instead of walking fields. Plan compiler stores luaL_ref to those functions; encode/decode entry points check first. This is also the extension point for future per-message codegen C (c0i) — same hook. Depends on 3a only (just adds a check at entry to the encode/decode loop). Acceptance: hello.Event (uses Timestamp, Duration, Any, FieldMask, Wrappers, Struct) round-trips with PB_ENABLE_C=1 producing byte-equal output to mode=full; the WKT module is unmodified.","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:21:13Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:21:13Z","dependencies":[{"issue_id":"tarantool-protobuf-rmf","depends_on_id":"tarantool-protobuf-mq7","type":"blocks","created_at":"2026-05-18T23:21:55Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0}
{"_type":"issue","id":"tarantool-protobuf-m7u","title":"ra6 3i: proto2 — required, defaults, groups, extensions","description":"Proto2 semantic differences from proto3 that the C runtime must honor: (1) Required fields: decode-time enforcement when explicitly opted in (or default to skipping per existing pure-Lua behavior — match exactly); encode-time enforcement same. (2) Field defaults: present in the plan; emit them when missing on encode and substitute on decode where proto2 semantics require. (3) Groups (SGROUP/EGROUP wire format): encode/decode the framed group syntax. (4) Extensions: walk extension ranges, use plan-cached extension descriptor pointers, store in result._extensions. (5) Closed enum semantics for proto2. Depends on 3b + 3c + 3d + 3e + 3g + 3j (it's the join point — proto2 touches scalars, sub-messages, repeated, oneofs, unknown-fields). Acceptance: proto2_basic.BenchPayload and test_messages_proto2 round-trip byte-equal to mode=full at all bench sizes; conformance proto2 suite passes under PB_ENABLE_C=1.","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:21:03Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:21:03Z","dependencies":[{"issue_id":"tarantool-protobuf-m7u","depends_on_id":"tarantool-protobuf-hwe","type":"blocks","created_at":"2026-05-18T23:22:21Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-m7u","depends_on_id":"tarantool-protobuf-jc9","type":"blocks","created_at":"2026-05-18T23:22:22Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-m7u","depends_on_id":"tarantool-protobuf-mz6","type":"blocks","created_at":"2026-05-18T23:22:20Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-m7u","depends_on_id":"tarantool-protobuf-w3u","type":"blocks","created_at":"2026-05-18T23:22:22Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-m7u","depends_on_id":"tarantool-protobuf-wyp","type":"blocks","created_at":"2026-05-18T23:22:23Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-m7u","depends_on_id":"tarantool-protobuf-y1n","type":"blocks","created_at":"2026-05-18T23:22:20Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":6,"dependent_count":1,"comment_count":0}
-{"_type":"issue","id":"tarantool-protobuf-asz","title":"ra6 3h: maps (entry-as-pseudo-message)","description":"map\u003cK,V\u003e fields are wire-encoded as repeated messages with synthetic Entry { key=1; value=2 } shape. Compile a map plan: outer field is K_REPEATED_MESSAGE with a synthesized Entry sub-plan. Encode: walk the Lua map via pairs(), emit each entry (this is the documented JIT exception — map fields are allowed pairs() in the hot path per CLAUDE.md). Decode: each entry yields a key + value, lua_settable into the result map. Depends on 3d (uses sub-message machinery). Acceptance: Person.ages_by_nickname / nickname_by_age / addresses_by_label round-trip; multi-key map fixtures behave correctly even though encode byte order differs (the existing map_test pattern).","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:57Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:20:57Z","dependencies":[{"issue_id":"tarantool-protobuf-asz","depends_on_id":"tarantool-protobuf-hwe","type":"blocks","created_at":"2026-05-18T23:22:19Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0}
+{"_type":"issue","id":"tarantool-protobuf-asz","title":"ra6 3h: maps (entry-as-pseudo-message)","description":"map\u003cK,V\u003e fields are wire-encoded as repeated messages with synthetic Entry { key=1; value=2 } shape. Compile a map plan: outer field is K_REPEATED_MESSAGE with a synthesized Entry sub-plan. Encode: walk the Lua map via pairs(), emit each entry (this is the documented JIT exception — map fields are allowed pairs() in the hot path per CLAUDE.md). Decode: each entry yields a key + value, lua_settable into the result map. Depends on 3d (uses sub-message machinery). Acceptance: Person.ages_by_nickname / nickname_by_age / addresses_by_label round-trip; multi-key map fixtures behave correctly even though encode byte order differs (the existing map_test pattern).","status":"closed","priority":2,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:57Z","created_by":"Eugene Blikh","updated_at":"2026-05-23T17:07:06Z","started_at":"2026-05-23T16:45:29Z","closed_at":"2026-05-23T17:07:06Z","close_reason":"ra6 3h: maps (entry-as-pseudo-message) — encode walks the user map with lua_next (CLAUDE.md JIT exception), builds each entry payload in a stack-backed sub-buffer with synthetic tag(1,key) + tag(2,value), proto3-elides defaults independently for key and value, and emits outer tag + len + payload into the parent buffer. Map\u003c,message\u003e resolves the value sub-plan once and recurses through encode_body. Decode reads the entry payload bounded, dispatches inner id=1/id=2 (skipping anything else per spec), and lua_rawsets into a lazy-created result map table; missing key or value falls back to the proto3 zero. Reuses the existing list_stack_idx[] slot for the lazy map cache (a field is either repeated or map, never both). 12 new tests cover round-trip for ages_by_nickname (string→int32), nickname_by_age (int32→string), and addresses_by_label (string→message), plus default-elision, multi-key correctness, empty maps, and unknown-inner-id tolerance. Full suite 964/964 with PB_ENABLE_C=1.","dependencies":[{"issue_id":"tarantool-protobuf-asz","depends_on_id":"tarantool-protobuf-hwe","type":"blocks","created_at":"2026-05-18T23:22:19Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0}
{"_type":"issue","id":"tarantool-protobuf-wyp","title":"ra6 3j: unknown-fields capture","description":"A C-runtime decoder that meets fields not in the plan must capture their raw bytes into result._unknown_fields, and a subsequent C encode must re-emit them verbatim — same contract as test/unknown_test.lua against the pure-Lua paths. Bytes go in as a Lua string keyed by field number (mirror the existing convention). Depends on 3b + 3c (need the basic decode loop to know where to splice the unknown bytes). Acceptance: test/unknown_test.lua passes under PB_ENABLE_C=1; an evolved schema decoding bytes written by an old schema preserves the unknown fields through a C-side re-encode.","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:40Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:20:40Z","dependencies":[{"issue_id":"tarantool-protobuf-wyp","depends_on_id":"tarantool-protobuf-mz6","type":"blocks","created_at":"2026-05-18T23:22:10Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-wyp","depends_on_id":"tarantool-protobuf-y1n","type":"blocks","created_at":"2026-05-18T23:22:09Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":2,"dependent_count":2,"comment_count":0}
{"_type":"issue","id":"tarantool-protobuf-w3u","title":"ra6 3g: oneofs","description":"Oneof groups: on encode, only the field with the current 'which' set is written; on decode, last-write-wins per the oneof group. Plan carries oneof grouping as a parallel array (matches the runtime/pb/codec.lua desc.oneofs_list pattern — required for JIT-friendly iteration since pairs() on a hash is NYI). Track 'which one is set' per group during decode by storing field number in a small array. Depends on 3b + 3c. Acceptance: hello.Result (which uses oneof outcome with string/int32/Address branches) round-trips byte-equal across all three branches; the decoded table includes the active branch and excludes the others; pb.lazy view's :which() returns the correct branch name.","status":"closed","priority":2,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:36Z","created_by":"Eugene Blikh","updated_at":"2026-05-23T13:17:51Z","started_at":"2026-05-23T13:15:43Z","closed_at":"2026-05-23T13:17:51Z","close_reason":"Closed","dependencies":[{"issue_id":"tarantool-protobuf-w3u","depends_on_id":"tarantool-protobuf-mz6","type":"blocks","created_at":"2026-05-18T23:22:08Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-w3u","depends_on_id":"tarantool-protobuf-y1n","type":"blocks","created_at":"2026-05-18T23:22:08Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":2,"dependent_count":2,"comment_count":0}
{"_type":"issue","id":"tarantool-protobuf-21d","title":"runtime/Person_encode emits 2.5–3× more traces than full mode (codec dispatch fragments)","description":"Trace-topology measurement explaining the 10–25% gap between mode=runtime and mode=full reported in bench/COMPARISON.md.\n\nMeasured on Tarantool 3.8.0-entrypoint / LuaJIT 2.1.0-beta3 / macOS arm64 via bench/jit_trace.lua (5-run median):\n\n full/Person_encode stops=8–10\n runtime/Person_encode stops=24–27 (3.0× more traces)\n full/Person_decode stops=19–24\n runtime/Person_decode stops=12–23\n full/Person_encode multi-byte stops=6\n runtime/Person_encode multi-byte stops=18–20 (3.0×)\n\nThe runtime-side encode in runtime/pb/codec.lua dispatches on field.kind / field.proto_type per field per iteration. Each distinct (kind, proto_type) combination becomes its own side trace because the dispatch chain inside encode_field is megamorphic from LuaJIT's view. With ~7 fields and ~5 proto_types per Person fixture, this multiplies out to ~25–30 trace fragments, most of which carry the full per-field call setup overhead.\n\nThe compile_writers / compile_readers hooks in runtime/pb/init.lua already attach per-field monomorphic functions (codec.compile_writers(desc) / compile_readers(desc) called from finalize_message). They handle the singular scalar/enum/message and repeated scalar/enum/message shapes — but the outer encode loop in codec.encode_message still iterates desc.fields and branches per-shape before calling the writer. The branch is what fragments.\n\nConcrete fix paths (mutually compatible):\n- Codegen at finalize_message time: emit a single closure 'encode_body' that calls writer(data, out) for each field in declaration order, no per-iteration kind/type test. The body is monomorphic because each writer is monomorphic.\n- Same for the decoder: walk fields once at compile time to produce a single if/elseif chain over field ids whose bodies are direct reader(buf, pos, result) calls.\n\nExpected impact: most of the runtime-vs-full gap in COMPARISON.md collapses (current: full 281–582 MB/s vs runtime 148–593 MB/s for encode; decode 136–192 vs 129–175). Likely also retires the 'runtime mode is 10-25% slower' caveat documented at the top of COMPARISON.md.\n\nValidation:\n- bench/jit_trace.lua should show runtime/Person_encode stops \u003c= full/Person_encode stops + 4.\n- bench/bench.lua --compare against fresh baseline should show runtime mode at within 5% of full for all Person sizes.","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T17:27:19Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T17:27:19Z","dependency_count":0,"dependent_count":0,"comment_count":0}
M runtime/pb/c/c_runtime.c => runtime/pb/c/c_runtime.c +438 -11
@@ 1181,10 1181,198 @@ encode_packed_element_at(lua_State *L, enc_buf *b, pb_plan_field *f,
}
}
+/* ---------------------------------------------------------------- *
+ * Map helpers (bd-asz / ra6 3h). *
+ * *
+ * A map<K,V> field is wire-encoded as a repeated message with a *
+ * synthetic Entry { key=1; value=2 } shape. Each entry payload *
+ * contains zero/one key followed by zero/one value (defaults are *
+ * proto3-elided on encode and re-defaulted on decode). *
+ * *
+ * These helpers work with raw kind values (uint8_t) rather than *
+ * pb_plan_field pointers because map K and V are not full *
+ * pb_plan_fields in the plan — only their kinds are recorded. *
+ * Enum values are accepted only as numeric ordinals (no enum_ref *
+ * is stored per map value; map<,enum> with string-form enums is *
+ * not in the acceptance set for 3h). *
+ * ---------------------------------------------------------------- */
+
+static uint8_t
+wire_type_for_kind(uint8_t kind)
+{
+ switch (kind) {
+ case PB_KIND_INT32: case PB_KIND_INT64:
+ case PB_KIND_UINT32: case PB_KIND_UINT64:
+ case PB_KIND_SINT32: case PB_KIND_SINT64:
+ case PB_KIND_BOOL: case PB_KIND_ENUM:
+ return PB_WIRE_VARINT;
+ case PB_KIND_FIXED32: case PB_KIND_SFIXED32: case PB_KIND_FLOAT:
+ return PB_WIRE_I32;
+ case PB_KIND_FIXED64: case PB_KIND_SFIXED64: case PB_KIND_DOUBLE:
+ return PB_WIRE_I64;
+ case PB_KIND_STRING: case PB_KIND_BYTES: case PB_KIND_MESSAGE:
+ return PB_WIRE_LEN;
+ default:
+ return 0;
+ }
+}
+
+/* Write a single value into `b` for the given kind. No tag, no length-
+ * prefix for non-LEN kinds. STRING/BYTES include the varint length per
+ * wire spec. MESSAGE is NOT handled here — sub-message values need a
+ * sub-plan and recurse through encode_body. */
+static void
+encode_kind_value(lua_State *L, enc_buf *b, uint8_t kind, int val_idx)
+{
+ switch (kind) {
+ case PB_KIND_INT32:
+ case PB_KIND_INT64:
+ case PB_KIND_UINT32:
+ case PB_KIND_UINT64: {
+ uint64_t u = to_uint64_at(L, val_idx);
+ ebuf_reserve(L, b, 10);
+ ebuf_put_varint(b, u);
+ return;
+ }
+ case PB_KIND_SINT32: {
+ int32_t s = to_int32_at(L, val_idx);
+ ebuf_reserve(L, b, 5);
+ ebuf_put_varint(b, zigzag32(s));
+ return;
+ }
+ case PB_KIND_SINT64: {
+ int64_t s = to_int64_at(L, val_idx);
+ ebuf_reserve(L, b, 10);
+ ebuf_put_varint(b, zigzag64(s));
+ return;
+ }
+ case PB_KIND_BOOL: {
+ int truthy = lua_toboolean(L, val_idx);
+ ebuf_reserve(L, b, 1);
+ ebuf_put_byte(b, truthy ? 1 : 0);
+ return;
+ }
+ case PB_KIND_FIXED32:
+ case PB_KIND_SFIXED32: {
+ uint64_t u = to_uint64_at(L, val_idx);
+ ebuf_reserve(L, b, 4);
+ ebuf_put_fixed32(b, (uint32_t)u);
+ return;
+ }
+ case PB_KIND_FIXED64:
+ case PB_KIND_SFIXED64: {
+ uint64_t u = to_uint64_at(L, val_idx);
+ ebuf_reserve(L, b, 8);
+ ebuf_put_fixed64(b, u);
+ return;
+ }
+ case PB_KIND_FLOAT: {
+ double d = lua_tonumber(L, val_idx);
+ ebuf_reserve(L, b, 4);
+ ebuf_put_fixed32(b, f32_to_u32((float)d));
+ return;
+ }
+ case PB_KIND_DOUBLE: {
+ double d = lua_tonumber(L, val_idx);
+ ebuf_reserve(L, b, 8);
+ ebuf_put_fixed64(b, f64_to_u64(d));
+ return;
+ }
+ case PB_KIND_ENUM: {
+ int32_t e = (int32_t)to_int64_at(L, val_idx);
+ ebuf_reserve(L, b, 10);
+ ebuf_put_varint(b, (uint64_t)(int64_t)e);
+ return;
+ }
+ case PB_KIND_STRING:
+ case PB_KIND_BYTES: {
+ size_t len;
+ const char *s = lua_tolstring(L, val_idx, &len);
+ if (s == NULL)
+ luaL_error(L, "map string/bytes value must be a string");
+ ebuf_reserve(L, b, 10 + len);
+ ebuf_put_varint(b, (uint64_t)len);
+ if (len > 0)
+ ebuf_put_bytes(b, (const uint8_t *)s, len);
+ return;
+ }
+ default:
+ luaL_error(L, "encode_kind_value: unsupported kind %d",
+ (int)kind);
+ }
+}
+
+/* Proto3 default predicate for a Lua value of a given kind. Used to
+ * elide key/value bytes in map entries when they collapse to the type
+ * zero, matching the runtime Lua codec's `is_default_scalar` behavior. */
+static int
+value_is_default_kind(lua_State *L, uint8_t kind, int val_idx)
+{
+ switch (kind) {
+ case PB_KIND_INT32: case PB_KIND_UINT32:
+ case PB_KIND_SINT32: case PB_KIND_FIXED32: case PB_KIND_SFIXED32:
+ case PB_KIND_ENUM:
+ if (lua_type(L, val_idx) == LUA_TNUMBER)
+ return lua_tonumber(L, val_idx) == 0;
+ return luaL_toint64(L, val_idx) == 0;
+ case PB_KIND_INT64: case PB_KIND_UINT64:
+ case PB_KIND_SINT64: case PB_KIND_FIXED64: case PB_KIND_SFIXED64:
+ if (lua_type(L, val_idx) == LUA_TNUMBER)
+ return lua_tonumber(L, val_idx) == 0;
+ return luaL_toint64(L, val_idx) == 0;
+ case PB_KIND_BOOL:
+ return !lua_toboolean(L, val_idx);
+ case PB_KIND_FLOAT:
+ case PB_KIND_DOUBLE: {
+ double d = lua_tonumber(L, val_idx);
+ if (d != 0.0) return 0;
+ /* -0.0 still emits — its bit pattern is non-zero. */
+ union { double d; uint64_t u; } pun;
+ pun.d = d;
+ return pun.u == 0;
+ }
+ case PB_KIND_STRING:
+ case PB_KIND_BYTES: {
+ size_t len;
+ lua_tolstring(L, val_idx, &len);
+ return len == 0;
+ }
+ default:
+ return 0;
+ }
+}
+
+/* Push the proto3 zero value for the given kind. MESSAGE pushes an
+ * empty table — used as a placeholder before the message decode body
+ * replaces it. */
+static void
+push_default_for_kind(lua_State *L, uint8_t kind)
+{
+ switch (kind) {
+ case PB_KIND_BOOL:
+ lua_pushboolean(L, 0); return;
+ case PB_KIND_STRING:
+ case PB_KIND_BYTES:
+ lua_pushlstring(L, "", 0); return;
+ case PB_KIND_INT64: luaL_pushint64(L, 0); return;
+ case PB_KIND_UINT64: luaL_pushuint64(L, 0); return;
+ case PB_KIND_SINT64: luaL_pushint64(L, 0); return;
+ case PB_KIND_FIXED64: luaL_pushuint64(L, 0); return;
+ case PB_KIND_SFIXED64: luaL_pushint64(L, 0); return;
+ case PB_KIND_MESSAGE:
+ lua_newtable(L); return;
+ default:
+ /* int32/uint32/sint32/fixed32/sfixed32/enum/float/double → 0 */
+ lua_pushinteger(L, 0); return;
+ }
+}
+
/* Forward decls for the recursive encode pair. */
static void encode_body(lua_State *L, enc_buf *b, pb_plan *plan, int msg_idx);
static void encode_submessage_field(lua_State *L, enc_buf *b, pb_plan *plan,
pb_plan_field *f, int val_idx);
+static void encode_map_field(lua_State *L, enc_buf *b, pb_plan *plan,
+ pb_plan_field *f, int val_idx);
/* Encode a repeated field's elements into `b`. Dispatches on element
* kind and the `packed` plan flag:
@@ 1307,6 1495,115 @@ encode_submessage_field(lua_State *L, enc_buf *b, pb_plan *plan,
lua_settop(L, saved_top);
}
+/* Encode a map<K,V> field into `b`.
+ *
+ * Wire shape: each (k, v) pair becomes a length-delimited entry sub-
+ * message with synthetic field-1 key + field-2 value. Proto3 default-
+ * elision applies independently to key and value (key=='' or k==0 skip
+ * the key tag; v==zero skips the value tag). Sub-message values are
+ * always emitted regardless of contents (proto3 message presence).
+ *
+ * Iteration uses `lua_next` (pairs semantics) — this is the documented
+ * JIT exception per CLAUDE.md. Map encode is not on the JIT-traced hot
+ * path; hash iteration is the only way to walk user-provided table keys.
+ *
+ * Stack/buffer lifecycle mirrors encode_submessage_field: the parent's
+ * heap is force-established before any per-entry sub-buffer can allocate,
+ * so the parent's heap_idx survives the per-iter `lua_settop` cleanup. */
+static void
+encode_map_field(lua_State *L, enc_buf *b, pb_plan *plan,
+ pb_plan_field *f, int val_idx)
+{
+ val_idx = abs_idx(L, val_idx);
+ if (lua_type(L, val_idx) != LUA_TTABLE)
+ luaL_error(L, "map field requires a table value");
+
+ /* Fast empty-check: probe the first iter step and bail if nothing. */
+ lua_pushnil(L);
+ if (lua_next(L, val_idx) == 0)
+ return;
+ lua_pop(L, 2); /* drop probe k+v */
+
+ /* Force parent's heap to exist before per-entry sub-bufs allocate. */
+ if (b->heap_idx == 0)
+ ebuf_grow(L, b, 1);
+
+ /* Pre-compute entry-internal tags. Both ids are < 16 so they fit
+ * in a single varint byte. */
+ uint8_t key_wt = wire_type_for_kind(f->map_key_kind);
+ uint8_t val_wt = wire_type_for_kind(f->map_value_kind);
+ uint8_t key_tag = (uint8_t)((1u << 3) | key_wt);
+ uint8_t val_tag = (uint8_t)((2u << 3) | val_wt);
+
+ /* Resolve value sub-plan once if this is map<K, message>. */
+ pb_plan *value_subplan = NULL;
+ if (f->map_value_kind == PB_KIND_MESSAGE) {
+ if (plan->sub_plans_ref == LUA_NOREF ||
+ f->map_value_sub_plan_idx <= 0)
+ luaL_error(L,
+ "map<,message> field has no value sub-plan");
+ lua_rawgeti(L, LUA_REGISTRYINDEX, plan->sub_plans_ref);
+ lua_rawgeti(L, -1, f->map_value_sub_plan_idx);
+ value_subplan = (pb_plan *)lua_touserdata(L, -1);
+ lua_pop(L, 2);
+ if (value_subplan == NULL)
+ luaL_error(L, "map value sub-plan is not a userdata");
+ }
+
+ int saved_top = lua_gettop(L);
+ lua_pushnil(L); /* iter sentinel */
+ while (lua_next(L, val_idx) != 0) {
+ int k_idx = saved_top + 1;
+ int v_idx = saved_top + 2;
+
+ /* Build the entry payload in a stack-backed sub-buffer. */
+ enc_buf entry;
+ ebuf_init(&entry);
+
+ /* Key (proto3-elide on default). */
+ if (!value_is_default_kind(L, f->map_key_kind, k_idx)) {
+ ebuf_reserve(L, &entry, 1);
+ ebuf_put_byte(&entry, key_tag);
+ encode_kind_value(L, &entry, f->map_key_kind, k_idx);
+ }
+
+ /* Value. Messages always emit (presence is meaningful); other
+ * kinds proto3-elide on default. */
+ if (f->map_value_kind == PB_KIND_MESSAGE) {
+ if (lua_type(L, v_idx) != LUA_TTABLE)
+ luaL_error(L,
+ "map<,message> value must be a table");
+ enc_buf vbody;
+ ebuf_init(&vbody);
+ encode_body(L, &vbody, value_subplan, v_idx);
+ ebuf_reserve(L, &entry,
+ 1 + 10 + vbody.used);
+ ebuf_put_byte(&entry, val_tag);
+ ebuf_put_varint(&entry, (uint64_t)vbody.used);
+ if (vbody.used > 0)
+ ebuf_put_bytes(&entry, ebuf_base(&vbody),
+ vbody.used);
+ } else if (!value_is_default_kind(L, f->map_value_kind,
+ v_idx)) {
+ ebuf_reserve(L, &entry, 1);
+ ebuf_put_byte(&entry, val_tag);
+ encode_kind_value(L, &entry, f->map_value_kind,
+ v_idx);
+ }
+
+ /* Emit outer tag + len-varint + entry body into parent. */
+ ebuf_reserve(L, b, f->tag_len + 10 + entry.used);
+ ebuf_put_tag(b, f);
+ ebuf_put_varint(b, (uint64_t)entry.used);
+ if (entry.used > 0)
+ ebuf_put_bytes(b, ebuf_base(&entry), entry.used);
+
+ /* Drop per-entry userdata frames, leave key for lua_next. */
+ lua_settop(L, saved_top + 1);
+ }
+ /* lua_next returned 0 — it has already popped the final key. */
+}
+
static void
encode_body(lua_State *L, enc_buf *b, pb_plan *plan, int msg_idx)
{
@@ 1347,9 1644,6 @@ encode_body(lua_State *L, enc_buf *b, pb_plan *plan, int msg_idx)
for (int i = 0; i < plan->n_fields; i++) {
pb_plan_field *f = &plan->fields[i];
- /* Map dispatches in 3h. */
- if (f->kind == PB_KIND_MAP) continue;
-
/* Oneof: skip every non-active member. The active member
* encodes with force_emit so default values (e.g. text="")
* still carry presence. */
@@ 1365,7 1659,9 @@ encode_body(lua_State *L, enc_buf *b, pb_plan *plan, int msg_idx)
continue;
}
- if (f->repeated) {
+ if (f->kind == PB_KIND_MAP) {
+ encode_map_field(L, b, plan, f, val_idx);
+ } else if (f->repeated) {
if (lua_type(L, val_idx) != LUA_TTABLE)
luaL_error(L,
"repeated field requires a table value");
@@ 1513,11 1809,14 @@ zigzag64_dec(uint64_t u)
return (int64_t)((u >> 1) ^ (~(u & 1) + 1));
}
-/* Decode a single value for field `f` and push it onto the Lua stack. */
+/* Decode a single value of the given `kind` from the stream and push it
+ * onto the Lua stack. Per-kind Lua representations match wire.lua's
+ * decoder (cdata int64/uint64 for 64-bit kinds, Lua number/integer for
+ * everything else, lstring for STRING/BYTES). */
static void
-dec_push_one(dec_ctx *c, pb_plan_field *f)
+dec_push_kind(dec_ctx *c, uint8_t kind)
{
- switch (f->kind) {
+ switch (kind) {
case PB_KIND_INT32: {
uint64_t u = dec_varint(c);
int32_t v = (int32_t)(uint32_t)u;
@@ 1607,15 1906,25 @@ dec_push_one(dec_ctx *c, pb_plan_field *f)
return;
}
default:
- luaL_error(c->L, "dec_push_one: unsupported kind %d",
- (int)f->kind);
+ luaL_error(c->L, "dec_push_kind: unsupported kind %d",
+ (int)kind);
}
}
+/* Decode a single value for field `f` and push it onto the Lua stack.
+ * Thin wrapper around dec_push_kind to preserve the existing call sites. */
+static void
+dec_push_one(dec_ctx *c, pb_plan_field *f)
+{
+ dec_push_kind(c, f->kind);
+}
+
/* Forward decl for the recursive decode. */
static void decode_body(dec_ctx *c, pb_plan *plan, int result_idx);
static void decode_submessage_field(dec_ctx *c, pb_plan_field *f,
int sub_plans_idx);
+static void decode_map_entry(dec_ctx *c, pb_plan_field *f, int sub_plans_idx,
+ int map_idx);
/* Decode one singular sub-message field. On entry, `c->pos` points at
* the length-varint byte; on exit, `c->pos == c->pos + plen`. Pushes
@@ 1652,6 1961,98 @@ decode_submessage_field(dec_ctx *c, pb_plan_field *f, int sub_plans_idx)
c->len = saved_len;
}
+/* Decode one map<K,V> entry from the wire and lua_rawset it into the
+ * map table at absolute stack index `map_idx`.
+ *
+ * On entry `c->pos` points at the entry's length-varint (one element of
+ * the outer repeated-message stream). On exit `c->pos` has advanced past
+ * the entry. Pushes the {key, value} pair temporarily, sets into the
+ * map, then pops both.
+ *
+ * Inner-tag semantics:
+ * id == 1: key value
+ * id == 2: value value
+ * anything else: skipped per the wire-format spec for synthetic Entry
+ *
+ * Missing key or value defaults to the proto3 zero for the corresponding
+ * kind — that's how an empty entry payload `{}` round-trips through a
+ * map<,> field (see test_map_defaults_round_trip). For message-typed
+ * values, the default is an empty Lua table. */
+static void
+decode_map_entry(dec_ctx *c, pb_plan_field *f, int sub_plans_idx,
+ int map_idx)
+{
+ lua_State *L = c->L;
+ uint64_t plen = dec_varint(c);
+ if (c->len - c->pos < plen)
+ luaL_error(L, "truncated map entry for field %d",
+ (int)f->field_number);
+
+ size_t saved_len = c->len;
+ c->len = c->pos + (size_t)plen;
+
+ /* Pre-push proto3 defaults; replace if the corresponding tag arrives. */
+ push_default_for_kind(L, f->map_key_kind);
+ int key_idx = lua_gettop(L);
+ push_default_for_kind(L, f->map_value_kind);
+ int val_idx = lua_gettop(L);
+
+ while (c->pos < c->len) {
+ uint64_t tag = dec_varint(c);
+ uint32_t id = (uint32_t)(tag >> 3);
+ uint8_t wt = (uint8_t)(tag & 0x07);
+ if (id == 1) {
+ dec_push_kind(c, f->map_key_kind);
+ lua_replace(L, key_idx);
+ } else if (id == 2) {
+ if (f->map_value_kind == PB_KIND_MESSAGE) {
+ if (f->map_value_sub_plan_idx <= 0)
+ luaL_error(L,
+ "map<,message> has no value sub-plan");
+ lua_rawgeti(L, sub_plans_idx,
+ f->map_value_sub_plan_idx);
+ pb_plan *vsub =
+ (pb_plan *)lua_touserdata(L, -1);
+ if (vsub == NULL)
+ luaL_error(L,
+ "map value sub-plan is not a userdata");
+ lua_pop(L, 1);
+
+ uint64_t sub_len = dec_varint(c);
+ if (c->len - c->pos < sub_len)
+ luaL_error(L,
+ "truncated nested map<,message> value");
+ lua_createtable(L, 0, vsub->n_fields);
+ int new_val = lua_gettop(L);
+ size_t saved2 = c->len;
+ c->len = c->pos + (size_t)sub_len;
+ decode_body(c, vsub, new_val);
+ if (c->pos != c->len)
+ luaL_error(L,
+ "nested map<,message> body underflow");
+ c->len = saved2;
+ lua_replace(L, val_idx);
+ } else {
+ dec_push_kind(c, f->map_value_kind);
+ lua_replace(L, val_idx);
+ }
+ } else {
+ dec_skip(c, wt);
+ }
+ }
+
+ /* map[key] = val — proto3 last-wins per key. cdata int64/uint64 keys
+ * would need a pointer-equality dedup pass (see runtime/pb/codec.lua's
+ * `key_dedup`); the acceptance set for 3h uses only string/int32 keys
+ * so we skip the dedup here. */
+ lua_pushvalue(L, key_idx);
+ lua_pushvalue(L, val_idx);
+ lua_rawset(L, map_idx);
+
+ lua_pop(L, 2); /* drop key + val defaults */
+ c->len = saved_len;
+}
+
/* Return 1 if the field's element wire type is varint/i32/i64 — i.e.
* the field is eligible for packed encoding. Length-delimited kinds
* (STRING/BYTES/MESSAGE) and MAP are never packable. Used on decode
@@ 1736,13 2137,39 @@ decode_body(dec_ctx *c, pb_plan *plan, int result_idx)
}
}
- /* Unknown tag, or map (3h territory). */
- if (f == NULL || f->kind == PB_KIND_MAP) {
+ /* Unknown tag. */
+ if (f == NULL) {
dec_skip(c, wt);
continue;
}
/* ------------------------------------------------------ *
+ * Map dispatch (bd-asz / ra6 3h) *
+ * ------------------------------------------------------ */
+ if (f->kind == PB_KIND_MAP) {
+ if (wt != PB_WIRE_LEN)
+ luaL_error(L,
+ "map field %d expected wire 2, got %d",
+ (int)field_number, (int)wt);
+
+ /* Lazy-create the map table on first hit. Reuses the
+ * list_stack_idx[] slot since a field is either
+ * repeated or map, never both. */
+ int m_idx = list_stack_idx[f_idx];
+ if (m_idx == 0) {
+ lua_createtable(L, 0, 0);
+ lua_pushvalue(L, -1);
+ lua_rawgeti(L, names_idx, f_idx + 1);
+ lua_insert(L, -2);
+ lua_rawset(L, result_idx);
+ m_idx = lua_gettop(L);
+ list_stack_idx[f_idx] = m_idx;
+ }
+ decode_map_entry(c, f, sub_plans_idx, m_idx);
+ continue;
+ }
+
+ /* ------------------------------------------------------ *
* Repeated dispatch *
* ------------------------------------------------------ */
if (f->repeated) {
M test/c_runtime_decode_test.lua => test/c_runtime_decode_test.lua +67 -14
@@ 129,26 129,79 @@ for _, mode in ipairs({'full', 'runtime'}) do
t.assert_equals(c_decoded.avatar, '\x00\x01\xff\xfe')
end
- function g.test_skips_map_tags()
- -- 3e scope: singular message + repeated now decode. Map tags
- -- (synthetic entry sub-messages) remain skipped — 3h replaces
- -- the gate.
+ -- ---------- Map decode (bd-asz / ra6 3h) ----------
+
+ function g.test_map_string_to_int32_round_trip()
local plan = c_runtime.compile_plan(hello.Person_descriptor)
- local rich = {
- name = 'x',
- emails = {'a@b'}, -- repeated string (decode)
- address = {street = 'Main'}, -- message (decode)
- lucky_numbers = {1, 2, 3}, -- repeated packed (decode)
- ages_by_nickname = {alice = 30}, -- map (skip)
- }
- local bytes = full_hello.Person_encode(rich)
+ local msg = {ages_by_nickname = {alice = 30, bob = 25, carol = 40}}
+ local bytes = full_hello.Person_encode(msg)
+ local c_decoded = c_runtime.decode(plan, bytes)
+ t.assert_equals(c_decoded.ages_by_nickname, msg.ages_by_nickname)
+ end
+
+ function g.test_map_int32_to_string_round_trip()
+ local plan = c_runtime.compile_plan(hello.Person_descriptor)
+ local msg = {nickname_by_age = {[30] = 'alice', [25] = 'bob'}}
+ local bytes = full_hello.Person_encode(msg)
+ local c_decoded = c_runtime.decode(plan, bytes)
+ t.assert_equals(c_decoded.nickname_by_age, msg.nickname_by_age)
+ end
+
+ function g.test_map_string_to_message_round_trip()
+ local plan = c_runtime.compile_plan(hello.Person_descriptor)
+ local msg = {addresses_by_label = {
+ home = {street = 'Main', city = 'X', zip = 1},
+ work = {street = '5th', city = 'Y', zip = 2},
+ }}
+ local bytes = full_hello.Person_encode(msg)
local c_decoded = c_runtime.decode(plan, bytes)
- t.assert_equals(c_decoded, {
+ t.assert_equals(c_decoded.addresses_by_label.home.street, 'Main')
+ t.assert_equals(c_decoded.addresses_by_label.home.zip, 1)
+ t.assert_equals(c_decoded.addresses_by_label.work.street, '5th')
+ t.assert_equals(c_decoded.addresses_by_label.work.city, 'Y')
+ end
+
+ function g.test_map_defaults_decode_as_proto3_zeros()
+ -- mode=full encodes {[''] = 0} as a single-entry map with both
+ -- key and value bytes elided. C decoder must surface the proto3
+ -- defaults from the (otherwise empty) entry payload.
+ local plan = c_runtime.compile_plan(hello.Person_descriptor)
+ local bytes = full_hello.Person_encode({ages_by_nickname = {[''] = 0}})
+ local c_decoded = c_runtime.decode(plan, bytes)
+ t.assert_equals(c_decoded.ages_by_nickname[''], 0)
+ end
+
+ function g.test_map_skips_unknown_entry_ids()
+ -- Synthetic Entry must tolerate unknown inner field ids (per the
+ -- proto wire spec for map<,>). Hand-craft an entry payload with a
+ -- spurious id=3 between key and value.
+ local plan = c_runtime.compile_plan(hello.Person_descriptor)
+ local entry = '\x0a\x05alice' -- tag(1, LEN) + len(5) + "alice"
+ .. '\x18\x07' -- tag(3, VARINT) + value 7 (spurious)
+ .. '\x10\x1e' -- tag(2, VARINT) + 30
+ local bytes = '\x6a' -- tag(13, LEN): outer map field
+ .. string.char(#entry)
+ .. entry
+ local c_decoded = c_runtime.decode(plan, bytes)
+ t.assert_equals(c_decoded.ages_by_nickname.alice, 30)
+ end
+
+ function g.test_map_mixed_with_other_fields()
+ local plan = c_runtime.compile_plan(hello.Person_descriptor)
+ local rich = {
name = 'x',
emails = {'a@b'},
address = {street = 'Main'},
lucky_numbers = {1, 2, 3},
- })
+ ages_by_nickname = {alice = 30, bob = 25},
+ }
+ local bytes = full_hello.Person_encode(rich)
+ local c_decoded = c_runtime.decode(plan, bytes)
+ t.assert_equals(c_decoded.name, 'x')
+ t.assert_equals(c_decoded.emails, {'a@b'})
+ t.assert_equals(c_decoded.address.street, 'Main')
+ t.assert_equals(c_decoded.lucky_numbers, {1, 2, 3})
+ t.assert_equals(c_decoded.ages_by_nickname, rich.ages_by_nickname)
end
-- ---------- Sub-message decode (bd-hwe / ra6 3d) ----------
M test/c_runtime_encode_test.lua => test/c_runtime_encode_test.lua +65 -15
@@ 144,25 144,75 @@ for _, mode in ipairs({'full', 'runtime'}) do
full_hello.Person_encode(msg))
end
- function g.test_skips_map_fields()
- -- 3e scope: singular message + repeated (packed and unpacked)
- -- now encode. Map fields remain silently skipped — 3h replaces
- -- the gate.
+ -- ---------- Map encode (bd-asz / ra6 3h) ----------
+
+ function g.test_map_string_to_int32_single_key()
+ -- Single-key fixtures lock down byte-for-byte equality. Multi-key
+ -- map encode order is hash-determined per Lua's pairs() and won't
+ -- match mode=full's pairs() order in general, so we exercise
+ -- multi-key behavior via round-trip below.
+ local plan = c_runtime.compile_plan(hello.Person_descriptor)
+ local msg = {ages_by_nickname = {alice = 30}}
+ t.assert_equals(c_runtime.encode(plan, msg),
+ full_hello.Person_encode(msg))
+ end
+
+ function g.test_map_int32_to_string_single_key()
+ local plan = c_runtime.compile_plan(hello.Person_descriptor)
+ local msg = {nickname_by_age = {[30] = 'alice'}}
+ t.assert_equals(c_runtime.encode(plan, msg),
+ full_hello.Person_encode(msg))
+ end
+
+ function g.test_map_string_to_message_single_key()
+ local plan = c_runtime.compile_plan(hello.Person_descriptor)
+ local msg = {addresses_by_label = {
+ home = {street = 'Main', city = 'X', zip = 1},
+ }}
+ t.assert_equals(c_runtime.encode(plan, msg),
+ full_hello.Person_encode(msg))
+ end
+
+ function g.test_map_empty_emits_nothing()
+ local plan = c_runtime.compile_plan(hello.Person_descriptor)
+ t.assert_equals(c_runtime.encode(plan, {ages_by_nickname = {}}), '')
+ end
+
+ function g.test_map_default_key_and_value_round_trip()
+ -- Empty-string key + zero value: per proto3 wire spec both bytes
+ -- are elided but the entry itself is still emitted (presence of
+ -- the key/value pair is meaningful even when both default).
+ local plan = c_runtime.compile_plan(hello.Person_descriptor)
+ local msg = {ages_by_nickname = {[''] = 0}}
+ local c_bytes = c_runtime.encode(plan, msg)
+ t.assert_equals(c_bytes, full_hello.Person_encode(msg))
+ local dec = full_hello.Person_decode(c_bytes)
+ t.assert_equals(dec.ages_by_nickname[''], 0)
+ end
+
+ function g.test_map_multi_key_round_trip()
+ -- Multi-key encode byte order is hash-determined; we only assert
+ -- that mode=full can decode our bytes back to the same table.
+ local plan = c_runtime.compile_plan(hello.Person_descriptor)
+ local msg = {ages_by_nickname = {alice = 30, bob = 25, carol = 40}}
+ local dec = full_hello.Person_decode(c_runtime.encode(plan, msg))
+ t.assert_equals(dec.ages_by_nickname, msg.ages_by_nickname)
+ end
+
+ function g.test_map_message_value_with_other_fields()
+ -- Ensure map<,message> sub-plan resolution doesn't disturb the
+ -- enclosing message's field-walk.
local plan = c_runtime.compile_plan(hello.Person_descriptor)
local msg = {
name = 'x',
- emails = {'a@b'}, -- repeated string (encode)
- address = {street = 'Main'}, -- message (encode)
- lucky_numbers = {1, 2, 3}, -- repeated packed (encode)
- ages_by_nickname = {alice = 30}, -- map (skip)
- }
- local expected = full_hello.Person_encode({
- name = 'x',
+ addresses_by_label = {home = {street = 'Main', zip = 7}},
emails = {'a@b'},
- address = {street = 'Main'},
- lucky_numbers = {1, 2, 3},
- })
- t.assert_equals(c_runtime.encode(plan, msg), expected)
+ }
+ local dec = full_hello.Person_decode(c_runtime.encode(plan, msg))
+ t.assert_equals(dec.name, 'x')
+ t.assert_equals(dec.emails, {'a@b'})
+ t.assert_equals(dec.addresses_by_label.home.street, 'Main')
+ t.assert_equals(dec.addresses_by_label.home.zip, 7)
end
-- ---------- Sub-message encode (bd-hwe / ra6 3d) ----------