From 4db0366a5008e874de1e26f690f4bff499de51bf Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Sun, 17 May 2026 08:09:42 +0300 Subject: [PATCH] runtime: parser + dynamic accept proto2 sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parser now captures `required=true` and `default_value=…` from the proto2 keywords (instead of dropping `required` silently and ignoring field options). Dynamic descriptor builder reads `parsed.syntax`, flips the repeated-scalar packing default for proto2, and calls `codec.compile_writers/compile_readers` so the per-field required-writer specialization actually fires — without that the generic encode_field path silently elides missing required fields. 64-bit-int defaults are coerced into the appropriate cdata type inside `coerce_default` so the codec's value-comparison rules match what generated code emits. Adds 7 dynamic-mode luatest cases including a static-vs-dynamic byte-parity check. 725 tests pass. --- runtime/pb/dynamic.lua | 65 +++++++++++++++++++++++++++--- runtime/pb/parser.lua | 34 ++++++++++------ test/proto2_test.lua | 91 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 171 insertions(+), 19 deletions(-) diff --git a/runtime/pb/dynamic.lua b/runtime/pb/dynamic.lua index 8e4878aef432dc8eda7ab0a9a6d6e7400c8240df..b0055afa49a2e343117593d793954622512f9ac9 100644 --- a/runtime/pb/dynamic.lua +++ b/runtime/pb/dynamic.lua @@ -7,6 +7,10 @@ -- M. - alias for by_name table -- M._descriptor - the enum descriptor -- +-- Accepts both syntax = "proto2" and syntax = "proto3". For proto2 sources +-- the descriptor surfaces `required=true`, `default_value=…`, and the +-- proto2-spec packing rule (repeated scalars NOT packed by default). +-- -- WKT references (google.protobuf.*) are resolved against pb.wkt so dynamic -- schemas can interop with the same Timestamp/Duration/wrapper sugar that -- generated code uses. @@ -71,8 +75,39 @@ local function resolve_type(typename, index) error("dynamic: cannot resolve type " .. typename, 0) end +-- Coerce a parser-captured default literal into the runtime form the codec +-- expects (cdata for 64-bit ints, numbers for floats, etc). String/bytes +-- pass through as Lua strings; enums stay as symbolic names. +local function coerce_default(proto_type, v) + if v == nil then return nil end + if proto_type == 'int64' or proto_type == 'sint64' or proto_type == 'sfixed64' then + return require('ffi').cast('int64_t', v) + end + if proto_type == 'uint64' or proto_type == 'fixed64' then + return require('ffi').cast('uint64_t', v) + end + if proto_type == 'bool' then + if type(v) == 'string' then return v == 'true' end + return v and true or false + end + if proto_type == 'float' or proto_type == 'double' then + if type(v) == 'string' then return tonumber(v) end + return v + end + if proto_type == 'string' or proto_type == 'bytes' then + return tostring(v) + end + -- int32 family, enums: numbers stay numbers, enum symbolic names stay strings. + if type(v) == 'string' then + local n = tonumber(v); if n then return n end + return v + end + return v +end + -- Build a field descriptor for an ordinary (non-map) field. -local function build_field(field_ast, index) +-- `is_proto2` flips presence + packing defaults from the proto3 baseline. +local function build_field(field_ast, index, is_proto2) local entry = {name = field_ast.name, id = field_ast.id} local kind, ref = resolve_type(field_ast.type, index) entry.kind = kind @@ -85,14 +120,23 @@ local function build_field(field_ast, index) end if field_ast.repeated then entry.repeated = true - -- proto3 packing default: packed for primitives + enums, - -- never for string/bytes/message. + -- Packing default differs by syntax: + -- proto3: packed for primitives + enums unless explicitly disabled. + -- proto2: NOT packed unless explicitly [packed = true]. if kind ~= 'message' and field_ast.type ~= 'string' and field_ast.type ~= 'bytes' then - entry.packed = (field_ast.packed ~= false) + if is_proto2 then + entry.packed = (field_ast.packed == true) + else + entry.packed = (field_ast.packed ~= false) + end end end if field_ast.oneof then entry.oneof = field_ast.oneof end if field_ast.optional then entry.optional = true end + if field_ast.required then entry.required = true end + if field_ast.default_value ~= nil then + entry.default_value = coerce_default(entry.proto_type, field_ast.default_value) + end return entry end @@ -147,6 +191,7 @@ function M.build(parsed) local out = {} local msgs, enums = flatten(parsed) local pkg = parsed.package + local is_proto2 = parsed.syntax == 'proto2' -- 1) Build enum descriptors. local enum_descs = {} @@ -176,7 +221,7 @@ function M.build(parsed) if f.kind == 'map' then desc.fields[#desc.fields + 1] = build_map_field(f, index) else - desc.fields[#desc.fields + 1] = build_field(f, index) + desc.fields[#desc.fields + 1] = build_field(f, index, is_proto2) end end if #m.ast.oneofs > 0 then @@ -229,6 +274,13 @@ function M.build(parsed) end desc.oneofs_list = list end + -- Attach the same per-field writers/readers that pb.finalize_message + -- produces for generated code. Without this, encode_message falls + -- through to the generic encode_field path which does not enforce + -- proto2 `required` — required-missing-on-encode would silently + -- elide instead of erroring. + codec.compile_writers(desc) + codec.compile_readers(desc) end -- 4) Emit wrapper functions per message. @@ -238,7 +290,8 @@ function M.build(parsed) out[flat .. '_new'] = function(t) return t or {} end out[flat .. '_encode'] = function(t) return codec.encode(desc, t) end out[flat .. '_decode'] = function(b) return codec.decode(desc, b) end - -- Optional accessors + -- Optional accessors (presence-tracked fields only — same convention + -- as the build-time codegen). for _, f in ipairs(desc.fields) do if f.optional then out[flat .. '_has_' .. f.name] = function(t) return t[f.name] ~= nil end diff --git a/runtime/pb/parser.lua b/runtime/pb/parser.lua index 8110776671b86397b656f7683b0929aba25ad1d0..0c81a8670c2bdeac0df92539614f807e1cd4f836 100644 --- a/runtime/pb/parser.lua +++ b/runtime/pb/parser.lua @@ -1,22 +1,24 @@ --- Pure-Lua .proto file parser (proto3). +-- Pure-Lua .proto file parser (proto2 + proto3). -- --- Adapted from tarantool-etcd/lib/protobuf/parser.lua with these changes: --- * proto3 explicit `optional` is tracked (sets field.optional = true) --- * the output AST is intentionally minimal — descriptor synthesis lives --- in `pb.dynamic`, which converts the AST into the runtime descriptor --- format used by `pb.encode`/`pb.decode`. +-- Adapted from tarantool-etcd/lib/protobuf/parser.lua. The output AST is +-- intentionally minimal — descriptor synthesis lives in `pb.dynamic`, +-- which converts the AST into the runtime descriptor format used by +-- `pb.encode` / `pb.decode`. -- -- Supported: --- syntax = "proto3" +-- syntax = "proto2" | "proto3" -- package, import (recorded but not resolved across files) -- message + nested messages + nested enums + oneofs + maps -- enum (top-level + nested) -- service { rpc Method(In) returns (Out); } +-- proto2 `required` / `optional` keywords (presence + custom defaults) -- proto3 explicit `optional` +-- field options including `[default = X]`, `[packed = true|false]` -- -- Not yet: -- custom options past simple `option name = value;` (skipped) -- extensions, reserved fields (skipped harmlessly) +-- proto2 `extend` blocks and `group` (deprecated) local M = {} -- --------------------------------------------------------------------------- @@ -204,25 +206,31 @@ local function parse(tokens) consume() skip_to_semi() elseif tok.type == 'ident' and tok.value == 'optional' then - -- proto3 explicit optional + -- proto3 explicit optional / every proto2 optional field consume() local ft = consume('ident').value local fn = consume('ident').value consume('punct', '=') local fid = tonumber(consume('number').value) - parse_field_options() + local opts = parse_field_options() consume('punct', ';') - emit_field({name = fn, type = ft, id = fid, optional = true}) + emit_field({ + name = fn, type = ft, id = fid, optional = true, + default_value = opts.default, + }) elseif tok.type == 'ident' and tok.value == 'required' then - -- Proto2-style required: tolerate by treating as a plain field. + -- Proto2 `required`. The codec enforces presence on encode. consume() local ft = consume('ident').value local fn = consume('ident').value consume('punct', '=') local fid = tonumber(consume('number').value) - parse_field_options() + local opts = parse_field_options() consume('punct', ';') - emit_field({name = fn, type = ft, id = fid}) + emit_field({ + name = fn, type = ft, id = fid, required = true, + default_value = opts.default, + }) elseif tok.type == 'ident' and tok.value == 'repeated' then consume() local ft = consume('ident').value diff --git a/test/proto2_test.lua b/test/proto2_test.lua index ff6558fa60354b5b0e47eaf062e23306e1ceb081..bf01975b8c4e7f55bd6f018c3568fc45c3276dc0 100644 --- a/test/proto2_test.lua +++ b/test/proto2_test.lua @@ -148,6 +148,97 @@ for _, mode in ipairs(MODES) do end end +-- Dynamic (source-parsed) proto2: load test/proto/proto2_basic.proto at +-- runtime and assert the same semantics as the build-time generated +-- modules. The static-vs-dynamic byte-parity is the conformance claim. +do + local g = t.group('proto2_basic.dynamic') + local pb = require('pb') + local fio = require('fio') + local REPO_ROOT = fio.abspath(fio.pathjoin( + fio.dirname(debug.getinfo(1, 'S').source:sub(2)), '..')) + local PROTO_PATH = fio.pathjoin(REPO_ROOT, 'test', 'proto', 'proto2_basic.proto') + + local source = (function() + local f = assert(io.open(PROTO_PATH, 'rb')) + local s = f:read('*a'); f:close(); return s + end)() + local dyn = pb.parse(source) + + g.test_parser_records_proto2_syntax = function() + local ast = pb.parser.parse(source) + t.assert_equals(ast.syntax, 'proto2') + end + + g.test_parser_captures_default = function() + local ast = pb.parser.parse(source) + local defaults + for _, m in ipairs(ast.messages) do + if m.name == 'Defaults' then defaults = m; break end + end + t.assert(defaults) + local fmap = {} + for _, f in ipairs(defaults.fields) do fmap[f.name] = f end + t.assert_equals(fmap.i.default_value, 17) + t.assert_equals(fmap.s.default_value, 'hello') + t.assert_equals(fmap.b.default_value, true) + t.assert_equals(fmap.color.default_value, 'GREEN') + end + + g.test_parser_marks_required = function() + local ast = pb.parser.parse(source) + local card + for _, m in ipairs(ast.messages) do + if m.name == 'Cardinality' then card = m; break end + end + t.assert(card) + local fmap = {} + for _, f in ipairs(card.fields) do fmap[f.name] = f end + t.assert(fmap.r.required, 'r is required') + t.assert(fmap.o.optional, 'o is optional') + t.assert_not(fmap.o.required) + end + + g.test_dynamic_descriptor_surfaces_required = function() + local desc = dyn.Cardinality_descriptor + local r = desc.field_by_name and desc.field_by_name.r + or (function() + for _, f in ipairs(desc.fields) do + if f.name == 'r' then return f end + end + end)() + t.assert(r and r.required, 'dynamic Cardinality.r must carry required=true') + end + + g.test_dynamic_required_missing_errors = function() + local ok, err = pcall(dyn.Cardinality_encode, {}) + t.assert_equals(ok, false) + t.assert_str_contains(err, 'required field missing') + end + + g.test_dynamic_repeated_unpacked_by_default = function() + -- Proto2 default for repeated scalars is NOT packed; the dynamic + -- builder must flip the rule based on parsed.syntax. + local enc = dyn.Cardinality_encode({r = 0, packed_default = {1, 2, 3}}) + t.assert_equals(hex(enc), '0800' .. '180118021803') + end + + g.test_dynamic_byte_parity_with_static = function() + -- Same value through dynamic and static (full mode) must produce + -- identical wire bytes. + local full = require('full.proto2_basic.proto2_basic_pb') + local val = { + r = 1, o = 2, + packed_default = {3, 4, 5}, + explicitly_packed = {6, 7, 8}, + explicitly_unpacked = {9, 10}, + } + t.assert_equals(hex(dyn.Cardinality_encode(val)), + hex(full.Cardinality_encode(val)), + 'dynamic and static must agree byte-for-byte') + end +end + -- Parity: full and runtime modes must produce byte-identical output for -- the same input. Equivalent to the existing parity.full_vs_runtime group. do