M runtime/pb/dynamic.lua => runtime/pb/dynamic.lua +59 -6
@@ 7,6 7,10 @@
-- M.<EnumName> - alias for by_name table
-- M.<EnumName>_descriptor - the enum descriptor
--
+-- Accepts both syntax = "proto2" and syntax = "proto3". For proto2 sources
+-- the descriptor surfaces `required=true`, `default_value=…`, and the
+-- proto2-spec packing rule (repeated scalars NOT packed by default).
+--
-- WKT references (google.protobuf.*) are resolved against pb.wkt so dynamic
-- schemas can interop with the same Timestamp/Duration/wrapper sugar that
-- generated code uses.
@@ 71,8 75,39 @@ local function resolve_type(typename, index)
error("dynamic: cannot resolve type " .. typename, 0)
end
+-- Coerce a parser-captured default literal into the runtime form the codec
+-- expects (cdata for 64-bit ints, numbers for floats, etc). String/bytes
+-- pass through as Lua strings; enums stay as symbolic names.
+local function coerce_default(proto_type, v)
+ if v == nil then return nil end
+ if proto_type == 'int64' or proto_type == 'sint64' or proto_type == 'sfixed64' then
+ return require('ffi').cast('int64_t', v)
+ end
+ if proto_type == 'uint64' or proto_type == 'fixed64' then
+ return require('ffi').cast('uint64_t', v)
+ end
+ if proto_type == 'bool' then
+ if type(v) == 'string' then return v == 'true' end
+ return v and true or false
+ end
+ if proto_type == 'float' or proto_type == 'double' then
+ if type(v) == 'string' then return tonumber(v) end
+ return v
+ end
+ if proto_type == 'string' or proto_type == 'bytes' then
+ return tostring(v)
+ end
+ -- int32 family, enums: numbers stay numbers, enum symbolic names stay strings.
+ if type(v) == 'string' then
+ local n = tonumber(v); if n then return n end
+ return v
+ end
+ return v
+end
+
-- Build a field descriptor for an ordinary (non-map) field.
-local function build_field(field_ast, index)
+-- `is_proto2` flips presence + packing defaults from the proto3 baseline.
+local function build_field(field_ast, index, is_proto2)
local entry = {name = field_ast.name, id = field_ast.id}
local kind, ref = resolve_type(field_ast.type, index)
entry.kind = kind
@@ 85,14 120,23 @@ local function build_field(field_ast, index)
end
if field_ast.repeated then
entry.repeated = true
- -- proto3 packing default: packed for primitives + enums,
- -- never for string/bytes/message.
+ -- Packing default differs by syntax:
+ -- proto3: packed for primitives + enums unless explicitly disabled.
+ -- proto2: NOT packed unless explicitly [packed = true].
if kind ~= 'message' and field_ast.type ~= 'string' and field_ast.type ~= 'bytes' then
- entry.packed = (field_ast.packed ~= false)
+ if is_proto2 then
+ entry.packed = (field_ast.packed == true)
+ else
+ entry.packed = (field_ast.packed ~= false)
+ end
end
end
if field_ast.oneof then entry.oneof = field_ast.oneof end
if field_ast.optional then entry.optional = true end
+ if field_ast.required then entry.required = true end
+ if field_ast.default_value ~= nil then
+ entry.default_value = coerce_default(entry.proto_type, field_ast.default_value)
+ end
return entry
end
@@ 147,6 191,7 @@ function M.build(parsed)
local out = {}
local msgs, enums = flatten(parsed)
local pkg = parsed.package
+ local is_proto2 = parsed.syntax == 'proto2'
-- 1) Build enum descriptors.
local enum_descs = {}
@@ 176,7 221,7 @@ function M.build(parsed)
if f.kind == 'map' then
desc.fields[#desc.fields + 1] = build_map_field(f, index)
else
- desc.fields[#desc.fields + 1] = build_field(f, index)
+ desc.fields[#desc.fields + 1] = build_field(f, index, is_proto2)
end
end
if #m.ast.oneofs > 0 then
@@ 229,6 274,13 @@ function M.build(parsed)
end
desc.oneofs_list = list
end
+ -- Attach the same per-field writers/readers that pb.finalize_message
+ -- produces for generated code. Without this, encode_message falls
+ -- through to the generic encode_field path which does not enforce
+ -- proto2 `required` — required-missing-on-encode would silently
+ -- elide instead of erroring.
+ codec.compile_writers(desc)
+ codec.compile_readers(desc)
end
-- 4) Emit wrapper functions per message.
@@ 238,7 290,8 @@ function M.build(parsed)
out[flat .. '_new'] = function(t) return t or {} end
out[flat .. '_encode'] = function(t) return codec.encode(desc, t) end
out[flat .. '_decode'] = function(b) return codec.decode(desc, b) end
- -- Optional accessors
+ -- Optional accessors (presence-tracked fields only — same convention
+ -- as the build-time codegen).
for _, f in ipairs(desc.fields) do
if f.optional then
out[flat .. '_has_' .. f.name] = function(t) return t[f.name] ~= nil end
M runtime/pb/parser.lua => runtime/pb/parser.lua +21 -13
@@ 1,22 1,24 @@
--- Pure-Lua .proto file parser (proto3).
+-- Pure-Lua .proto file parser (proto2 + proto3).
--
--- Adapted from tarantool-etcd/lib/protobuf/parser.lua with these changes:
--- * proto3 explicit `optional` is tracked (sets field.optional = true)
--- * the output AST is intentionally minimal — descriptor synthesis lives
--- in `pb.dynamic`, which converts the AST into the runtime descriptor
--- format used by `pb.encode`/`pb.decode`.
+-- Adapted from tarantool-etcd/lib/protobuf/parser.lua. The output AST is
+-- intentionally minimal — descriptor synthesis lives in `pb.dynamic`,
+-- which converts the AST into the runtime descriptor format used by
+-- `pb.encode` / `pb.decode`.
--
-- Supported:
--- syntax = "proto3"
+-- syntax = "proto2" | "proto3"
-- package, import (recorded but not resolved across files)
-- message + nested messages + nested enums + oneofs + maps
-- enum (top-level + nested)
-- service { rpc Method(In) returns (Out); }
+-- proto2 `required` / `optional` keywords (presence + custom defaults)
-- proto3 explicit `optional`
+-- field options including `[default = X]`, `[packed = true|false]`
--
-- Not yet:
-- custom options past simple `option name = value;` (skipped)
-- extensions, reserved fields (skipped harmlessly)
+-- proto2 `extend` blocks and `group` (deprecated)
local M = {}
-- ---------------------------------------------------------------------------
@@ 204,25 206,31 @@ local function parse(tokens)
consume()
skip_to_semi()
elseif tok.type == 'ident' and tok.value == 'optional' then
- -- proto3 explicit optional
+ -- proto3 explicit optional / every proto2 optional field
consume()
local ft = consume('ident').value
local fn = consume('ident').value
consume('punct', '=')
local fid = tonumber(consume('number').value)
- parse_field_options()
+ local opts = parse_field_options()
consume('punct', ';')
- emit_field({name = fn, type = ft, id = fid, optional = true})
+ emit_field({
+ name = fn, type = ft, id = fid, optional = true,
+ default_value = opts.default,
+ })
elseif tok.type == 'ident' and tok.value == 'required' then
- -- Proto2-style required: tolerate by treating as a plain field.
+ -- Proto2 `required`. The codec enforces presence on encode.
consume()
local ft = consume('ident').value
local fn = consume('ident').value
consume('punct', '=')
local fid = tonumber(consume('number').value)
- parse_field_options()
+ local opts = parse_field_options()
consume('punct', ';')
- emit_field({name = fn, type = ft, id = fid})
+ emit_field({
+ name = fn, type = ft, id = fid, required = true,
+ default_value = opts.default,
+ })
elseif tok.type == 'ident' and tok.value == 'repeated' then
consume()
local ft = consume('ident').value
M test/proto2_test.lua => test/proto2_test.lua +91 -0
@@ 148,6 148,97 @@ for _, mode in ipairs(MODES) do
end
end
+-- Dynamic (source-parsed) proto2: load test/proto/proto2_basic.proto at
+-- runtime and assert the same semantics as the build-time generated
+-- modules. The static-vs-dynamic byte-parity is the conformance claim.
+do
+ local g = t.group('proto2_basic.dynamic')
+ local pb = require('pb')
+ local fio = require('fio')
+ local REPO_ROOT = fio.abspath(fio.pathjoin(
+ fio.dirname(debug.getinfo(1, 'S').source:sub(2)), '..'))
+ local PROTO_PATH = fio.pathjoin(REPO_ROOT, 'test', 'proto', 'proto2_basic.proto')
+
+ local source = (function()
+ local f = assert(io.open(PROTO_PATH, 'rb'))
+ local s = f:read('*a'); f:close(); return s
+ end)()
+ local dyn = pb.parse(source)
+
+ g.test_parser_records_proto2_syntax = function()
+ local ast = pb.parser.parse(source)
+ t.assert_equals(ast.syntax, 'proto2')
+ end
+
+ g.test_parser_captures_default = function()
+ local ast = pb.parser.parse(source)
+ local defaults
+ for _, m in ipairs(ast.messages) do
+ if m.name == 'Defaults' then defaults = m; break end
+ end
+ t.assert(defaults)
+ local fmap = {}
+ for _, f in ipairs(defaults.fields) do fmap[f.name] = f end
+ t.assert_equals(fmap.i.default_value, 17)
+ t.assert_equals(fmap.s.default_value, 'hello')
+ t.assert_equals(fmap.b.default_value, true)
+ t.assert_equals(fmap.color.default_value, 'GREEN')
+ end
+
+ g.test_parser_marks_required = function()
+ local ast = pb.parser.parse(source)
+ local card
+ for _, m in ipairs(ast.messages) do
+ if m.name == 'Cardinality' then card = m; break end
+ end
+ t.assert(card)
+ local fmap = {}
+ for _, f in ipairs(card.fields) do fmap[f.name] = f end
+ t.assert(fmap.r.required, 'r is required')
+ t.assert(fmap.o.optional, 'o is optional')
+ t.assert_not(fmap.o.required)
+ end
+
+ g.test_dynamic_descriptor_surfaces_required = function()
+ local desc = dyn.Cardinality_descriptor
+ local r = desc.field_by_name and desc.field_by_name.r
+ or (function()
+ for _, f in ipairs(desc.fields) do
+ if f.name == 'r' then return f end
+ end
+ end)()
+ t.assert(r and r.required, 'dynamic Cardinality.r must carry required=true')
+ end
+
+ g.test_dynamic_required_missing_errors = function()
+ local ok, err = pcall(dyn.Cardinality_encode, {})
+ t.assert_equals(ok, false)
+ t.assert_str_contains(err, 'required field missing')
+ end
+
+ g.test_dynamic_repeated_unpacked_by_default = function()
+ -- Proto2 default for repeated scalars is NOT packed; the dynamic
+ -- builder must flip the rule based on parsed.syntax.
+ local enc = dyn.Cardinality_encode({r = 0, packed_default = {1, 2, 3}})
+ t.assert_equals(hex(enc), '0800' .. '180118021803')
+ end
+
+ g.test_dynamic_byte_parity_with_static = function()
+ -- Same value through dynamic and static (full mode) must produce
+ -- identical wire bytes.
+ local full = require('full.proto2_basic.proto2_basic_pb')
+ local val = {
+ r = 1, o = 2,
+ packed_default = {3, 4, 5},
+ explicitly_packed = {6, 7, 8},
+ explicitly_unpacked = {9, 10},
+ }
+ t.assert_equals(hex(dyn.Cardinality_encode(val)),
+ hex(full.Cardinality_encode(val)),
+ 'dynamic and static must agree byte-for-byte')
+ end
+end
+
-- Parity: full and runtime modes must produce byte-identical output for
-- the same input. Equivalent to the existing parity.full_vs_runtime group.
do