A runtime/pb/descriptor_pb.lua => runtime/pb/descriptor_pb.lua +151 -0
@@ 0,0 1,151 @@
+-- Hand-built descriptors for the subset of google.protobuf.descriptor needed
+-- to decode FileDescriptorSet wire bytes back into AST form. Used by
+-- pb.from_pb (see runtime/pb/fileset.lua).
+--
+-- We model FieldDescriptorProto.type and .label as int32 scalars instead of
+-- enums; the codec returns the wire integer either way and the translator
+-- maps it to a proto3 type-name string by hand.
+local codec = require('pb.codec')
+
+local M = {}
+
+local function finalize(desc)
+ local fbi, fbn = {}, {}
+ for _, f in ipairs(desc.fields) do
+ fbi[f.id] = f
+ fbn[f.name] = f
+ end
+ desc.field_by_id = fbi
+ desc.field_by_name = fbn
+ codec.compile_writers(desc)
+ codec.compile_readers(desc)
+ return desc
+end
+
+-- FieldOptions: subset.
+M.FieldOptions = {
+ name = 'google.protobuf.FieldOptions',
+ fields = {
+ {name = 'packed', id = 2, kind = 'scalar', proto_type = 'bool'},
+ },
+}
+
+-- MessageOptions: subset.
+M.MessageOptions = {
+ name = 'google.protobuf.MessageOptions',
+ fields = {
+ {name = 'map_entry', id = 7, kind = 'scalar', proto_type = 'bool'},
+ },
+}
+
+-- FieldDescriptorProto.
+M.FieldDescriptorProto = {
+ name = 'google.protobuf.FieldDescriptorProto',
+ fields = {
+ {name = 'name', id = 1, kind = 'scalar', proto_type = 'string'},
+ {name = 'extendee', id = 2, kind = 'scalar', proto_type = 'string'},
+ {name = 'number', id = 3, kind = 'scalar', proto_type = 'int32'},
+ {name = 'label', id = 4, kind = 'scalar', proto_type = 'int32'},
+ {name = 'type', id = 5, kind = 'scalar', proto_type = 'int32'},
+ {name = 'type_name', id = 6, kind = 'scalar', proto_type = 'string'},
+ {name = 'default_value', id = 7, kind = 'scalar', proto_type = 'string'},
+ {name = 'options', id = 8, kind = 'message', message = M.FieldOptions},
+ {name = 'oneof_index', id = 9, kind = 'scalar', proto_type = 'int32'},
+ {name = 'json_name', id = 10, kind = 'scalar', proto_type = 'string'},
+ {name = 'proto3_optional', id = 17, kind = 'scalar', proto_type = 'bool'},
+ },
+}
+
+M.OneofDescriptorProto = {
+ name = 'google.protobuf.OneofDescriptorProto',
+ fields = {
+ {name = 'name', id = 1, kind = 'scalar', proto_type = 'string'},
+ },
+}
+
+M.EnumValueDescriptorProto = {
+ name = 'google.protobuf.EnumValueDescriptorProto',
+ fields = {
+ {name = 'name', id = 1, kind = 'scalar', proto_type = 'string'},
+ {name = 'number', id = 2, kind = 'scalar', proto_type = 'int32'},
+ },
+}
+
+M.EnumDescriptorProto = {
+ name = 'google.protobuf.EnumDescriptorProto',
+ fields = {
+ {name = 'name', id = 1, kind = 'scalar', proto_type = 'string'},
+ {name = 'value', id = 2, kind = 'message',
+ message = M.EnumValueDescriptorProto, repeated = true},
+ },
+}
+
+M.MethodDescriptorProto = {
+ name = 'google.protobuf.MethodDescriptorProto',
+ fields = {
+ {name = 'name', id = 1, kind = 'scalar', proto_type = 'string'},
+ {name = 'input_type', id = 2, kind = 'scalar', proto_type = 'string'},
+ {name = 'output_type', id = 3, kind = 'scalar', proto_type = 'string'},
+ {name = 'client_streaming', id = 5, kind = 'scalar', proto_type = 'bool'},
+ {name = 'server_streaming', id = 6, kind = 'scalar', proto_type = 'bool'},
+ },
+}
+
+M.ServiceDescriptorProto = {
+ name = 'google.protobuf.ServiceDescriptorProto',
+ fields = {
+ {name = 'name', id = 1, kind = 'scalar', proto_type = 'string'},
+ {name = 'method', id = 2, kind = 'message',
+ message = M.MethodDescriptorProto, repeated = true},
+ },
+}
+
+-- DescriptorProto recurses through nested_type. We pre-declare the table,
+-- then patch in the recursive reference.
+M.DescriptorProto = {name = 'google.protobuf.DescriptorProto'}
+M.DescriptorProto.fields = {
+ {name = 'name', id = 1, kind = 'scalar', proto_type = 'string'},
+ {name = 'field', id = 2, kind = 'message', message = M.FieldDescriptorProto, repeated = true},
+ {name = 'nested_type', id = 3, kind = 'message', message = M.DescriptorProto, repeated = true},
+ {name = 'enum_type', id = 4, kind = 'message', message = M.EnumDescriptorProto, repeated = true},
+ {name = 'options', id = 7, kind = 'message', message = M.MessageOptions},
+ {name = 'oneof_decl', id = 8, kind = 'message', message = M.OneofDescriptorProto, repeated = true},
+}
+
+M.FileDescriptorProto = {
+ name = 'google.protobuf.FileDescriptorProto',
+ fields = {
+ {name = 'name', id = 1, kind = 'scalar', proto_type = 'string'},
+ {name = 'package', id = 2, kind = 'scalar', proto_type = 'string'},
+ {name = 'dependency', id = 3, kind = 'scalar', proto_type = 'string', repeated = true, packed = false},
+ {name = 'message_type', id = 4, kind = 'message', message = M.DescriptorProto, repeated = true},
+ {name = 'enum_type', id = 5, kind = 'message', message = M.EnumDescriptorProto, repeated = true},
+ {name = 'service', id = 6, kind = 'message', message = M.ServiceDescriptorProto, repeated = true},
+ {name = 'syntax', id = 12, kind = 'scalar', proto_type = 'string'},
+ },
+}
+
+M.FileDescriptorSet = {
+ name = 'google.protobuf.FileDescriptorSet',
+ fields = {
+ {name = 'file', id = 1, kind = 'message',
+ message = M.FileDescriptorProto, repeated = true},
+ },
+}
+
+-- Finalize bottom-up. Order matters only in that we should finalize a
+-- descriptor before any descriptor that references it as a message field;
+-- the codec's compile_readers walks field.message at compile time.
+finalize(M.FieldOptions)
+finalize(M.MessageOptions)
+finalize(M.FieldDescriptorProto)
+finalize(M.OneofDescriptorProto)
+finalize(M.EnumValueDescriptorProto)
+finalize(M.EnumDescriptorProto)
+finalize(M.MethodDescriptorProto)
+finalize(M.ServiceDescriptorProto)
+finalize(M.DescriptorProto)
+finalize(M.FileDescriptorProto)
+finalize(M.FileDescriptorSet)
+
+return M
A runtime/pb/fileset.lua => runtime/pb/fileset.lua +275 -0
@@ 0,0 1,275 @@
+-- pb.from_pb: build runtime modules from a binary FileDescriptorSet.
+--
+-- Complements pb.parse (which consumes .proto source text) by accepting the
+-- output of `protoc --descriptor_set_out=...`. The pipeline:
+-- bytes -> FileDescriptorSet decoded via descriptor.proto descriptors
+-- -> per-file AST (same shape as pb.parser produces)
+-- -> per-file module via pb.dynamic.build
+--
+-- Returns:
+-- {
+-- files = {[file_name] = module, ...}, -- one entry per FileDescriptorProto
+-- order = {file_name, ...}, -- declaration order
+-- lookup = function(full_name) -> descriptor | nil,
+-- }
+local codec = require('pb.codec')
+local dynamic = require('pb.dynamic')
+local descpb = require('pb.descriptor_pb')
+
+local M = {}
+
+-- FieldDescriptorProto.Type wire values -> proto3 type-name strings.
+local TYPE_NAMES = {
+ [1] = 'double',
+ [2] = 'float',
+ [3] = 'int64',
+ [4] = 'uint64',
+ [5] = 'int32',
+ [6] = 'fixed64',
+ [7] = 'fixed32',
+ [8] = 'bool',
+ [9] = 'string',
+ -- 10 = TYPE_GROUP (proto2 only; not supported)
+ [11] = 'message',
+ [12] = 'bytes',
+ [13] = 'uint32',
+ [14] = 'enum',
+ [15] = 'sfixed32',
+ [16] = 'sfixed64',
+ [17] = 'sint32',
+ [18] = 'sint64',
+}
+
+local LABEL_REPEATED = 3
+
+local function strip_dot(name)
+ if name == nil then return nil end
+ return (name:gsub('^%.', ''))
+end
+
+-- Collect the full names of nested map-entry messages so the parent
+-- translator knows which TYPE_MESSAGE fields are really map fields. Returns
+-- {[full_name] = {key_type=, value_type=}, ...}.
+local function collect_map_entries(msg_proto, scope, out)
+ out = out or {}
+ local full = scope == '' and msg_proto.name or (scope .. '.' .. msg_proto.name)
+ if msg_proto.options and msg_proto.options.map_entry then
+ local key_t, val_t
+ local key_typename, val_typename
+ for _, f in ipairs(msg_proto.field or {}) do
+ local tn = TYPE_NAMES[f.type]
+ if f.name == 'key' then
+ if tn == 'message' or tn == 'enum' then
+ key_typename = strip_dot(f.type_name)
+ else
+ key_t = tn
+ end
+ elseif f.name == 'value' then
+ if tn == 'message' or tn == 'enum' then
+ val_typename = strip_dot(f.type_name)
+ else
+ val_t = tn
+ end
+ end
+ end
+ out[full] = {
+ key_type = key_t or key_typename,
+ value_type = val_t or val_typename,
+ }
+ end
+ for _, nested in ipairs(msg_proto.nested_type or {}) do
+ collect_map_entries(nested, full, out)
+ end
+ return out
+end
+
+-- Translate a FieldDescriptorProto into an AST field.
+local function translate_field(f, oneofs_decl, map_entries)
+ local type_id = f.type
+ local typename = TYPE_NAMES[type_id]
+ local repeated = (f.label == LABEL_REPEATED)
+
+ -- Map fields are represented as repeated synthetic-entry messages.
+ if repeated and typename == 'message' then
+ local entry_full = strip_dot(f.type_name)
+ local map_info = entry_full and map_entries[entry_full]
+ if map_info then
+ return {
+ name = f.name,
+ id = f.number,
+ kind = 'map',
+ key_type = map_info.key_type,
+ value_type = map_info.value_type,
+ }, true -- second return signals "skip the entry message"
+ end
+ end
+
+ -- Resolve the type string the AST consumer expects.
+ local ast_type
+ if typename == 'message' or typename == 'enum' then
+ ast_type = strip_dot(f.type_name)
+ else
+ ast_type = typename
+ end
+
+ local entry = {
+ name = f.name,
+ id = f.number,
+ type = ast_type,
+ }
+ if repeated then
+ entry.repeated = true
+ if f.options and f.options.packed ~= nil then
+ entry.packed = f.options.packed
+ end
+ end
+ if f.proto3_optional then
+ entry.optional = true
+ elseif f.oneof_index ~= nil then
+ -- oneof_index references oneof_decl[]. It is zero-based in
+ -- descriptor.proto wire form, but we get Lua-decoded plain ints.
+ local decl = oneofs_decl[f.oneof_index + 1]
+ if decl then entry.oneof = decl.name end
+ end
+ return entry, false
+end
+
+-- Translate a DescriptorProto into the AST message shape produced by
+-- pb.parser. Returns {name=, fields=, nested_messages=, nested_enums=, oneofs=}.
+local function translate_message(msg_proto, scope, map_entries)
+ local full = scope == '' and msg_proto.name or (scope .. '.' .. msg_proto.name)
+ map_entries = map_entries or collect_map_entries(msg_proto, scope)
+
+ local ast = {
+ name = msg_proto.name,
+ fields = {},
+ nested_messages = {},
+ nested_enums = {},
+ oneofs = {},
+ }
+
+ local oneofs_decl = msg_proto.oneof_decl or {}
+
+ -- Group oneof field names by oneof_decl index, excluding proto3_optional
+ -- synthetic oneofs.
+ local oneof_groups = {} -- {[idx] = {field_names...}}
+ for _, f in ipairs(msg_proto.field or {}) do
+ if f.oneof_index ~= nil and not f.proto3_optional then
+ local idx = f.oneof_index + 1
+ oneof_groups[idx] = oneof_groups[idx] or {}
+ table.insert(oneof_groups[idx], f.name)
+ end
+ end
+
+ for _, f in ipairs(msg_proto.field or {}) do
+ local entry, _ = translate_field(f, oneofs_decl, map_entries)
+ table.insert(ast.fields, entry)
+ end
+
+ -- Emit oneofs in oneof_decl order.
+ for i, decl in ipairs(oneofs_decl) do
+ local names = oneof_groups[i]
+ if names and #names > 0 then
+ table.insert(ast.oneofs, {name = decl.name, fields = names})
+ end
+ end
+
+ -- Nested types — skip map-entry synthetic messages.
+ for _, nested in ipairs(msg_proto.nested_type or {}) do
+ local nested_full = full .. '.' .. nested.name
+ if not map_entries[nested_full] then
+ table.insert(ast.nested_messages,
+ translate_message(nested, full, map_entries))
+ end
+ end
+
+ for _, en in ipairs(msg_proto.enum_type or {}) do
+ local values = {}
+ for _, v in ipairs(en.value or {}) do values[v.name] = v.number or 0 end
+ table.insert(ast.nested_enums, {name = en.name, values = values})
+ end
+
+ return ast
+end
+
+-- Translate a FileDescriptorProto into the AST shape pb.dynamic.build expects.
+local function translate_file(file_proto)
+ local ast = {
+ syntax = file_proto.syntax ~= '' and file_proto.syntax or 'proto3',
+ package = file_proto.package or '',
+ imports = {},
+ messages = {},
+ enums = {},
+ services = {},
+ }
+ for _, dep in ipairs(file_proto.dependency or {}) do
+ table.insert(ast.imports, dep)
+ end
+
+ -- Gather top-level map entries (none, by construction — map entries are
+ -- always nested — but we keep the recursion uniform).
+ for _, msg in ipairs(file_proto.message_type or {}) do
+ table.insert(ast.messages, translate_message(msg, ast.package))
+ end
+
+ for _, en in ipairs(file_proto.enum_type or {}) do
+ local values = {}
+ for _, v in ipairs(en.value or {}) do values[v.name] = v.number or 0 end
+ table.insert(ast.enums, {name = en.name, values = values})
+ end
+
+ for _, svc in ipairs(file_proto.service or {}) do
+ local methods = {}
+ for _, m in ipairs(svc.method or {}) do
+ table.insert(methods, {
+ name = m.name,
+ input = strip_dot(m.input_type),
+ output = strip_dot(m.output_type),
+ client_streaming = m.client_streaming or false,
+ server_streaming = m.server_streaming or false,
+ })
+ end
+ table.insert(ast.services, {name = svc.name, methods = methods})
+ end
+
+ return ast
+end
+
+-- Build a name-lookup that walks every module and exposes _descriptor
+-- entries by their proto full name.
+local function build_lookup(files, order)
+ local by_full = {}
+ for _, fname in ipairs(order) do
+ local module = files[fname]
+ -- Re-create the descriptor index by iterating over the module's
+ -- _descriptor entries. The descriptor's .name field is the proto
+ -- full name (e.g., "pkg.sub.Foo").
+ for k, v in pairs(module) do
+ if type(k) == 'string' and k:match('_descriptor$') and type(v) == 'table' then
+ if v.name then by_full[v.name] = v end
+ end
+ end
+ end
+ return function(full_name) return by_full[full_name] end
+end
+
+-- Public entry.
+function M.parse(bytes)
+ local set = codec.decode(descpb.FileDescriptorSet, bytes)
+ local files = {}
+ local order = {}
+ for _, f in ipairs(set.file or {}) do
+ local ast = translate_file(f)
+ local module = dynamic.build(ast)
+ local name = f.name ~= '' and f.name or ('file_' .. tostring(#order + 1))
+ files[name] = module
+ table.insert(order, name)
+ end
+ return {
+ files = files,
+ order = order,
+ lookup = build_lookup(files, order),
+ }
+end
+
+return M
M runtime/pb/init.lua => runtime/pb/init.lua +13 -0
@@ 14,6 14,7 @@ local wkt = require('pb.wkt')
local grpc = require('pb.grpc')
local parser = require('pb.parser')
local dynamic = require('pb.dynamic')
+local fileset = require('pb.fileset')
local pbjson = require('pb.json')
local pbtext = require('pb.text')
local lazy = require('pb.lazy')
@@ 57,9 58,21 @@ return {
-- Output shape mirrors what protoc-gen-tarantool emits in `mode=runtime`.
parse = function(source) return dynamic.build(parser.parse(source)) end,
+ -- Build runtime modules from a binary FileDescriptorSet, the output of
+ -- `protoc --descriptor_set_out=...`. Useful for ingesting compiled
+ -- artifacts or gRPC reflection responses without shipping .proto source.
+ --
+ -- local set = pb.from_pb(bytes)
+ -- local hello = set.files['hello.proto']
+ -- local desc = set.lookup('hello.Person')
+ --
+ -- Returns {files = {[name] = module}, order = {names...}, lookup = fn}.
+ from_pb = fileset.parse,
+
-- Low-level access for advanced use.
parser = parser,
dynamic = dynamic,
+ fileset = fileset,
-- proto3 JSON (canonical mapping). pb.json.encode(desc, t) -> string;
-- pb.json.decode(desc, s) -> table.
A test/fileset_test.lua => test/fileset_test.lua +168 -0
@@ 0,0 1,168 @@
+-- pb.from_pb: build runtime modules from a binary FileDescriptorSet.
+--
+-- We shell out to mainline `protoc --descriptor_set_out` at test start to
+-- produce a fresh descriptor set for examples/proto/hello.proto, then
+-- assert that the module pb.from_pb returns produces wire-compatible bytes
+-- when matched against the statically-generated module.
+local t = require('luatest')
+local fio = require('fio')
+local pb = require('pb')
+
+local REPO_ROOT = fio.abspath(fio.pathjoin(
+ fio.dirname(debug.getinfo(1, 'S').source:sub(2)), '..'))
+local PROTO_DIR = fio.pathjoin(REPO_ROOT, 'examples', 'proto')
+local OPTIONS_DIR = fio.pathjoin(REPO_ROOT, 'options')
+local FIXTURE_PATH = fio.pathjoin(fio.tempdir(), 'hello.descpb')
+
+local function slurp(path)
+ local f = assert(io.open(path, 'rb'))
+ local s = f:read('*a')
+ f:close()
+ return s
+end
+
+-- Materialize once for all tests in this file.
+local SET_BYTES
+do
+ local cmd = string.format(
+ 'protoc --descriptor_set_out=%q -I %q -I %q %q',
+ FIXTURE_PATH, PROTO_DIR, OPTIONS_DIR,
+ fio.pathjoin(PROTO_DIR, 'hello.proto'))
+ local ok = os.execute(cmd)
+ assert(ok == 0 or ok == true,
+ 'protoc --descriptor_set_out failed: ' .. cmd)
+ SET_BYTES = slurp(FIXTURE_PATH)
+end
+
+local g = t.group('fileset')
+
+-- ---- structure ---------------------------------------------------------
+
+g.test_set_top_level_shape = function()
+ local set = pb.from_pb(SET_BYTES)
+ t.assert_type(set.files, 'table')
+ t.assert_type(set.order, 'table')
+ t.assert_type(set.lookup, 'function')
+ t.assert_equals(#set.order, 1)
+ -- protoc encodes paths relative to the -I argument that matched.
+ t.assert_str_contains(set.order[1], 'hello.proto')
+end
+
+g.test_module_has_expected_descriptors = function()
+ local set = pb.from_pb(SET_BYTES)
+ local m = set.files[set.order[1]]
+ t.assert_type(m.Person_descriptor, 'table')
+ t.assert_type(m.Address_descriptor, 'table')
+ t.assert_type(m.Result_descriptor, 'table')
+ t.assert_type(m.Event_descriptor, 'table')
+ t.assert_type(m.Status_descriptor, 'table')
+ t.assert_type(m.Person_encode, 'function')
+ t.assert_type(m.Person_decode, 'function')
+end
+
+g.test_lookup_by_full_name = function()
+ local set = pb.from_pb(SET_BYTES)
+ local desc = set.lookup('hello.Person')
+ t.assert_type(desc, 'table')
+ t.assert_equals(desc.name, 'hello.Person')
+ t.assert_equals(set.lookup('does.not.Exist'), nil)
+end
+
+-- ---- parity with the statically-generated module ----------------------
+
+local function static() return require('full.hello.hello_pb') end
+local function dyn()
+ local set = pb.from_pb(SET_BYTES)
+ return set.files[set.order[1]]
+end
+
+g.test_parity_scalars = function()
+ local input = {street = 'Pushkina 1', city = 'Moscow', zip = 123456}
+ t.assert_equals(dyn().Address_encode(input), static().Address_encode(input))
+end
+
+g.test_parity_repeated_packed = function()
+ local input = {name = 'P', lucky_numbers = {1, 2, 3, 4, 5}}
+ t.assert_equals(dyn().Person_encode(input), static().Person_encode(input))
+end
+
+g.test_parity_optional_field_presence = function()
+ -- Empty `apartment` is meaningful and must round-trip.
+ local input = {street = 'Main', apartment = ''}
+ t.assert_equals(dyn().Address_encode(input), static().Address_encode(input))
+end
+
+g.test_parity_oneof = function()
+ local input = {id = 7, text = 'hi'}
+ t.assert_equals(dyn().Result_encode(input), static().Result_encode(input))
+end
+
+g.test_parity_oneof_message = function()
+ local input = {id = 3, details = {street = 'X', zip = 99}}
+ t.assert_equals(dyn().Result_encode(input), static().Result_encode(input))
+end
+
+g.test_parity_map_scalar = function()
+ -- Single-key map; multi-key bytes depend on Lua hash order (see
+ -- "Map fixtures are fragile" note in CLAUDE.md), so use one entry.
+ local input = {name = 'P', ages_by_nickname = {alice = 30}}
+ t.assert_equals(dyn().Person_encode(input), static().Person_encode(input))
+end
+
+g.test_parity_map_message_value = function()
+ local input = {name = 'P', addresses_by_label = {home = {street = 'Main', zip = 1}}}
+ t.assert_equals(dyn().Person_encode(input), static().Person_encode(input))
+end
+
+g.test_parity_self_reference = function()
+ local input = {
+ name = 'P',
+ friends = {{name = 'A'}, {name = 'B', age = 30}},
+ }
+ t.assert_equals(dyn().Person_encode(input), static().Person_encode(input))
+end
+
+g.test_parity_enum_by_name = function()
+ local s = static()
+ local input = {name = 'P', status = s.Status.ERROR}
+ t.assert_equals(dyn().Person_encode(input), s.Person_encode(input))
+end
+
+-- ---- WKT references resolved through pb.wkt ----------------------------
+
+g.test_parity_wkt_timestamp_table = function()
+ local input = {title = 'x', created_at = {seconds = 1700000000, nanos = 5}}
+ t.assert_equals(dyn().Event_encode(input), static().Event_encode(input))
+end
+
+g.test_parity_wkt_wrappers = function()
+ local input = {title = 'x', retry_count = 7, note = 'hello', is_admin = true}
+ t.assert_equals(dyn().Event_encode(input), static().Event_encode(input))
+end
+
+g.test_parity_wkt_fieldmask = function()
+ local input = {title = 'x', update_mask = {'a', 'b', 'c'}}
+ t.assert_equals(dyn().Event_encode(input), static().Event_encode(input))
+end
+
+g.test_parity_wkt_empty = function()
+ local input = {title = 'x', ack = {}}
+ t.assert_equals(dyn().Event_encode(input), static().Event_encode(input))
+end
+
+-- ---- round-trip via dynamic decode then static decode ------------------
+
+g.test_round_trip_through_dynamic = function()
+ -- Encode with the static module, decode with the dynamic one, re-encode
+ -- with the dynamic one; check the bytes match.
+ local s = static()
+ local input = {
+ name = 'P', age = 33, emails = {'a@x', 'b@y'},
+ status = s.Status.OK,
+ address = {street = 'Main', city = 'X', zip = 1},
+ lucky_numbers = {1, 2, 3},
+ }
+ local bytes = s.Person_encode(input)
+ local decoded = dyn().Person_decode(bytes)
+ t.assert_equals(dyn().Person_encode(decoded), bytes)
+end