M PLAN.md => PLAN.md +9 -7
@@ 145,17 145,19 @@ fiber and bridges client ↔ handler via `fiber.channel`. All four flavors
and `test/conformance/known_failures_text.txt` (text-format
suite). Current baseline (2026-05-16):
- Binary+JSON suite: 1478 ✓ / 1313 skipped / 15 expected fails
- - Text-format suite: 2 ✓ / 426 skipped / 6 expected fails
+ - Text-format suite: 8 ✓ / 426 skipped / 0 expected fails
JSON output runs end-to-end; remaining expected fails are
Recommended-only edge cases (FieldMask round-trip,
duplicate-field-name rejection, null-in-collection rejection,
unknown-enum-name rejection, NullValue oneof validator).
- Text-format output is wired through `pb.text.encode`; the six
- expected fails are all unknown-field cases (Group/Repeated parse
- rejection, *_Print not rendering captured unknowns). Text-format
- *input* parsing is still deferred. The `PB_CONFORMANCE_SKIP_JSON=1`
- env var still short-circuits JSON output if a new encoder bug
- crashes jsoncpp.
+ Text-format output is wired through `pb.text.encode` and the
+ proto3 text suite is fully clean: SGROUP/EGROUP are tolerated in
+ `wire.skip_field` and `pb.text` renders captured unknown bytes in
+ numeric field-ID form under `opts.print_unknown_fields`. The 426
+ still-skipped tests are proto2/editions message types we don't
+ register. Text-format *input* parsing is still deferred. The
+ `PB_CONFORMANCE_SKIP_JSON=1` env var still short-circuits JSON
+ output if a new encoder bug crashes jsoncpp.
CI wire-up pending — the image build is the long pole (~10–15 min
on a clean cache).
- [x] Cross-impl interop: 18-fixture corpus in `test/interop/fixtures/`
M cmd/conformance/core.lua => cmd/conformance/core.lua +2 -1
@@ 96,7 96,8 @@ local function dispatch(req)
end
return {json_payload = jbytes}
elseif out_fmt == TEXT_FORMAT then
- local ok, tbytes = pcall(pb.text.encode, desc, msg)
+ local opts = {print_unknown_fields = req.print_unknown_fields == true}
+ local ok, tbytes = pcall(pb.text.encode, desc, msg, opts)
if not ok then
return {serialize_error = 'text encode failed: ' ..
tostring(tbytes)}
M cmd/protoc-gen-tarantool/internal/gen/inline.go => cmd/protoc-gen-tarantool/internal/gen/inline.go +2 -2
@@ 270,7 270,7 @@ func emitInlineDecode(w *writer, name string, m *protogen.Message, file *protoge
w.line(" if true then")
}
w.line(" else")
- w.line(" pos = wire.skip_field(buf, pos, wt)")
+ w.line(" pos = wire.skip_field(buf, pos, wt, id)")
w.line(" if _uf == nil then _uf = {} end")
w.line(" _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)")
w.line(" end")
@@ 527,7 527,7 @@ func emitInlineDecodeMap(w *writer, f *protogen.Field, fname string, file *proto
w.line(" elseif eid == 2 then")
emitMapDecode(w, "_val", valF, file, selfPath, imports, prefix)
w.line(" else")
- w.line(" _ep = wire.skip_field(payload, _ep, ewt)")
+ w.line(" _ep = wire.skip_field(payload, _ep, ewt, eid)")
w.line(" end")
w.line(" end")
if mapKeyNeedsCdataDedup(keyF) {
M examples/expected/full/conformance/conformance_pb.lua => examples/expected/full/conformance/conformance_pb.lua +5 -5
@@ 193,7 193,7 @@ function M.TestStatus_decode(buf)
val, pos = wire.decode_string(buf, pos)
result.matched_name = val
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
if _uf == nil then _uf = {} end
_uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
end
@@ 258,7 258,7 @@ function M.FailureSet_decode(buf)
payload, pos = wire.decode_len(buf, pos)
list[#list + 1] = M.TestStatus_decode(payload)
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
if _uf == nil then _uf = {} end
_uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
end
@@ 439,7 439,7 @@ function M.ConformanceRequest_decode(buf)
val, pos = wire.decode_bool(buf, pos)
result.print_unknown_fields = val
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
if _uf == nil then _uf = {} end
_uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
end
@@ 668,7 668,7 @@ function M.ConformanceResponse_decode(buf)
result.skipped = nil
result.jspb_payload = nil
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
if _uf == nil then _uf = {} end
_uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
end
@@ 726,7 726,7 @@ function M.JspbEncodingConfig_decode(buf)
val, pos = wire.decode_bool(buf, pos)
result.use_jspb_array_any_format = val
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
if _uf == nil then _uf = {} end
_uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
end
M examples/expected/full/hello/hello_pb.lua => examples/expected/full/hello/hello_pb.lua +9 -9
@@ 235,7 235,7 @@ function M.Result_decode(buf)
result.text = nil
result.code = nil
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
if _uf == nil then _uf = {} end
_uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
end
@@ 294,7 294,7 @@ function M.HelloRequest_decode(buf)
val, pos = wire.decode_string(buf, pos)
result.name = val
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
if _uf == nil then _uf = {} end
_uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
end
@@ 353,7 353,7 @@ function M.HelloReply_decode(buf)
val, pos = wire.decode_string(buf, pos)
result.greeting = val
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
if _uf == nil then _uf = {} end
_uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
end
@@ 544,7 544,7 @@ function M.Event_decode(buf)
payload, pos = wire.decode_len(buf, pos)
result.update_mask = pb.wkt.FieldMask_decode(payload)
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
if _uf == nil then _uf = {} end
_uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
end
@@ 635,7 635,7 @@ function M.Address_decode(buf)
val, pos = wire.decode_string(buf, pos)
result.apartment = val
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
if _uf == nil then _uf = {} end
_uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
end
@@ 915,7 915,7 @@ function M.Person_decode(buf)
elseif eid == 2 then
_val, _ep = wire.decode_int32(payload, _ep)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
map[_key] = _val
@@ 934,7 934,7 @@ function M.Person_decode(buf)
elseif eid == 2 then
_val, _ep = wire.decode_string(payload, _ep)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
map[_key] = _val
@@ 955,12 955,12 @@ function M.Person_decode(buf)
_payload, _ep = wire.decode_len(payload, _ep)
_val = M.Address_decode(_payload)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
map[_key] = _val
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
if _uf == nil then _uf = {} end
_uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
end
M examples/expected/full/protobuf_test_messages/proto3/test_messages_proto3_pb.lua => examples/expected/full/protobuf_test_messages/proto3/test_messages_proto3_pb.lua +24 -24
@@ 2936,7 2936,7 @@ function M.TestAllTypesProto3_decode(buf)
elseif eid == 2 then
_val, _ep = wire.decode_int32(payload, _ep)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
map[_key] = _val
@@ 2955,7 2955,7 @@ function M.TestAllTypesProto3_decode(buf)
elseif eid == 2 then
_val, _ep = wire.decode_int64(payload, _ep)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
for _k in pairs(map) do
@@ 2977,7 2977,7 @@ function M.TestAllTypesProto3_decode(buf)
elseif eid == 2 then
_val, _ep = wire.decode_uint32(payload, _ep)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
map[_key] = _val
@@ 2996,7 2996,7 @@ function M.TestAllTypesProto3_decode(buf)
elseif eid == 2 then
_val, _ep = wire.decode_uint64(payload, _ep)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
for _k in pairs(map) do
@@ 3018,7 3018,7 @@ function M.TestAllTypesProto3_decode(buf)
elseif eid == 2 then
_val, _ep = wire.decode_sint32(payload, _ep)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
map[_key] = _val
@@ 3037,7 3037,7 @@ function M.TestAllTypesProto3_decode(buf)
elseif eid == 2 then
_val, _ep = wire.decode_sint64(payload, _ep)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
for _k in pairs(map) do
@@ 3059,7 3059,7 @@ function M.TestAllTypesProto3_decode(buf)
elseif eid == 2 then
_val, _ep = wire.decode_fixed32(payload, _ep)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
map[_key] = _val
@@ 3078,7 3078,7 @@ function M.TestAllTypesProto3_decode(buf)
elseif eid == 2 then
_val, _ep = wire.decode_fixed64(payload, _ep)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
for _k in pairs(map) do
@@ 3100,7 3100,7 @@ function M.TestAllTypesProto3_decode(buf)
elseif eid == 2 then
_val, _ep = wire.decode_sfixed32(payload, _ep)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
map[_key] = _val
@@ 3119,7 3119,7 @@ function M.TestAllTypesProto3_decode(buf)
elseif eid == 2 then
_val, _ep = wire.decode_sfixed64(payload, _ep)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
for _k in pairs(map) do
@@ 3141,7 3141,7 @@ function M.TestAllTypesProto3_decode(buf)
elseif eid == 2 then
_val, _ep = wire.decode_float(payload, _ep)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
map[_key] = _val
@@ 3160,7 3160,7 @@ function M.TestAllTypesProto3_decode(buf)
elseif eid == 2 then
_val, _ep = wire.decode_double(payload, _ep)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
map[_key] = _val
@@ 3179,7 3179,7 @@ function M.TestAllTypesProto3_decode(buf)
elseif eid == 2 then
_val, _ep = wire.decode_bool(payload, _ep)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
map[_key] = _val
@@ 3198,7 3198,7 @@ function M.TestAllTypesProto3_decode(buf)
elseif eid == 2 then
_val, _ep = wire.decode_string(payload, _ep)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
map[_key] = _val
@@ 3217,7 3217,7 @@ function M.TestAllTypesProto3_decode(buf)
elseif eid == 2 then
_val, _ep = wire.decode_bytes(payload, _ep)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
map[_key] = _val
@@ 3238,7 3238,7 @@ function M.TestAllTypesProto3_decode(buf)
_payload, _ep = wire.decode_len(payload, _ep)
_val = M.TestAllTypesProto3_NestedMessage_decode(_payload)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
map[_key] = _val
@@ 3259,7 3259,7 @@ function M.TestAllTypesProto3_decode(buf)
_payload, _ep = wire.decode_len(payload, _ep)
_val = M.ForeignMessage_decode(_payload)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
map[_key] = _val
@@ 3280,7 3280,7 @@ function M.TestAllTypesProto3_decode(buf)
_u, _ep = wire.decode_varint(payload, _ep)
_val = wire.varint_to_int32(_u)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
map[_key] = _val
@@ 3301,7 3301,7 @@ function M.TestAllTypesProto3_decode(buf)
_u, _ep = wire.decode_varint(payload, _ep)
_val = wire.varint_to_int32(_u)
else
- _ep = wire.skip_field(payload, _ep, ewt)
+ _ep = wire.skip_field(payload, _ep, ewt, eid)
end
end
map[_key] = _val
@@ 3683,7 3683,7 @@ function M.TestAllTypesProto3_decode(buf)
val, pos = wire.decode_int32(buf, pos)
result.Field_name18__ = val
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
if _uf == nil then _uf = {} end
_uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
end
@@ 3758,7 3758,7 @@ function M.TestAllTypesProto3_NestedMessage_decode(buf)
pb.codec.merge_message(M.TestAllTypesProto3_descriptor, prev, M.TestAllTypesProto3_decode(payload))
end
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
if _uf == nil then _uf = {} end
_uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
end
@@ 3816,7 3816,7 @@ function M.ForeignMessage_decode(buf)
val, pos = wire.decode_int32(buf, pos)
result.c = val
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
if _uf == nil then _uf = {} end
_uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
end
@@ 3865,7 3865,7 @@ function M.NullHypothesisProto3_decode(buf)
id, wt, pos = wire.decode_tag(buf, pos)
if true then
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
if _uf == nil then _uf = {} end
_uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
end
@@ 3914,7 3914,7 @@ function M.EnumOnlyProto3_decode(buf)
id, wt, pos = wire.decode_tag(buf, pos)
if true then
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
if _uf == nil then _uf = {} end
_uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
end
M runtime/pb/codec.lua => runtime/pb/codec.lua +2 -2
@@ 813,7 813,7 @@ decode_message = function(desc, buf)
local f = fbi[id]
if f == nil then
-- Unknown field: capture verbatim for round-trip.
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
if unknown == nil then unknown = {} end
unknown[#unknown + 1] = buf:sub(tag_start, pos - 1)
else
@@ 839,7 839,7 @@ decode_message = function(desc, buf)
elseif eid == 2 then
val, ep = decode_one(f.value, payload, ep)
else
- ep = wire.skip_field(payload, ep, ewt)
+ ep = wire.skip_field(payload, ep, ewt, eid)
end
end
if key == nil then key = default_value(f.key) end
M runtime/pb/lazy.lua => runtime/pb/lazy.lua +2 -2
@@ 81,7 81,7 @@ local function index_bytes(desc, bytes)
local tag_start = pos
local id, wt, npos = wire.decode_tag(bytes, pos)
local val_start = npos
- local next_start = wire.skip_field(bytes, npos, wt)
+ local next_start = wire.skip_field(bytes, npos, wt, id)
n = n + 1
s_id[n] = id
s_tag[n] = tag_start
@@ 298,7 298,7 @@ local function decode_map_entry(field, bytes, val_start)
end
end
else
- p = wire.skip_field(bytes, p, ewt)
+ p = wire.skip_field(bytes, p, ewt, eid)
end
end
if key == nil then
M runtime/pb/text.lua => runtime/pb/text.lua +101 -0
@@ 20,6 20,7 @@
local ffi = require('ffi')
local datetime = require('datetime')
local pbwkt = require('pb.wkt')
+local wire = require('pb.wire')
local M = {}
@@ 116,6 117,7 @@ local function new_buf(opts)
chunks = {}, n = 0,
single_line = opts.single_line and true or false,
indent_unit = opts.indent or ' ',
+ print_unknown_fields = opts.print_unknown_fields and true or false,
}
end
@@ 207,6 209,96 @@ end
local WKT_TEXT -- forward (filled below)
+-- ---------------------------------------------------------------------------
+-- Unknown-field rendering (drives `_Print` conformance tests).
+--
+-- Walks the captured _unknown_fields byte stream (tag+value chunks the
+-- decoder stashed for round-trip) and emits each entry in TextFormat
+-- numeric-field form so the conformance harness's TextFormat::Parser can
+-- round-trip the output back under AllowFieldNumber:
+-- VARINT -> "<id>: <uint64>"
+-- I64 -> "<id>: 0x<16-hex>"
+-- LEN -> "<id> { <walk> }" if the payload parses as a sub-message,
+-- else "<id>: <byte-string>"
+-- I32 -> "<id>: 0x<8-hex>"
+-- SGROUP -> "<id> { <walk> }" (recurses until matching EGROUP)
+local walk_unknown -- forward; recurses through SGROUP + LEN(message)
+
+walk_unknown = function(buf, raw, pos, lim, depth, end_field_id)
+ while pos <= lim do
+ local id, wt, npos = wire.decode_tag(raw, pos)
+ pos = npos
+ if wt == wire.WIRE_VARINT then
+ local v
+ v, pos = wire.decode_varint(raw, pos)
+ newline(buf, depth)
+ push(buf, tostring(id)); push(buf, ': '); push(buf, int_to_string(v))
+ elseif wt == wire.WIRE_I64 then
+ if pos + 7 > lim then error("truncated I64 in unknown", 0) end
+ local lo = raw:byte(pos) +
+ raw:byte(pos + 1) * 0x100 +
+ raw:byte(pos + 2) * 0x10000 +
+ raw:byte(pos + 3) * 0x1000000
+ local hi = raw:byte(pos + 4) +
+ raw:byte(pos + 5) * 0x100 +
+ raw:byte(pos + 6) * 0x10000 +
+ raw:byte(pos + 7) * 0x1000000
+ newline(buf, depth)
+ push(buf, tostring(id)); push(buf, ': ')
+ push(buf, string.format('0x%08x%08x', hi, lo))
+ pos = pos + 8
+ elseif wt == wire.WIRE_LEN then
+ local payload
+ payload, pos = wire.decode_len(raw, pos)
+ -- Speculative: render as `<id> { ... }`. Roll back to byte
+ -- form if the payload doesn't parse as a clean sub-message.
+ local n_before = buf.n
+ local ok = pcall(function()
+ emit_block(buf, tostring(id), depth, function(b, d)
+ walk_unknown(b, payload, 1, #payload, d, nil)
+ end)
+ end)
+ if not ok then
+ buf.n = n_before
+ newline(buf, depth)
+ push(buf, tostring(id)); push(buf, ': ')
+ push(buf, escape_string(payload))
+ end
+ elseif wt == wire.WIRE_I32 then
+ if pos + 3 > lim then error("truncated I32 in unknown", 0) end
+ local v = raw:byte(pos) +
+ raw:byte(pos + 1) * 0x100 +
+ raw:byte(pos + 2) * 0x10000 +
+ raw:byte(pos + 3) * 0x1000000
+ newline(buf, depth)
+ push(buf, tostring(id)); push(buf, ': ')
+ push(buf, string.format('0x%08x', v))
+ pos = pos + 4
+ elseif wt == wire.WIRE_SGROUP then
+ -- Recurse into the body; the closure mutates `pos` so the
+ -- outer loop continues past the matching EGROUP.
+ emit_block(buf, tostring(id), depth, function(b, d)
+ pos = walk_unknown(b, raw, pos, lim, d, id)
+ end)
+ elseif wt == wire.WIRE_EGROUP then
+ if end_field_id == nil then
+ error("unexpected EGROUP in unknown stream", 0)
+ end
+ if id ~= end_field_id then
+ error(("EGROUP id %d does not match SGROUP id %d"):
+ format(id, end_field_id), 0)
+ end
+ return pos
+ else
+ error("unknown wire type " .. tostring(wt), 0)
+ end
+ end
+ if end_field_id ~= nil then
+ error("missing EGROUP for field " .. tostring(end_field_id), 0)
+ end
+ return pos
+end
+
emit_message = function(buf, desc, t, depth)
-- Use type() rather than == nil so box.NULL (a nil-equal cdata used as
-- the Value WKT's null_value sentinel) survives the guard.
@@ 228,6 320,15 @@ emit_message = function(buf, desc, t, depth)
emit_field(buf, f, v, depth)
end
end
+ -- Captured unknown bytes go last, mirroring the codec's re-encode
+ -- order. Gated by `opts.print_unknown_fields` to match the protoc
+ -- TextFormat::Printer default (off).
+ if buf.print_unknown_fields then
+ local raw = t._unknown_fields
+ if type(raw) == 'string' and #raw > 0 then
+ walk_unknown(buf, raw, 1, #raw, depth, nil)
+ end
+ end
end
-- ---------------------------------------------------------------------------
M runtime/pb/wire.lua => runtime/pb/wire.lua +30 -2
@@ 9,6 9,11 @@ local M = {}
M.WIRE_VARINT = 0
M.WIRE_I64 = 1
M.WIRE_LEN = 2
+-- Wire 3/4 are proto2 groups. We never emit them, but unknown-field
+-- skip must tolerate them so proto2-shaped payloads round-trip through
+-- a proto3 decoder (conformance suite exercises this).
+M.WIRE_SGROUP = 3
+M.WIRE_EGROUP = 4
M.WIRE_I32 = 5
local UINT64 = ffi.typeof('uint64_t')
@@ 601,9 606,12 @@ M.TYPE_INFO = {
-- ---------------------------------------------------------------------------
-- Skip an unknown field (used by decoder when an unrecognized id appears).
--- skip_field(buf, pos, wire_type) -> new_pos
+-- skip_field(buf, pos, wire_type, field_id) -> new_pos
+--
+-- field_id is required for SGROUP (wire 3) so the closing EGROUP can
+-- be matched. Other wire types ignore it.
-- ---------------------------------------------------------------------------
-local function skip_field(buf, pos, wire_type)
+local function skip_field(buf, pos, wire_type, field_id)
if wire_type == M.WIRE_VARINT then
local b = buf:byte(pos)
if b == nil then error("truncated varint at offset " .. pos, 0) end
@@ 630,6 638,26 @@ local function skip_field(buf, pos, wire_type)
local np = pos + 4
if np > #buf + 1 then error("truncated I32 at offset " .. pos, 0) end
return np
+ elseif wire_type == M.WIRE_SGROUP then
+ -- Read inner tags until the matching EGROUP; recurse on nested
+ -- groups. EGROUP id mismatch is a hard error per spec.
+ if field_id == nil then
+ error("skip_field SGROUP requires field_id", 0)
+ end
+ while true do
+ local iid, iwt
+ iid, iwt, pos = decode_tag(buf, pos)
+ if iwt == M.WIRE_EGROUP then
+ if iid ~= field_id then
+ error(("EGROUP id %d does not match SGROUP id %d"):
+ format(iid, field_id), 0)
+ end
+ return pos
+ end
+ pos = skip_field(buf, pos, iwt, iid)
+ end
+ elseif wire_type == M.WIRE_EGROUP then
+ error("unexpected EGROUP for field " .. tostring(field_id), 0)
end
error("unknown wire type " .. tostring(wire_type), 0)
end
M runtime/pb/wkt.lua => runtime/pb/wkt.lua +9 -9
@@ 80,7 80,7 @@ local function timestamp_decode(buf)
elseif id == 2 then
nanos, pos = wire.decode_int32(buf, pos)
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
end
end
-- datetime.new validates nanos/seconds ranges. Out-of-spec Timestamps
@@ 139,7 139,7 @@ local function duration_decode(buf)
elseif id == 2 then
nanos, pos = wire.decode_int32(buf, pos)
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
end
end
return {seconds = seconds, nanos = nanos}
@@ 217,7 217,7 @@ for _, spec in ipairs(WRAPPERS) do
if id == 1 then
val, pos = decode(buf, pos)
else
- pos = wire.skip_field(buf, pos, wt2)
+ pos = wire.skip_field(buf, pos, wt2, id)
end
end
return val
@@ 374,7 374,7 @@ value_decode = function(buf)
payload, pos = wire.decode_len(buf, pos)
result = list_decode(payload)
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
end
end
return result
@@ 401,12 401,12 @@ struct_decode = function(buf)
vbuf, ep = wire.decode_len(payload, ep)
val = value_decode(vbuf)
else
- ep = wire.skip_field(payload, ep, ewt)
+ ep = wire.skip_field(payload, ep, ewt, eid)
end
end
result[key] = val
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
end
end
return result
@@ 423,7 423,7 @@ list_decode = function(buf)
payload, pos = wire.decode_len(buf, pos)
result[#result + 1] = value_decode(payload)
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
end
end
return result
@@ 485,7 485,7 @@ any_decode = function(buf)
elseif id == 2 then
value, pos = wire.decode_bytes(buf, pos)
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
end
end
return {type_url = type_url, value = value}
@@ 571,7 571,7 @@ local function fieldmask_decode(buf)
s, pos = wire.decode_string(buf, pos)
result[#result + 1] = s
else
- pos = wire.skip_field(buf, pos, wt)
+ pos = wire.skip_field(buf, pos, wt, id)
end
end
return result
M test/conformance/known_failures_text.txt => test/conformance/known_failures_text.txt +7 -22
@@ 1,26 1,11 @@
# conformance_test_runner --text_format_failure_list
#
-# Text-format OUTPUT is now wired through cmd/conformance/core.lua to
-# pb.text.encode (protobuf/JSON input → text output). Input parsing is
-# still deferred — pb.text only encodes — so any test whose payload is
-# text_payload still returns `skipped`.
+# Text-format OUTPUT runs through pb.text.encode (protobuf/JSON input →
+# text output). Group/Repeated unknown-field decode is supported by the
+# SGROUP-recursive `wire.skip_field`, and unknown bytes are rendered in
+# numeric field-ID form when `print_unknown_fields=true`.
#
-# Remaining expected failures all stem from unknown-field handling:
+# Text-format INPUT is still deferred — pb.text remains encode-only —
+# so any test whose payload is `text_payload` returns `skipped`.
#
-# * Group/Repeated *_Drop: our decoder rejects payloads carrying
-# wire types 3/4 (proto2 groups) and length-mismatched repeated
-# unknown bytes, so we never reach the encoder. Expected output is
-# an empty text body, which we'd produce trivially if the decoder
-# accepted the input. Fixing requires skip_field to tolerate groups.
-#
-# * *_Print: with `print_unknown_fields: true` the harness expects the
-# text output to include the original unknown fields. We capture
-# unknown bytes during decode but pb.text doesn't render them, so
-# output comes back empty. Fixing requires the text encoder to walk
-# the captured unknown set.
-Recommended.Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput
-Recommended.Proto3.ProtobufInput.GroupUnknownFields_Print.TextFormatOutput
-Recommended.Proto3.ProtobufInput.MessageUnknownFields_Print.TextFormatOutput
-Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput
-Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Print.TextFormatOutput
-Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Print.TextFormatOutput
+# No expected failures in the proto3 text suite as of the last refresh.
M test/conformance_test.lua => test/conformance_test.lua +32 -40
@@ 1215,8 1215,9 @@ local SCALAR_UNKNOWN = '\xc8\x3e\x7b' -- field 1001 varin
local MESSAGE_UNKNOWN = '\xda\x3e\x02\x08\x6f' -- field 1003 LEN {1:111}
local GROUP_UNKNOWN = '\xe3\x3e\x08\xc1\x02\xe4\x3e' -- field 1004 SGROUP {a:321} EGROUP
-- Repeated builds on Group then appends three repeated_int32 entries.
+-- Tag for field 1011 (varint): 1011*8 = 8088 = 24 | (63 << 7) = `\x98\x3f`.
local REPEATED_UNKNOWN = GROUP_UNKNOWN
- .. '\xd8\x3e\x01\xd8\x3e\x02\xd8\x3e\x03' -- field 1011 varint 1,2,3
+ .. '\x98\x3f\x01\x98\x3f\x02\x98\x3f\x03'
core_g.test_scalar_unknown_fields_drop = function()
-- ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput
@@ 1229,18 1230,14 @@ end
core_g.test_scalar_unknown_fields_print = function()
-- ProtobufInput.ScalarUnknownFields_Print.TextFormatOutput
- -- TARGET: text_payload must round-trip to a message containing the
- -- three unknown fields. Canonical form is numeric IDs, e.g.
- -- 1001: 123\n1002: "hello"\n1006: 1\n
- -- CURRENT: pb.text doesn't walk _unknown_fields, so output is empty.
- -- Pin current behavior; flip this assertion when the renderer lands.
+ -- Numeric field-ID form; LEN payload that doesn't parse as a
+ -- sub-message falls back to byte-string rendering ("hello" here).
local resp = pb_to_text_print_unknowns(SCALAR_UNKNOWN)
t.assert_not(resp.parse_error, resp.parse_error)
t.assert_not(resp.serialize_error, resp.serialize_error)
- t.assert_equals(resp.text_payload, '',
- 'current behavior pins empty output; ' ..
- 'replace with assert_str_contains(...,"1001: 123") when ' ..
- 'pb.text renders captured _unknown_fields')
+ t.assert_str_contains(resp.text_payload, '1001: 123')
+ t.assert_str_contains(resp.text_payload, '1002: "hello"')
+ t.assert_str_contains(resp.text_payload, '1006: 1')
end
core_g.test_message_unknown_fields_drop = function()
@@ 1252,58 1249,53 @@ end
core_g.test_message_unknown_fields_print = function()
-- ProtobufInput.MessageUnknownFields_Print.TextFormatOutput
- -- TARGET text: 1003 {\n 1: 111\n}\n
- -- CURRENT: pb.text drops _unknown_fields, so output is empty.
+ -- LEN payload parses cleanly as a sub-message ({c:111}), so the
+ -- speculative block-form succeeds: "1003 { 1: 111 }".
local resp = pb_to_text_print_unknowns(MESSAGE_UNKNOWN)
t.assert_not(resp.parse_error, resp.parse_error)
- t.assert_equals(resp.text_payload, '',
- 'replace with target "1003 { 1: 111 }" rendering when ' ..
- 'pb.text grows submessage unknown-field support')
+ t.assert_str_contains(resp.text_payload, '1003 {')
+ t.assert_str_contains(resp.text_payload, '1: 111')
end
core_g.test_group_unknown_fields_drop = function()
-- ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput
- -- Wire types 3/4 (SGROUP/EGROUP) — proto3 must tolerate them as
- -- unknown groups. Our skip_field currently errors on wire 3, so the
- -- decoder reports parse_error before reaching the text encoder.
- -- TARGET: parse_error == nil, text_payload == ''.
+ -- Wire 3/4 are proto2 groups; the proto3 decoder must skip them
+ -- (wire.skip_field recurses on SGROUP until matching EGROUP).
local resp = pb_to_text(GROUP_UNKNOWN)
- t.assert_not_equals(resp.parse_error, nil,
- 'current: skip_field rejects SGROUP wire type 3. ' ..
- 'Fix in runtime/pb/wire.lua to recursively skip until matching ' ..
- 'EGROUP, then flip this to assert_not + text_payload == ""')
+ t.assert_not(resp.parse_error, resp.parse_error)
+ t.assert_equals(resp.text_payload, '')
end
core_g.test_group_unknown_fields_print = function()
-- ProtobufInput.GroupUnknownFields_Print.TextFormatOutput
- -- TARGET text: 1004 {\n 1: 321\n}\n (groups render as submessages)
- -- CURRENT: blocked by same SGROUP-skip issue as the Drop variant.
+ -- Group bytes are captured verbatim (SGROUP..EGROUP); the renderer
+ -- recurses through them and emits "1004 { 1: 321 }".
local resp = pb_to_text_print_unknowns(GROUP_UNKNOWN)
- t.assert_not_equals(resp.parse_error, nil,
- 'blocked on SGROUP support in wire.skip_field; once decoder ' ..
- 'accepts groups and pb.text renders unknowns, expect ' ..
- 'text_payload containing "1004 {" and "1: 321"')
+ t.assert_not(resp.parse_error, resp.parse_error)
+ t.assert_str_contains(resp.text_payload, '1004 {')
+ t.assert_str_contains(resp.text_payload, '1: 321')
end
core_g.test_repeated_unknown_fields_drop = function()
-- ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput
- -- Payload prepends a group, so the same SGROUP issue blocks decode.
- -- TARGET: parse_error == nil, text_payload == ''.
+ -- Payload starts with the same group bytes; after SGROUP-skip the
+ -- repeated_int32 unknowns drop cleanly too.
local resp = pb_to_text(REPEATED_UNKNOWN)
- t.assert_not_equals(resp.parse_error, nil,
- 'blocked on SGROUP-skip in wire.lua (group bytes come first)')
+ t.assert_not(resp.parse_error, resp.parse_error)
+ t.assert_equals(resp.text_payload, '')
end
core_g.test_repeated_unknown_fields_print = function()
-- ProtobufInput.RepeatedUnknownFields_Print.TextFormatOutput
- -- TARGET text:
- -- 1004 {\n 1: 321\n}\n1011: 1\n1011: 2\n1011: 3\n
- -- CURRENT: blocked by SGROUP-skip, then by unknown-field rendering.
+ -- Group bytes followed by three repeated-int32 entries. Each varint
+ -- becomes its own "<id>: <value>" line.
local resp = pb_to_text_print_unknowns(REPEATED_UNKNOWN)
- t.assert_not_equals(resp.parse_error, nil,
- 'two-stage fix: (1) accept SGROUP in wire.skip_field, ' ..
- '(2) render captured _unknown_fields from pb.text. ' ..
- 'Then assert_str_contains for "1011: 1", "1011: 2", "1011: 3"')
+ t.assert_not(resp.parse_error, resp.parse_error)
+ t.assert_str_contains(resp.text_payload, '1004 {')
+ t.assert_str_contains(resp.text_payload, '1: 321')
+ t.assert_str_contains(resp.text_payload, '1011: 1')
+ t.assert_str_contains(resp.text_payload, '1011: 2')
+ t.assert_str_contains(resp.text_payload, '1011: 3')
end
-- ---------------------------------------------------------------------------