From 343e4ba4d5e8d8b6e1a046b5db40d8e58ca3eeb7 Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Sat, 16 May 2026 10:12:46 +0300 Subject: [PATCH] text: render captured unknown fields, tolerate SGROUP in skip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes close the proto3 text-format conformance suite: 1. `wire.skip_field` learns SGROUP/EGROUP. Wire 3 recurses through inner tags until a matching EGROUP, with field_id checked against the SGROUP's id. Callers (codec.lua, lazy.lua, wkt.lua, generated full-mode `_pb.lua`) now pass the tag's field_id so groups inside unknown-field skips don't error. 2. `pb.text.encode` walks the captured `_unknown_fields` buffer when `opts.print_unknown_fields=true` and emits each entry in TextFormat numeric-field form: VARINT -> ": " I64/I32 -> ": 0x" LEN -> speculative " { }"; rolls back to byte-string form if the inner bytes don't parse as a sub-message SGROUP -> " { }" through matching EGROUP `cmd/conformance/core.lua` threads `req.print_unknown_fields` into `pb.text.encode` so `_Drop` tests drop unknowns and `_Print` tests render them. Conformance: text-format suite goes from 2 ✓ / 6 expected fails to 8 ✓ / 0 expected fails. All eight regression tests in `conformance_test.lua` (one per upstream test, plus the fixed field-1011 tag bytes that were miscomputed earlier) now assert the target output. --- PLAN.md | 16 +-- cmd/conformance/core.lua | 3 +- .../internal/gen/inline.go | 4 +- .../full/conformance/conformance_pb.lua | 10 +- examples/expected/full/hello/hello_pb.lua | 18 ++-- .../proto3/test_messages_proto3_pb.lua | 48 ++++----- runtime/pb/codec.lua | 4 +- runtime/pb/lazy.lua | 4 +- runtime/pb/text.lua | 101 ++++++++++++++++++ runtime/pb/wire.lua | 32 +++++- runtime/pb/wkt.lua | 18 ++-- test/conformance/known_failures_text.txt | 29 ++--- test/conformance_test.lua | 72 ++++++------- 13 files changed, 234 insertions(+), 125 deletions(-) diff --git a/PLAN.md b/PLAN.md index 09cbc6c00dd8d30f031ff64c49c6670bc8605168..24e417d5d6b3c8727fde6361080d18aad37fab8b 100644 --- a/PLAN.md +++ b/PLAN.md @@ -145,17 +145,19 @@ fiber and bridges client ↔ handler via `fiber.channel`. All four flavors and `test/conformance/known_failures_text.txt` (text-format suite). Current baseline (2026-05-16): - Binary+JSON suite: 1478 ✓ / 1313 skipped / 15 expected fails - - Text-format suite: 2 ✓ / 426 skipped / 6 expected fails + - Text-format suite: 8 ✓ / 426 skipped / 0 expected fails JSON output runs end-to-end; remaining expected fails are Recommended-only edge cases (FieldMask round-trip, duplicate-field-name rejection, null-in-collection rejection, unknown-enum-name rejection, NullValue oneof validator). - Text-format output is wired through `pb.text.encode`; the six - expected fails are all unknown-field cases (Group/Repeated parse - rejection, *_Print not rendering captured unknowns). Text-format - *input* parsing is still deferred. The `PB_CONFORMANCE_SKIP_JSON=1` - env var still short-circuits JSON output if a new encoder bug - crashes jsoncpp. + Text-format output is wired through `pb.text.encode` and the + proto3 text suite is fully clean: SGROUP/EGROUP are tolerated in + `wire.skip_field` and `pb.text` renders captured unknown bytes in + numeric field-ID form under `opts.print_unknown_fields`. The 426 + still-skipped tests are proto2/editions message types we don't + register. Text-format *input* parsing is still deferred. The + `PB_CONFORMANCE_SKIP_JSON=1` env var still short-circuits JSON + output if a new encoder bug crashes jsoncpp. CI wire-up pending — the image build is the long pole (~10–15 min on a clean cache). - [x] Cross-impl interop: 18-fixture corpus in `test/interop/fixtures/` diff --git a/cmd/conformance/core.lua b/cmd/conformance/core.lua index ff267c4aadbba3968358588097ce9d7f6ba7002b..667016f967145987b256e30bcb1c40f8953aac2d 100644 --- a/cmd/conformance/core.lua +++ b/cmd/conformance/core.lua @@ -96,7 +96,8 @@ local function dispatch(req) end return {json_payload = jbytes} elseif out_fmt == TEXT_FORMAT then - local ok, tbytes = pcall(pb.text.encode, desc, msg) + local opts = {print_unknown_fields = req.print_unknown_fields == true} + local ok, tbytes = pcall(pb.text.encode, desc, msg, opts) if not ok then return {serialize_error = 'text encode failed: ' .. tostring(tbytes)} diff --git a/cmd/protoc-gen-tarantool/internal/gen/inline.go b/cmd/protoc-gen-tarantool/internal/gen/inline.go index a14801529e0f901d4faf50072f48dcf0c6a2ffac..b37029aee3afafbd74f985664ee3150d3c80b002 100644 --- a/cmd/protoc-gen-tarantool/internal/gen/inline.go +++ b/cmd/protoc-gen-tarantool/internal/gen/inline.go @@ -270,7 +270,7 @@ func emitInlineDecode(w *writer, name string, m *protogen.Message, file *protoge w.line(" if true then") } w.line(" else") - w.line(" pos = wire.skip_field(buf, pos, wt)") + w.line(" pos = wire.skip_field(buf, pos, wt, id)") w.line(" if _uf == nil then _uf = {} end") w.line(" _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)") w.line(" end") @@ -527,7 +527,7 @@ func emitInlineDecodeMap(w *writer, f *protogen.Field, fname string, file *proto w.line(" elseif eid == 2 then") emitMapDecode(w, "_val", valF, file, selfPath, imports, prefix) w.line(" else") - w.line(" _ep = wire.skip_field(payload, _ep, ewt)") + w.line(" _ep = wire.skip_field(payload, _ep, ewt, eid)") w.line(" end") w.line(" end") if mapKeyNeedsCdataDedup(keyF) { diff --git a/examples/expected/full/conformance/conformance_pb.lua b/examples/expected/full/conformance/conformance_pb.lua index 12439fb41ac9f9dd9caca5745154b08d6dba85b0..b0986f5f9df42d297cd6b5a7cb29ebeb4c635f0f 100644 --- a/examples/expected/full/conformance/conformance_pb.lua +++ b/examples/expected/full/conformance/conformance_pb.lua @@ -193,7 +193,7 @@ function M.TestStatus_decode(buf) val, pos = wire.decode_string(buf, pos) result.matched_name = val else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) if _uf == nil then _uf = {} end _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) end @@ -258,7 +258,7 @@ function M.FailureSet_decode(buf) payload, pos = wire.decode_len(buf, pos) list[#list + 1] = M.TestStatus_decode(payload) else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) if _uf == nil then _uf = {} end _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) end @@ -439,7 +439,7 @@ function M.ConformanceRequest_decode(buf) val, pos = wire.decode_bool(buf, pos) result.print_unknown_fields = val else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) if _uf == nil then _uf = {} end _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) end @@ -668,7 +668,7 @@ function M.ConformanceResponse_decode(buf) result.skipped = nil result.jspb_payload = nil else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) if _uf == nil then _uf = {} end _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) end @@ -726,7 +726,7 @@ function M.JspbEncodingConfig_decode(buf) val, pos = wire.decode_bool(buf, pos) result.use_jspb_array_any_format = val else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) if _uf == nil then _uf = {} end _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) end diff --git a/examples/expected/full/hello/hello_pb.lua b/examples/expected/full/hello/hello_pb.lua index 1ac6a88a2d490ea66a5dc6dba6791ab0871dbb95..cb8ba6a8263931c3cb05d95204dfbf91fc69783e 100644 --- a/examples/expected/full/hello/hello_pb.lua +++ b/examples/expected/full/hello/hello_pb.lua @@ -235,7 +235,7 @@ function M.Result_decode(buf) result.text = nil result.code = nil else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) if _uf == nil then _uf = {} end _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) end @@ -294,7 +294,7 @@ function M.HelloRequest_decode(buf) val, pos = wire.decode_string(buf, pos) result.name = val else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) if _uf == nil then _uf = {} end _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) end @@ -353,7 +353,7 @@ function M.HelloReply_decode(buf) val, pos = wire.decode_string(buf, pos) result.greeting = val else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) if _uf == nil then _uf = {} end _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) end @@ -544,7 +544,7 @@ function M.Event_decode(buf) payload, pos = wire.decode_len(buf, pos) result.update_mask = pb.wkt.FieldMask_decode(payload) else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) if _uf == nil then _uf = {} end _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) end @@ -635,7 +635,7 @@ function M.Address_decode(buf) val, pos = wire.decode_string(buf, pos) result.apartment = val else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) if _uf == nil then _uf = {} end _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) end @@ -915,7 +915,7 @@ function M.Person_decode(buf) elseif eid == 2 then _val, _ep = wire.decode_int32(payload, _ep) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end map[_key] = _val @@ -934,7 +934,7 @@ function M.Person_decode(buf) elseif eid == 2 then _val, _ep = wire.decode_string(payload, _ep) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end map[_key] = _val @@ -955,12 +955,12 @@ function M.Person_decode(buf) _payload, _ep = wire.decode_len(payload, _ep) _val = M.Address_decode(_payload) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end map[_key] = _val else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) if _uf == nil then _uf = {} end _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) end diff --git a/examples/expected/full/protobuf_test_messages/proto3/test_messages_proto3_pb.lua b/examples/expected/full/protobuf_test_messages/proto3/test_messages_proto3_pb.lua index 759e89fa99d95fe5c4e866c30d811a0275d63fde..99cc5681b9c4ac79ce3dcfbc1879b9a516572f59 100644 --- a/examples/expected/full/protobuf_test_messages/proto3/test_messages_proto3_pb.lua +++ b/examples/expected/full/protobuf_test_messages/proto3/test_messages_proto3_pb.lua @@ -2936,7 +2936,7 @@ function M.TestAllTypesProto3_decode(buf) elseif eid == 2 then _val, _ep = wire.decode_int32(payload, _ep) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end map[_key] = _val @@ -2955,7 +2955,7 @@ function M.TestAllTypesProto3_decode(buf) elseif eid == 2 then _val, _ep = wire.decode_int64(payload, _ep) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end for _k in pairs(map) do @@ -2977,7 +2977,7 @@ function M.TestAllTypesProto3_decode(buf) elseif eid == 2 then _val, _ep = wire.decode_uint32(payload, _ep) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end map[_key] = _val @@ -2996,7 +2996,7 @@ function M.TestAllTypesProto3_decode(buf) elseif eid == 2 then _val, _ep = wire.decode_uint64(payload, _ep) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end for _k in pairs(map) do @@ -3018,7 +3018,7 @@ function M.TestAllTypesProto3_decode(buf) elseif eid == 2 then _val, _ep = wire.decode_sint32(payload, _ep) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end map[_key] = _val @@ -3037,7 +3037,7 @@ function M.TestAllTypesProto3_decode(buf) elseif eid == 2 then _val, _ep = wire.decode_sint64(payload, _ep) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end for _k in pairs(map) do @@ -3059,7 +3059,7 @@ function M.TestAllTypesProto3_decode(buf) elseif eid == 2 then _val, _ep = wire.decode_fixed32(payload, _ep) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end map[_key] = _val @@ -3078,7 +3078,7 @@ function M.TestAllTypesProto3_decode(buf) elseif eid == 2 then _val, _ep = wire.decode_fixed64(payload, _ep) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end for _k in pairs(map) do @@ -3100,7 +3100,7 @@ function M.TestAllTypesProto3_decode(buf) elseif eid == 2 then _val, _ep = wire.decode_sfixed32(payload, _ep) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end map[_key] = _val @@ -3119,7 +3119,7 @@ function M.TestAllTypesProto3_decode(buf) elseif eid == 2 then _val, _ep = wire.decode_sfixed64(payload, _ep) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end for _k in pairs(map) do @@ -3141,7 +3141,7 @@ function M.TestAllTypesProto3_decode(buf) elseif eid == 2 then _val, _ep = wire.decode_float(payload, _ep) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end map[_key] = _val @@ -3160,7 +3160,7 @@ function M.TestAllTypesProto3_decode(buf) elseif eid == 2 then _val, _ep = wire.decode_double(payload, _ep) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end map[_key] = _val @@ -3179,7 +3179,7 @@ function M.TestAllTypesProto3_decode(buf) elseif eid == 2 then _val, _ep = wire.decode_bool(payload, _ep) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end map[_key] = _val @@ -3198,7 +3198,7 @@ function M.TestAllTypesProto3_decode(buf) elseif eid == 2 then _val, _ep = wire.decode_string(payload, _ep) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end map[_key] = _val @@ -3217,7 +3217,7 @@ function M.TestAllTypesProto3_decode(buf) elseif eid == 2 then _val, _ep = wire.decode_bytes(payload, _ep) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end map[_key] = _val @@ -3238,7 +3238,7 @@ function M.TestAllTypesProto3_decode(buf) _payload, _ep = wire.decode_len(payload, _ep) _val = M.TestAllTypesProto3_NestedMessage_decode(_payload) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end map[_key] = _val @@ -3259,7 +3259,7 @@ function M.TestAllTypesProto3_decode(buf) _payload, _ep = wire.decode_len(payload, _ep) _val = M.ForeignMessage_decode(_payload) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end map[_key] = _val @@ -3280,7 +3280,7 @@ function M.TestAllTypesProto3_decode(buf) _u, _ep = wire.decode_varint(payload, _ep) _val = wire.varint_to_int32(_u) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end map[_key] = _val @@ -3301,7 +3301,7 @@ function M.TestAllTypesProto3_decode(buf) _u, _ep = wire.decode_varint(payload, _ep) _val = wire.varint_to_int32(_u) else - _ep = wire.skip_field(payload, _ep, ewt) + _ep = wire.skip_field(payload, _ep, ewt, eid) end end map[_key] = _val @@ -3683,7 +3683,7 @@ function M.TestAllTypesProto3_decode(buf) val, pos = wire.decode_int32(buf, pos) result.Field_name18__ = val else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) if _uf == nil then _uf = {} end _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) end @@ -3758,7 +3758,7 @@ function M.TestAllTypesProto3_NestedMessage_decode(buf) pb.codec.merge_message(M.TestAllTypesProto3_descriptor, prev, M.TestAllTypesProto3_decode(payload)) end else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) if _uf == nil then _uf = {} end _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) end @@ -3816,7 +3816,7 @@ function M.ForeignMessage_decode(buf) val, pos = wire.decode_int32(buf, pos) result.c = val else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) if _uf == nil then _uf = {} end _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) end @@ -3865,7 +3865,7 @@ function M.NullHypothesisProto3_decode(buf) id, wt, pos = wire.decode_tag(buf, pos) if true then else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) if _uf == nil then _uf = {} end _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) end @@ -3914,7 +3914,7 @@ function M.EnumOnlyProto3_decode(buf) id, wt, pos = wire.decode_tag(buf, pos) if true then else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) if _uf == nil then _uf = {} end _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1) end diff --git a/runtime/pb/codec.lua b/runtime/pb/codec.lua index 3bc4eb58fd561b29eca7b3fa63abaa3b03939cad..a9549193bd4e765ee331ae568b0eaf2db65967b8 100644 --- a/runtime/pb/codec.lua +++ b/runtime/pb/codec.lua @@ -813,7 +813,7 @@ decode_message = function(desc, buf) local f = fbi[id] if f == nil then -- Unknown field: capture verbatim for round-trip. - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) if unknown == nil then unknown = {} end unknown[#unknown + 1] = buf:sub(tag_start, pos - 1) else @@ -839,7 +839,7 @@ decode_message = function(desc, buf) elseif eid == 2 then val, ep = decode_one(f.value, payload, ep) else - ep = wire.skip_field(payload, ep, ewt) + ep = wire.skip_field(payload, ep, ewt, eid) end end if key == nil then key = default_value(f.key) end diff --git a/runtime/pb/lazy.lua b/runtime/pb/lazy.lua index 91d3366592ee05a64b2c120e1dc10dfb6e29e99a..3ae02cab15a96f2bb784c59cba83f949cc6151db 100644 --- a/runtime/pb/lazy.lua +++ b/runtime/pb/lazy.lua @@ -81,7 +81,7 @@ local function index_bytes(desc, bytes) local tag_start = pos local id, wt, npos = wire.decode_tag(bytes, pos) local val_start = npos - local next_start = wire.skip_field(bytes, npos, wt) + local next_start = wire.skip_field(bytes, npos, wt, id) n = n + 1 s_id[n] = id s_tag[n] = tag_start @@ -298,7 +298,7 @@ local function decode_map_entry(field, bytes, val_start) end end else - p = wire.skip_field(bytes, p, ewt) + p = wire.skip_field(bytes, p, ewt, eid) end end if key == nil then diff --git a/runtime/pb/text.lua b/runtime/pb/text.lua index 9a78629d755e4d81925fe19d745f4f19ec62a383..a9a5465d6251b0b1c08c7a0cfb9f0dac879e5e38 100644 --- a/runtime/pb/text.lua +++ b/runtime/pb/text.lua @@ -20,6 +20,7 @@ local ffi = require('ffi') local datetime = require('datetime') local pbwkt = require('pb.wkt') +local wire = require('pb.wire') local M = {} @@ -116,6 +117,7 @@ local function new_buf(opts) chunks = {}, n = 0, single_line = opts.single_line and true or false, indent_unit = opts.indent or ' ', + print_unknown_fields = opts.print_unknown_fields and true or false, } end @@ -207,6 +209,96 @@ end local WKT_TEXT -- forward (filled below) +-- --------------------------------------------------------------------------- +-- Unknown-field rendering (drives `_Print` conformance tests). +-- +-- Walks the captured _unknown_fields byte stream (tag+value chunks the +-- decoder stashed for round-trip) and emits each entry in TextFormat +-- numeric-field form so the conformance harness's TextFormat::Parser can +-- round-trip the output back under AllowFieldNumber: +-- VARINT -> ": " +-- I64 -> ": 0x<16-hex>" +-- LEN -> " { }" if the payload parses as a sub-message, +-- else ": " +-- I32 -> ": 0x<8-hex>" +-- SGROUP -> " { }" (recurses until matching EGROUP) +local walk_unknown -- forward; recurses through SGROUP + LEN(message) + +walk_unknown = function(buf, raw, pos, lim, depth, end_field_id) + while pos <= lim do + local id, wt, npos = wire.decode_tag(raw, pos) + pos = npos + if wt == wire.WIRE_VARINT then + local v + v, pos = wire.decode_varint(raw, pos) + newline(buf, depth) + push(buf, tostring(id)); push(buf, ': '); push(buf, int_to_string(v)) + elseif wt == wire.WIRE_I64 then + if pos + 7 > lim then error("truncated I64 in unknown", 0) end + local lo = raw:byte(pos) + + raw:byte(pos + 1) * 0x100 + + raw:byte(pos + 2) * 0x10000 + + raw:byte(pos + 3) * 0x1000000 + local hi = raw:byte(pos + 4) + + raw:byte(pos + 5) * 0x100 + + raw:byte(pos + 6) * 0x10000 + + raw:byte(pos + 7) * 0x1000000 + newline(buf, depth) + push(buf, tostring(id)); push(buf, ': ') + push(buf, string.format('0x%08x%08x', hi, lo)) + pos = pos + 8 + elseif wt == wire.WIRE_LEN then + local payload + payload, pos = wire.decode_len(raw, pos) + -- Speculative: render as ` { ... }`. Roll back to byte + -- form if the payload doesn't parse as a clean sub-message. + local n_before = buf.n + local ok = pcall(function() + emit_block(buf, tostring(id), depth, function(b, d) + walk_unknown(b, payload, 1, #payload, d, nil) + end) + end) + if not ok then + buf.n = n_before + newline(buf, depth) + push(buf, tostring(id)); push(buf, ': ') + push(buf, escape_string(payload)) + end + elseif wt == wire.WIRE_I32 then + if pos + 3 > lim then error("truncated I32 in unknown", 0) end + local v = raw:byte(pos) + + raw:byte(pos + 1) * 0x100 + + raw:byte(pos + 2) * 0x10000 + + raw:byte(pos + 3) * 0x1000000 + newline(buf, depth) + push(buf, tostring(id)); push(buf, ': ') + push(buf, string.format('0x%08x', v)) + pos = pos + 4 + elseif wt == wire.WIRE_SGROUP then + -- Recurse into the body; the closure mutates `pos` so the + -- outer loop continues past the matching EGROUP. + emit_block(buf, tostring(id), depth, function(b, d) + pos = walk_unknown(b, raw, pos, lim, d, id) + end) + elseif wt == wire.WIRE_EGROUP then + if end_field_id == nil then + error("unexpected EGROUP in unknown stream", 0) + end + if id ~= end_field_id then + error(("EGROUP id %d does not match SGROUP id %d"): + format(id, end_field_id), 0) + end + return pos + else + error("unknown wire type " .. tostring(wt), 0) + end + end + if end_field_id ~= nil then + error("missing EGROUP for field " .. tostring(end_field_id), 0) + end + return pos +end + emit_message = function(buf, desc, t, depth) -- Use type() rather than == nil so box.NULL (a nil-equal cdata used as -- the Value WKT's null_value sentinel) survives the guard. @@ -228,6 +320,15 @@ emit_message = function(buf, desc, t, depth) emit_field(buf, f, v, depth) end end + -- Captured unknown bytes go last, mirroring the codec's re-encode + -- order. Gated by `opts.print_unknown_fields` to match the protoc + -- TextFormat::Printer default (off). + if buf.print_unknown_fields then + local raw = t._unknown_fields + if type(raw) == 'string' and #raw > 0 then + walk_unknown(buf, raw, 1, #raw, depth, nil) + end + end end -- --------------------------------------------------------------------------- diff --git a/runtime/pb/wire.lua b/runtime/pb/wire.lua index 6a9f58a49d67483a7a4719ddd01d443ebc435c80..3675156c629b055f0f81e89d416be024d8d74c0a 100644 --- a/runtime/pb/wire.lua +++ b/runtime/pb/wire.lua @@ -9,6 +9,11 @@ local M = {} M.WIRE_VARINT = 0 M.WIRE_I64 = 1 M.WIRE_LEN = 2 +-- Wire 3/4 are proto2 groups. We never emit them, but unknown-field +-- skip must tolerate them so proto2-shaped payloads round-trip through +-- a proto3 decoder (conformance suite exercises this). +M.WIRE_SGROUP = 3 +M.WIRE_EGROUP = 4 M.WIRE_I32 = 5 local UINT64 = ffi.typeof('uint64_t') @@ -601,9 +606,12 @@ M.TYPE_INFO = { -- --------------------------------------------------------------------------- -- Skip an unknown field (used by decoder when an unrecognized id appears). --- skip_field(buf, pos, wire_type) -> new_pos +-- skip_field(buf, pos, wire_type, field_id) -> new_pos +-- +-- field_id is required for SGROUP (wire 3) so the closing EGROUP can +-- be matched. Other wire types ignore it. -- --------------------------------------------------------------------------- -local function skip_field(buf, pos, wire_type) +local function skip_field(buf, pos, wire_type, field_id) if wire_type == M.WIRE_VARINT then local b = buf:byte(pos) if b == nil then error("truncated varint at offset " .. pos, 0) end @@ -630,6 +638,26 @@ local function skip_field(buf, pos, wire_type) local np = pos + 4 if np > #buf + 1 then error("truncated I32 at offset " .. pos, 0) end return np + elseif wire_type == M.WIRE_SGROUP then + -- Read inner tags until the matching EGROUP; recurse on nested + -- groups. EGROUP id mismatch is a hard error per spec. + if field_id == nil then + error("skip_field SGROUP requires field_id", 0) + end + while true do + local iid, iwt + iid, iwt, pos = decode_tag(buf, pos) + if iwt == M.WIRE_EGROUP then + if iid ~= field_id then + error(("EGROUP id %d does not match SGROUP id %d"): + format(iid, field_id), 0) + end + return pos + end + pos = skip_field(buf, pos, iwt, iid) + end + elseif wire_type == M.WIRE_EGROUP then + error("unexpected EGROUP for field " .. tostring(field_id), 0) end error("unknown wire type " .. tostring(wire_type), 0) end diff --git a/runtime/pb/wkt.lua b/runtime/pb/wkt.lua index c13bda34c6cdb022b2fb1907acb71bd49908fa2d..c68f6678db9ef01ca0fc975bafccc39d7068da38 100644 --- a/runtime/pb/wkt.lua +++ b/runtime/pb/wkt.lua @@ -80,7 +80,7 @@ local function timestamp_decode(buf) elseif id == 2 then nanos, pos = wire.decode_int32(buf, pos) else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) end end -- datetime.new validates nanos/seconds ranges. Out-of-spec Timestamps @@ -139,7 +139,7 @@ local function duration_decode(buf) elseif id == 2 then nanos, pos = wire.decode_int32(buf, pos) else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) end end return {seconds = seconds, nanos = nanos} @@ -217,7 +217,7 @@ for _, spec in ipairs(WRAPPERS) do if id == 1 then val, pos = decode(buf, pos) else - pos = wire.skip_field(buf, pos, wt2) + pos = wire.skip_field(buf, pos, wt2, id) end end return val @@ -374,7 +374,7 @@ value_decode = function(buf) payload, pos = wire.decode_len(buf, pos) result = list_decode(payload) else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) end end return result @@ -401,12 +401,12 @@ struct_decode = function(buf) vbuf, ep = wire.decode_len(payload, ep) val = value_decode(vbuf) else - ep = wire.skip_field(payload, ep, ewt) + ep = wire.skip_field(payload, ep, ewt, eid) end end result[key] = val else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) end end return result @@ -423,7 +423,7 @@ list_decode = function(buf) payload, pos = wire.decode_len(buf, pos) result[#result + 1] = value_decode(payload) else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) end end return result @@ -485,7 +485,7 @@ any_decode = function(buf) elseif id == 2 then value, pos = wire.decode_bytes(buf, pos) else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) end end return {type_url = type_url, value = value} @@ -571,7 +571,7 @@ local function fieldmask_decode(buf) s, pos = wire.decode_string(buf, pos) result[#result + 1] = s else - pos = wire.skip_field(buf, pos, wt) + pos = wire.skip_field(buf, pos, wt, id) end end return result diff --git a/test/conformance/known_failures_text.txt b/test/conformance/known_failures_text.txt index b1c89e3148828b4823151c9b423848efb5187196..84afa733695a7cc3c33204983f736891e2876908 100644 --- a/test/conformance/known_failures_text.txt +++ b/test/conformance/known_failures_text.txt @@ -1,26 +1,11 @@ # conformance_test_runner --text_format_failure_list # -# Text-format OUTPUT is now wired through cmd/conformance/core.lua to -# pb.text.encode (protobuf/JSON input → text output). Input parsing is -# still deferred — pb.text only encodes — so any test whose payload is -# text_payload still returns `skipped`. +# Text-format OUTPUT runs through pb.text.encode (protobuf/JSON input → +# text output). Group/Repeated unknown-field decode is supported by the +# SGROUP-recursive `wire.skip_field`, and unknown bytes are rendered in +# numeric field-ID form when `print_unknown_fields=true`. # -# Remaining expected failures all stem from unknown-field handling: +# Text-format INPUT is still deferred — pb.text remains encode-only — +# so any test whose payload is `text_payload` returns `skipped`. # -# * Group/Repeated *_Drop: our decoder rejects payloads carrying -# wire types 3/4 (proto2 groups) and length-mismatched repeated -# unknown bytes, so we never reach the encoder. Expected output is -# an empty text body, which we'd produce trivially if the decoder -# accepted the input. Fixing requires skip_field to tolerate groups. -# -# * *_Print: with `print_unknown_fields: true` the harness expects the -# text output to include the original unknown fields. We capture -# unknown bytes during decode but pb.text doesn't render them, so -# output comes back empty. Fixing requires the text encoder to walk -# the captured unknown set. -Recommended.Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput -Recommended.Proto3.ProtobufInput.GroupUnknownFields_Print.TextFormatOutput -Recommended.Proto3.ProtobufInput.MessageUnknownFields_Print.TextFormatOutput -Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput -Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Print.TextFormatOutput -Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Print.TextFormatOutput +# No expected failures in the proto3 text suite as of the last refresh. diff --git a/test/conformance_test.lua b/test/conformance_test.lua index 8972b1eceb44776e6e4c60272abcdae257e8beff..97bfff753e85520f4e4ab4eac6d73a9542a99eae 100644 --- a/test/conformance_test.lua +++ b/test/conformance_test.lua @@ -1215,8 +1215,9 @@ local SCALAR_UNKNOWN = '\xc8\x3e\x7b' -- field 1001 varin local MESSAGE_UNKNOWN = '\xda\x3e\x02\x08\x6f' -- field 1003 LEN {1:111} local GROUP_UNKNOWN = '\xe3\x3e\x08\xc1\x02\xe4\x3e' -- field 1004 SGROUP {a:321} EGROUP -- Repeated builds on Group then appends three repeated_int32 entries. +-- Tag for field 1011 (varint): 1011*8 = 8088 = 24 | (63 << 7) = `\x98\x3f`. local REPEATED_UNKNOWN = GROUP_UNKNOWN - .. '\xd8\x3e\x01\xd8\x3e\x02\xd8\x3e\x03' -- field 1011 varint 1,2,3 + .. '\x98\x3f\x01\x98\x3f\x02\x98\x3f\x03' core_g.test_scalar_unknown_fields_drop = function() -- ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput @@ -1229,18 +1230,14 @@ end core_g.test_scalar_unknown_fields_print = function() -- ProtobufInput.ScalarUnknownFields_Print.TextFormatOutput - -- TARGET: text_payload must round-trip to a message containing the - -- three unknown fields. Canonical form is numeric IDs, e.g. - -- 1001: 123\n1002: "hello"\n1006: 1\n - -- CURRENT: pb.text doesn't walk _unknown_fields, so output is empty. - -- Pin current behavior; flip this assertion when the renderer lands. + -- Numeric field-ID form; LEN payload that doesn't parse as a + -- sub-message falls back to byte-string rendering ("hello" here). local resp = pb_to_text_print_unknowns(SCALAR_UNKNOWN) t.assert_not(resp.parse_error, resp.parse_error) t.assert_not(resp.serialize_error, resp.serialize_error) - t.assert_equals(resp.text_payload, '', - 'current behavior pins empty output; ' .. - 'replace with assert_str_contains(...,"1001: 123") when ' .. - 'pb.text renders captured _unknown_fields') + t.assert_str_contains(resp.text_payload, '1001: 123') + t.assert_str_contains(resp.text_payload, '1002: "hello"') + t.assert_str_contains(resp.text_payload, '1006: 1') end core_g.test_message_unknown_fields_drop = function() @@ -1252,58 +1249,53 @@ end core_g.test_message_unknown_fields_print = function() -- ProtobufInput.MessageUnknownFields_Print.TextFormatOutput - -- TARGET text: 1003 {\n 1: 111\n}\n - -- CURRENT: pb.text drops _unknown_fields, so output is empty. + -- LEN payload parses cleanly as a sub-message ({c:111}), so the + -- speculative block-form succeeds: "1003 { 1: 111 }". local resp = pb_to_text_print_unknowns(MESSAGE_UNKNOWN) t.assert_not(resp.parse_error, resp.parse_error) - t.assert_equals(resp.text_payload, '', - 'replace with target "1003 { 1: 111 }" rendering when ' .. - 'pb.text grows submessage unknown-field support') + t.assert_str_contains(resp.text_payload, '1003 {') + t.assert_str_contains(resp.text_payload, '1: 111') end core_g.test_group_unknown_fields_drop = function() -- ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput - -- Wire types 3/4 (SGROUP/EGROUP) — proto3 must tolerate them as - -- unknown groups. Our skip_field currently errors on wire 3, so the - -- decoder reports parse_error before reaching the text encoder. - -- TARGET: parse_error == nil, text_payload == ''. + -- Wire 3/4 are proto2 groups; the proto3 decoder must skip them + -- (wire.skip_field recurses on SGROUP until matching EGROUP). local resp = pb_to_text(GROUP_UNKNOWN) - t.assert_not_equals(resp.parse_error, nil, - 'current: skip_field rejects SGROUP wire type 3. ' .. - 'Fix in runtime/pb/wire.lua to recursively skip until matching ' .. - 'EGROUP, then flip this to assert_not + text_payload == ""') + t.assert_not(resp.parse_error, resp.parse_error) + t.assert_equals(resp.text_payload, '') end core_g.test_group_unknown_fields_print = function() -- ProtobufInput.GroupUnknownFields_Print.TextFormatOutput - -- TARGET text: 1004 {\n 1: 321\n}\n (groups render as submessages) - -- CURRENT: blocked by same SGROUP-skip issue as the Drop variant. + -- Group bytes are captured verbatim (SGROUP..EGROUP); the renderer + -- recurses through them and emits "1004 { 1: 321 }". local resp = pb_to_text_print_unknowns(GROUP_UNKNOWN) - t.assert_not_equals(resp.parse_error, nil, - 'blocked on SGROUP support in wire.skip_field; once decoder ' .. - 'accepts groups and pb.text renders unknowns, expect ' .. - 'text_payload containing "1004 {" and "1: 321"') + t.assert_not(resp.parse_error, resp.parse_error) + t.assert_str_contains(resp.text_payload, '1004 {') + t.assert_str_contains(resp.text_payload, '1: 321') end core_g.test_repeated_unknown_fields_drop = function() -- ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput - -- Payload prepends a group, so the same SGROUP issue blocks decode. - -- TARGET: parse_error == nil, text_payload == ''. + -- Payload starts with the same group bytes; after SGROUP-skip the + -- repeated_int32 unknowns drop cleanly too. local resp = pb_to_text(REPEATED_UNKNOWN) - t.assert_not_equals(resp.parse_error, nil, - 'blocked on SGROUP-skip in wire.lua (group bytes come first)') + t.assert_not(resp.parse_error, resp.parse_error) + t.assert_equals(resp.text_payload, '') end core_g.test_repeated_unknown_fields_print = function() -- ProtobufInput.RepeatedUnknownFields_Print.TextFormatOutput - -- TARGET text: - -- 1004 {\n 1: 321\n}\n1011: 1\n1011: 2\n1011: 3\n - -- CURRENT: blocked by SGROUP-skip, then by unknown-field rendering. + -- Group bytes followed by three repeated-int32 entries. Each varint + -- becomes its own ": " line. local resp = pb_to_text_print_unknowns(REPEATED_UNKNOWN) - t.assert_not_equals(resp.parse_error, nil, - 'two-stage fix: (1) accept SGROUP in wire.skip_field, ' .. - '(2) render captured _unknown_fields from pb.text. ' .. - 'Then assert_str_contains for "1011: 1", "1011: 2", "1011: 3"') + t.assert_not(resp.parse_error, resp.parse_error) + t.assert_str_contains(resp.text_payload, '1004 {') + t.assert_str_contains(resp.text_payload, '1: 321') + t.assert_str_contains(resp.text_payload, '1011: 1') + t.assert_str_contains(resp.text_payload, '1011: 2') + t.assert_str_contains(resp.text_payload, '1011: 3') end -- ---------------------------------------------------------------------------