~bigbes/tarantool

tarantool-protobuf

343e4ba4d5e8d8b6e1a046b5db40d8e58ca3eeb7 — Eugene Blikh 3 months ago b1d9c75
text: render captured unknown fields, tolerate SGROUP in skip

Two changes close the proto3 text-format conformance suite:

  1. `wire.skip_field` learns SGROUP/EGROUP. Wire 3 recurses through
     inner tags until a matching EGROUP, with field_id checked against
     the SGROUP's id. Callers (codec.lua, lazy.lua, wkt.lua, generated
     full-mode `_pb.lua`) now pass the tag's field_id so groups inside
     unknown-field skips don't error.

  2. `pb.text.encode` walks the captured `_unknown_fields` buffer when
     `opts.print_unknown_fields=true` and emits each entry in
     TextFormat numeric-field form:
       VARINT  -> "<id>: <uint64>"
       I64/I32 -> "<id>: 0x<hex>"
       LEN     -> speculative "<id> { <recurse> }"; rolls back to
                  byte-string form if the inner bytes don't parse as a
                  sub-message
       SGROUP  -> "<id> { <recurse> }" through matching EGROUP

`cmd/conformance/core.lua` threads `req.print_unknown_fields` into
`pb.text.encode` so `_Drop` tests drop unknowns and `_Print` tests
render them.

Conformance: text-format suite goes from 2 ✓ / 6 expected fails to
8 ✓ / 0 expected fails. All eight regression tests in
`conformance_test.lua` (one per upstream test, plus the fixed field-1011
tag bytes that were miscomputed earlier) now assert the target output.
M PLAN.md => PLAN.md +9 -7
@@ 145,17 145,19 @@ fiber and bridges client ↔ handler via `fiber.channel`. All four flavors
      and `test/conformance/known_failures_text.txt` (text-format
      suite). Current baseline (2026-05-16):
        - Binary+JSON suite: 1478 ✓ / 1313 skipped /  15 expected fails
        - Text-format suite:    2 ✓ /  426 skipped /   6 expected fails
        - Text-format suite:    8 ✓ /  426 skipped /   0 expected fails
      JSON output runs end-to-end; remaining expected fails are
      Recommended-only edge cases (FieldMask round-trip,
      duplicate-field-name rejection, null-in-collection rejection,
      unknown-enum-name rejection, NullValue oneof validator).
      Text-format output is wired through `pb.text.encode`; the six
      expected fails are all unknown-field cases (Group/Repeated parse
      rejection, *_Print not rendering captured unknowns). Text-format
      *input* parsing is still deferred. The `PB_CONFORMANCE_SKIP_JSON=1`
      env var still short-circuits JSON output if a new encoder bug
      crashes jsoncpp.
      Text-format output is wired through `pb.text.encode` and the
      proto3 text suite is fully clean: SGROUP/EGROUP are tolerated in
      `wire.skip_field` and `pb.text` renders captured unknown bytes in
      numeric field-ID form under `opts.print_unknown_fields`. The 426
      still-skipped tests are proto2/editions message types we don't
      register. Text-format *input* parsing is still deferred. The
      `PB_CONFORMANCE_SKIP_JSON=1` env var still short-circuits JSON
      output if a new encoder bug crashes jsoncpp.
      CI wire-up pending — the image build is the long pole (~10–15 min
      on a clean cache).
- [x] Cross-impl interop: 18-fixture corpus in `test/interop/fixtures/`

M cmd/conformance/core.lua => cmd/conformance/core.lua +2 -1
@@ 96,7 96,8 @@ local function dispatch(req)
        end
        return {json_payload = jbytes}
    elseif out_fmt == TEXT_FORMAT then
        local ok, tbytes = pcall(pb.text.encode, desc, msg)
        local opts = {print_unknown_fields = req.print_unknown_fields == true}
        local ok, tbytes = pcall(pb.text.encode, desc, msg, opts)
        if not ok then
            return {serialize_error = 'text encode failed: ' ..
                tostring(tbytes)}

M cmd/protoc-gen-tarantool/internal/gen/inline.go => cmd/protoc-gen-tarantool/internal/gen/inline.go +2 -2
@@ 270,7 270,7 @@ func emitInlineDecode(w *writer, name string, m *protogen.Message, file *protoge
		w.line("        if true then")
	}
	w.line("        else")
	w.line("            pos = wire.skip_field(buf, pos, wt)")
	w.line("            pos = wire.skip_field(buf, pos, wt, id)")
	w.line("            if _uf == nil then _uf = {} end")
	w.line("            _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)")
	w.line("        end")


@@ 527,7 527,7 @@ func emitInlineDecodeMap(w *writer, f *protogen.Field, fname string, file *proto
	w.line("                elseif eid == 2 then")
	emitMapDecode(w, "_val", valF, file, selfPath, imports, prefix)
	w.line("                else")
	w.line("                    _ep = wire.skip_field(payload, _ep, ewt)")
	w.line("                    _ep = wire.skip_field(payload, _ep, ewt, eid)")
	w.line("                end")
	w.line("            end")
	if mapKeyNeedsCdataDedup(keyF) {

M examples/expected/full/conformance/conformance_pb.lua => examples/expected/full/conformance/conformance_pb.lua +5 -5
@@ 193,7 193,7 @@ function M.TestStatus_decode(buf)
            val, pos = wire.decode_string(buf, pos)
            result.matched_name = val
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
            if _uf == nil then _uf = {} end
            _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
        end


@@ 258,7 258,7 @@ function M.FailureSet_decode(buf)
            payload, pos = wire.decode_len(buf, pos)
            list[#list + 1] = M.TestStatus_decode(payload)
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
            if _uf == nil then _uf = {} end
            _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
        end


@@ 439,7 439,7 @@ function M.ConformanceRequest_decode(buf)
            val, pos = wire.decode_bool(buf, pos)
            result.print_unknown_fields = val
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
            if _uf == nil then _uf = {} end
            _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
        end


@@ 668,7 668,7 @@ function M.ConformanceResponse_decode(buf)
            result.skipped = nil
            result.jspb_payload = nil
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
            if _uf == nil then _uf = {} end
            _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
        end


@@ 726,7 726,7 @@ function M.JspbEncodingConfig_decode(buf)
            val, pos = wire.decode_bool(buf, pos)
            result.use_jspb_array_any_format = val
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
            if _uf == nil then _uf = {} end
            _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
        end

M examples/expected/full/hello/hello_pb.lua => examples/expected/full/hello/hello_pb.lua +9 -9
@@ 235,7 235,7 @@ function M.Result_decode(buf)
            result.text = nil
            result.code = nil
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
            if _uf == nil then _uf = {} end
            _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
        end


@@ 294,7 294,7 @@ function M.HelloRequest_decode(buf)
            val, pos = wire.decode_string(buf, pos)
            result.name = val
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
            if _uf == nil then _uf = {} end
            _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
        end


@@ 353,7 353,7 @@ function M.HelloReply_decode(buf)
            val, pos = wire.decode_string(buf, pos)
            result.greeting = val
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
            if _uf == nil then _uf = {} end
            _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
        end


@@ 544,7 544,7 @@ function M.Event_decode(buf)
            payload, pos = wire.decode_len(buf, pos)
            result.update_mask = pb.wkt.FieldMask_decode(payload)
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
            if _uf == nil then _uf = {} end
            _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
        end


@@ 635,7 635,7 @@ function M.Address_decode(buf)
            val, pos = wire.decode_string(buf, pos)
            result.apartment = val
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
            if _uf == nil then _uf = {} end
            _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
        end


@@ 915,7 915,7 @@ function M.Person_decode(buf)
                elseif eid == 2 then
                    _val, _ep = wire.decode_int32(payload, _ep)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            map[_key] = _val


@@ 934,7 934,7 @@ function M.Person_decode(buf)
                elseif eid == 2 then
                    _val, _ep = wire.decode_string(payload, _ep)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            map[_key] = _val


@@ 955,12 955,12 @@ function M.Person_decode(buf)
                    _payload, _ep = wire.decode_len(payload, _ep)
                    _val = M.Address_decode(_payload)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            map[_key] = _val
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
            if _uf == nil then _uf = {} end
            _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
        end

M examples/expected/full/protobuf_test_messages/proto3/test_messages_proto3_pb.lua => examples/expected/full/protobuf_test_messages/proto3/test_messages_proto3_pb.lua +24 -24
@@ 2936,7 2936,7 @@ function M.TestAllTypesProto3_decode(buf)
                elseif eid == 2 then
                    _val, _ep = wire.decode_int32(payload, _ep)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            map[_key] = _val


@@ 2955,7 2955,7 @@ function M.TestAllTypesProto3_decode(buf)
                elseif eid == 2 then
                    _val, _ep = wire.decode_int64(payload, _ep)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            for _k in pairs(map) do


@@ 2977,7 2977,7 @@ function M.TestAllTypesProto3_decode(buf)
                elseif eid == 2 then
                    _val, _ep = wire.decode_uint32(payload, _ep)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            map[_key] = _val


@@ 2996,7 2996,7 @@ function M.TestAllTypesProto3_decode(buf)
                elseif eid == 2 then
                    _val, _ep = wire.decode_uint64(payload, _ep)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            for _k in pairs(map) do


@@ 3018,7 3018,7 @@ function M.TestAllTypesProto3_decode(buf)
                elseif eid == 2 then
                    _val, _ep = wire.decode_sint32(payload, _ep)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            map[_key] = _val


@@ 3037,7 3037,7 @@ function M.TestAllTypesProto3_decode(buf)
                elseif eid == 2 then
                    _val, _ep = wire.decode_sint64(payload, _ep)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            for _k in pairs(map) do


@@ 3059,7 3059,7 @@ function M.TestAllTypesProto3_decode(buf)
                elseif eid == 2 then
                    _val, _ep = wire.decode_fixed32(payload, _ep)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            map[_key] = _val


@@ 3078,7 3078,7 @@ function M.TestAllTypesProto3_decode(buf)
                elseif eid == 2 then
                    _val, _ep = wire.decode_fixed64(payload, _ep)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            for _k in pairs(map) do


@@ 3100,7 3100,7 @@ function M.TestAllTypesProto3_decode(buf)
                elseif eid == 2 then
                    _val, _ep = wire.decode_sfixed32(payload, _ep)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            map[_key] = _val


@@ 3119,7 3119,7 @@ function M.TestAllTypesProto3_decode(buf)
                elseif eid == 2 then
                    _val, _ep = wire.decode_sfixed64(payload, _ep)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            for _k in pairs(map) do


@@ 3141,7 3141,7 @@ function M.TestAllTypesProto3_decode(buf)
                elseif eid == 2 then
                    _val, _ep = wire.decode_float(payload, _ep)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            map[_key] = _val


@@ 3160,7 3160,7 @@ function M.TestAllTypesProto3_decode(buf)
                elseif eid == 2 then
                    _val, _ep = wire.decode_double(payload, _ep)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            map[_key] = _val


@@ 3179,7 3179,7 @@ function M.TestAllTypesProto3_decode(buf)
                elseif eid == 2 then
                    _val, _ep = wire.decode_bool(payload, _ep)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            map[_key] = _val


@@ 3198,7 3198,7 @@ function M.TestAllTypesProto3_decode(buf)
                elseif eid == 2 then
                    _val, _ep = wire.decode_string(payload, _ep)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            map[_key] = _val


@@ 3217,7 3217,7 @@ function M.TestAllTypesProto3_decode(buf)
                elseif eid == 2 then
                    _val, _ep = wire.decode_bytes(payload, _ep)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            map[_key] = _val


@@ 3238,7 3238,7 @@ function M.TestAllTypesProto3_decode(buf)
                    _payload, _ep = wire.decode_len(payload, _ep)
                    _val = M.TestAllTypesProto3_NestedMessage_decode(_payload)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            map[_key] = _val


@@ 3259,7 3259,7 @@ function M.TestAllTypesProto3_decode(buf)
                    _payload, _ep = wire.decode_len(payload, _ep)
                    _val = M.ForeignMessage_decode(_payload)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            map[_key] = _val


@@ 3280,7 3280,7 @@ function M.TestAllTypesProto3_decode(buf)
                    _u, _ep = wire.decode_varint(payload, _ep)
                    _val = wire.varint_to_int32(_u)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            map[_key] = _val


@@ 3301,7 3301,7 @@ function M.TestAllTypesProto3_decode(buf)
                    _u, _ep = wire.decode_varint(payload, _ep)
                    _val = wire.varint_to_int32(_u)
                else
                    _ep = wire.skip_field(payload, _ep, ewt)
                    _ep = wire.skip_field(payload, _ep, ewt, eid)
                end
            end
            map[_key] = _val


@@ 3683,7 3683,7 @@ function M.TestAllTypesProto3_decode(buf)
            val, pos = wire.decode_int32(buf, pos)
            result.Field_name18__ = val
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
            if _uf == nil then _uf = {} end
            _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
        end


@@ 3758,7 3758,7 @@ function M.TestAllTypesProto3_NestedMessage_decode(buf)
                pb.codec.merge_message(M.TestAllTypesProto3_descriptor, prev, M.TestAllTypesProto3_decode(payload))
            end
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
            if _uf == nil then _uf = {} end
            _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
        end


@@ 3816,7 3816,7 @@ function M.ForeignMessage_decode(buf)
            val, pos = wire.decode_int32(buf, pos)
            result.c = val
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
            if _uf == nil then _uf = {} end
            _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
        end


@@ 3865,7 3865,7 @@ function M.NullHypothesisProto3_decode(buf)
        id, wt, pos = wire.decode_tag(buf, pos)
        if true then
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
            if _uf == nil then _uf = {} end
            _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
        end


@@ 3914,7 3914,7 @@ function M.EnumOnlyProto3_decode(buf)
        id, wt, pos = wire.decode_tag(buf, pos)
        if true then
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
            if _uf == nil then _uf = {} end
            _uf[#_uf + 1] = buf:sub(_tag_start, pos - 1)
        end

M runtime/pb/codec.lua => runtime/pb/codec.lua +2 -2
@@ 813,7 813,7 @@ decode_message = function(desc, buf)
        local f = fbi[id]
        if f == nil then
            -- Unknown field: capture verbatim for round-trip.
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
            if unknown == nil then unknown = {} end
            unknown[#unknown + 1] = buf:sub(tag_start, pos - 1)
        else


@@ 839,7 839,7 @@ decode_message = function(desc, buf)
                    elseif eid == 2 then
                        val, ep = decode_one(f.value, payload, ep)
                    else
                        ep = wire.skip_field(payload, ep, ewt)
                        ep = wire.skip_field(payload, ep, ewt, eid)
                    end
                end
                if key == nil then key = default_value(f.key) end

M runtime/pb/lazy.lua => runtime/pb/lazy.lua +2 -2
@@ 81,7 81,7 @@ local function index_bytes(desc, bytes)
        local tag_start = pos
        local id, wt, npos = wire.decode_tag(bytes, pos)
        local val_start = npos
        local next_start = wire.skip_field(bytes, npos, wt)
        local next_start = wire.skip_field(bytes, npos, wt, id)
        n = n + 1
        s_id[n]   = id
        s_tag[n]  = tag_start


@@ 298,7 298,7 @@ local function decode_map_entry(field, bytes, val_start)
                end
            end
        else
            p = wire.skip_field(bytes, p, ewt)
            p = wire.skip_field(bytes, p, ewt, eid)
        end
    end
    if key == nil then

M runtime/pb/text.lua => runtime/pb/text.lua +101 -0
@@ 20,6 20,7 @@
local ffi      = require('ffi')
local datetime = require('datetime')
local pbwkt    = require('pb.wkt')
local wire     = require('pb.wire')

local M = {}



@@ 116,6 117,7 @@ local function new_buf(opts)
        chunks = {}, n = 0,
        single_line = opts.single_line and true or false,
        indent_unit = opts.indent or '  ',
        print_unknown_fields = opts.print_unknown_fields and true or false,
    }
end



@@ 207,6 209,96 @@ end

local WKT_TEXT  -- forward (filled below)

-- ---------------------------------------------------------------------------
-- Unknown-field rendering (drives `_Print` conformance tests).
--
-- Walks the captured _unknown_fields byte stream (tag+value chunks the
-- decoder stashed for round-trip) and emits each entry in TextFormat
-- numeric-field form so the conformance harness's TextFormat::Parser can
-- round-trip the output back under AllowFieldNumber:
--   VARINT  -> "<id>: <uint64>"
--   I64     -> "<id>: 0x<16-hex>"
--   LEN     -> "<id> { <walk> }" if the payload parses as a sub-message,
--              else "<id>: <byte-string>"
--   I32     -> "<id>: 0x<8-hex>"
--   SGROUP  -> "<id> { <walk> }" (recurses until matching EGROUP)
local walk_unknown  -- forward; recurses through SGROUP + LEN(message)

walk_unknown = function(buf, raw, pos, lim, depth, end_field_id)
    while pos <= lim do
        local id, wt, npos = wire.decode_tag(raw, pos)
        pos = npos
        if wt == wire.WIRE_VARINT then
            local v
            v, pos = wire.decode_varint(raw, pos)
            newline(buf, depth)
            push(buf, tostring(id)); push(buf, ': '); push(buf, int_to_string(v))
        elseif wt == wire.WIRE_I64 then
            if pos + 7 > lim then error("truncated I64 in unknown", 0) end
            local lo = raw:byte(pos)     +
                       raw:byte(pos + 1) * 0x100 +
                       raw:byte(pos + 2) * 0x10000 +
                       raw:byte(pos + 3) * 0x1000000
            local hi = raw:byte(pos + 4) +
                       raw:byte(pos + 5) * 0x100 +
                       raw:byte(pos + 6) * 0x10000 +
                       raw:byte(pos + 7) * 0x1000000
            newline(buf, depth)
            push(buf, tostring(id)); push(buf, ': ')
            push(buf, string.format('0x%08x%08x', hi, lo))
            pos = pos + 8
        elseif wt == wire.WIRE_LEN then
            local payload
            payload, pos = wire.decode_len(raw, pos)
            -- Speculative: render as `<id> { ... }`. Roll back to byte
            -- form if the payload doesn't parse as a clean sub-message.
            local n_before = buf.n
            local ok = pcall(function()
                emit_block(buf, tostring(id), depth, function(b, d)
                    walk_unknown(b, payload, 1, #payload, d, nil)
                end)
            end)
            if not ok then
                buf.n = n_before
                newline(buf, depth)
                push(buf, tostring(id)); push(buf, ': ')
                push(buf, escape_string(payload))
            end
        elseif wt == wire.WIRE_I32 then
            if pos + 3 > lim then error("truncated I32 in unknown", 0) end
            local v = raw:byte(pos)     +
                      raw:byte(pos + 1) * 0x100 +
                      raw:byte(pos + 2) * 0x10000 +
                      raw:byte(pos + 3) * 0x1000000
            newline(buf, depth)
            push(buf, tostring(id)); push(buf, ': ')
            push(buf, string.format('0x%08x', v))
            pos = pos + 4
        elseif wt == wire.WIRE_SGROUP then
            -- Recurse into the body; the closure mutates `pos` so the
            -- outer loop continues past the matching EGROUP.
            emit_block(buf, tostring(id), depth, function(b, d)
                pos = walk_unknown(b, raw, pos, lim, d, id)
            end)
        elseif wt == wire.WIRE_EGROUP then
            if end_field_id == nil then
                error("unexpected EGROUP in unknown stream", 0)
            end
            if id ~= end_field_id then
                error(("EGROUP id %d does not match SGROUP id %d"):
                    format(id, end_field_id), 0)
            end
            return pos
        else
            error("unknown wire type " .. tostring(wt), 0)
        end
    end
    if end_field_id ~= nil then
        error("missing EGROUP for field " .. tostring(end_field_id), 0)
    end
    return pos
end

emit_message = function(buf, desc, t, depth)
    -- Use type() rather than == nil so box.NULL (a nil-equal cdata used as
    -- the Value WKT's null_value sentinel) survives the guard.


@@ 228,6 320,15 @@ emit_message = function(buf, desc, t, depth)
            emit_field(buf, f, v, depth)
        end
    end
    -- Captured unknown bytes go last, mirroring the codec's re-encode
    -- order. Gated by `opts.print_unknown_fields` to match the protoc
    -- TextFormat::Printer default (off).
    if buf.print_unknown_fields then
        local raw = t._unknown_fields
        if type(raw) == 'string' and #raw > 0 then
            walk_unknown(buf, raw, 1, #raw, depth, nil)
        end
    end
end

-- ---------------------------------------------------------------------------

M runtime/pb/wire.lua => runtime/pb/wire.lua +30 -2
@@ 9,6 9,11 @@ local M = {}
M.WIRE_VARINT = 0
M.WIRE_I64    = 1
M.WIRE_LEN    = 2
-- Wire 3/4 are proto2 groups. We never emit them, but unknown-field
-- skip must tolerate them so proto2-shaped payloads round-trip through
-- a proto3 decoder (conformance suite exercises this).
M.WIRE_SGROUP = 3
M.WIRE_EGROUP = 4
M.WIRE_I32    = 5

local UINT64 = ffi.typeof('uint64_t')


@@ 601,9 606,12 @@ M.TYPE_INFO = {

-- ---------------------------------------------------------------------------
-- Skip an unknown field (used by decoder when an unrecognized id appears).
-- skip_field(buf, pos, wire_type) -> new_pos
-- skip_field(buf, pos, wire_type, field_id) -> new_pos
--
-- field_id is required for SGROUP (wire 3) so the closing EGROUP can
-- be matched. Other wire types ignore it.
-- ---------------------------------------------------------------------------
local function skip_field(buf, pos, wire_type)
local function skip_field(buf, pos, wire_type, field_id)
    if wire_type == M.WIRE_VARINT then
        local b = buf:byte(pos)
        if b == nil then error("truncated varint at offset " .. pos, 0) end


@@ 630,6 638,26 @@ local function skip_field(buf, pos, wire_type)
        local np = pos + 4
        if np > #buf + 1 then error("truncated I32 at offset " .. pos, 0) end
        return np
    elseif wire_type == M.WIRE_SGROUP then
        -- Read inner tags until the matching EGROUP; recurse on nested
        -- groups. EGROUP id mismatch is a hard error per spec.
        if field_id == nil then
            error("skip_field SGROUP requires field_id", 0)
        end
        while true do
            local iid, iwt
            iid, iwt, pos = decode_tag(buf, pos)
            if iwt == M.WIRE_EGROUP then
                if iid ~= field_id then
                    error(("EGROUP id %d does not match SGROUP id %d"):
                        format(iid, field_id), 0)
                end
                return pos
            end
            pos = skip_field(buf, pos, iwt, iid)
        end
    elseif wire_type == M.WIRE_EGROUP then
        error("unexpected EGROUP for field " .. tostring(field_id), 0)
    end
    error("unknown wire type " .. tostring(wire_type), 0)
end

M runtime/pb/wkt.lua => runtime/pb/wkt.lua +9 -9
@@ 80,7 80,7 @@ local function timestamp_decode(buf)
        elseif id == 2 then
            nanos, pos = wire.decode_int32(buf, pos)
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
        end
    end
    -- datetime.new validates nanos/seconds ranges. Out-of-spec Timestamps


@@ 139,7 139,7 @@ local function duration_decode(buf)
        elseif id == 2 then
            nanos, pos = wire.decode_int32(buf, pos)
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
        end
    end
    return {seconds = seconds, nanos = nanos}


@@ 217,7 217,7 @@ for _, spec in ipairs(WRAPPERS) do
            if id == 1 then
                val, pos = decode(buf, pos)
            else
                pos = wire.skip_field(buf, pos, wt2)
                pos = wire.skip_field(buf, pos, wt2, id)
            end
        end
        return val


@@ 374,7 374,7 @@ value_decode = function(buf)
            payload, pos = wire.decode_len(buf, pos)
            result = list_decode(payload)
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
        end
    end
    return result


@@ 401,12 401,12 @@ struct_decode = function(buf)
                    vbuf, ep = wire.decode_len(payload, ep)
                    val = value_decode(vbuf)
                else
                    ep = wire.skip_field(payload, ep, ewt)
                    ep = wire.skip_field(payload, ep, ewt, eid)
                end
            end
            result[key] = val
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
        end
    end
    return result


@@ 423,7 423,7 @@ list_decode = function(buf)
            payload, pos = wire.decode_len(buf, pos)
            result[#result + 1] = value_decode(payload)
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
        end
    end
    return result


@@ 485,7 485,7 @@ any_decode = function(buf)
        elseif id == 2 then
            value, pos = wire.decode_bytes(buf, pos)
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
        end
    end
    return {type_url = type_url, value = value}


@@ 571,7 571,7 @@ local function fieldmask_decode(buf)
            s, pos = wire.decode_string(buf, pos)
            result[#result + 1] = s
        else
            pos = wire.skip_field(buf, pos, wt)
            pos = wire.skip_field(buf, pos, wt, id)
        end
    end
    return result

M test/conformance/known_failures_text.txt => test/conformance/known_failures_text.txt +7 -22
@@ 1,26 1,11 @@
# conformance_test_runner --text_format_failure_list
#
# Text-format OUTPUT is now wired through cmd/conformance/core.lua to
# pb.text.encode (protobuf/JSON input → text output). Input parsing is
# still deferred — pb.text only encodes — so any test whose payload is
# text_payload still returns `skipped`.
# Text-format OUTPUT runs through pb.text.encode (protobuf/JSON input →
# text output). Group/Repeated unknown-field decode is supported by the
# SGROUP-recursive `wire.skip_field`, and unknown bytes are rendered in
# numeric field-ID form when `print_unknown_fields=true`.
#
# Remaining expected failures all stem from unknown-field handling:
# Text-format INPUT is still deferred — pb.text remains encode-only —
# so any test whose payload is `text_payload` returns `skipped`.
#
#   * Group/Repeated *_Drop: our decoder rejects payloads carrying
#     wire types 3/4 (proto2 groups) and length-mismatched repeated
#     unknown bytes, so we never reach the encoder. Expected output is
#     an empty text body, which we'd produce trivially if the decoder
#     accepted the input. Fixing requires skip_field to tolerate groups.
#
#   * *_Print: with `print_unknown_fields: true` the harness expects the
#     text output to include the original unknown fields. We capture
#     unknown bytes during decode but pb.text doesn't render them, so
#     output comes back empty. Fixing requires the text encoder to walk
#     the captured unknown set.
Recommended.Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput
Recommended.Proto3.ProtobufInput.GroupUnknownFields_Print.TextFormatOutput
Recommended.Proto3.ProtobufInput.MessageUnknownFields_Print.TextFormatOutput
Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput
Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Print.TextFormatOutput
Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Print.TextFormatOutput
# No expected failures in the proto3 text suite as of the last refresh.

M test/conformance_test.lua => test/conformance_test.lua +32 -40
@@ 1215,8 1215,9 @@ local SCALAR_UNKNOWN  = '\xc8\x3e\x7b'                       -- field 1001 varin
local MESSAGE_UNKNOWN = '\xda\x3e\x02\x08\x6f'               -- field 1003 LEN {1:111}
local GROUP_UNKNOWN   = '\xe3\x3e\x08\xc1\x02\xe4\x3e'       -- field 1004 SGROUP {a:321} EGROUP
-- Repeated builds on Group then appends three repeated_int32 entries.
-- Tag for field 1011 (varint): 1011*8 = 8088 = 24 | (63 << 7) = `\x98\x3f`.
local REPEATED_UNKNOWN = GROUP_UNKNOWN
                      .. '\xd8\x3e\x01\xd8\x3e\x02\xd8\x3e\x03'  -- field 1011 varint 1,2,3
                      .. '\x98\x3f\x01\x98\x3f\x02\x98\x3f\x03'

core_g.test_scalar_unknown_fields_drop = function()
    -- ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput


@@ 1229,18 1230,14 @@ end

core_g.test_scalar_unknown_fields_print = function()
    -- ProtobufInput.ScalarUnknownFields_Print.TextFormatOutput
    -- TARGET: text_payload must round-trip to a message containing the
    -- three unknown fields. Canonical form is numeric IDs, e.g.
    --   1001: 123\n1002: "hello"\n1006: 1\n
    -- CURRENT: pb.text doesn't walk _unknown_fields, so output is empty.
    -- Pin current behavior; flip this assertion when the renderer lands.
    -- Numeric field-ID form; LEN payload that doesn't parse as a
    -- sub-message falls back to byte-string rendering ("hello" here).
    local resp = pb_to_text_print_unknowns(SCALAR_UNKNOWN)
    t.assert_not(resp.parse_error, resp.parse_error)
    t.assert_not(resp.serialize_error, resp.serialize_error)
    t.assert_equals(resp.text_payload, '',
        'current behavior pins empty output; ' ..
        'replace with assert_str_contains(...,"1001: 123") when ' ..
        'pb.text renders captured _unknown_fields')
    t.assert_str_contains(resp.text_payload, '1001: 123')
    t.assert_str_contains(resp.text_payload, '1002: "hello"')
    t.assert_str_contains(resp.text_payload, '1006: 1')
end

core_g.test_message_unknown_fields_drop = function()


@@ 1252,58 1249,53 @@ end

core_g.test_message_unknown_fields_print = function()
    -- ProtobufInput.MessageUnknownFields_Print.TextFormatOutput
    -- TARGET text:   1003 {\n  1: 111\n}\n
    -- CURRENT: pb.text drops _unknown_fields, so output is empty.
    -- LEN payload parses cleanly as a sub-message ({c:111}), so the
    -- speculative block-form succeeds: "1003 { 1: 111 }".
    local resp = pb_to_text_print_unknowns(MESSAGE_UNKNOWN)
    t.assert_not(resp.parse_error, resp.parse_error)
    t.assert_equals(resp.text_payload, '',
        'replace with target "1003 { 1: 111 }" rendering when ' ..
        'pb.text grows submessage unknown-field support')
    t.assert_str_contains(resp.text_payload, '1003 {')
    t.assert_str_contains(resp.text_payload, '1: 111')
end

core_g.test_group_unknown_fields_drop = function()
    -- ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput
    -- Wire types 3/4 (SGROUP/EGROUP) — proto3 must tolerate them as
    -- unknown groups. Our skip_field currently errors on wire 3, so the
    -- decoder reports parse_error before reaching the text encoder.
    -- TARGET: parse_error == nil, text_payload == ''.
    -- Wire 3/4 are proto2 groups; the proto3 decoder must skip them
    -- (wire.skip_field recurses on SGROUP until matching EGROUP).
    local resp = pb_to_text(GROUP_UNKNOWN)
    t.assert_not_equals(resp.parse_error, nil,
        'current: skip_field rejects SGROUP wire type 3. ' ..
        'Fix in runtime/pb/wire.lua to recursively skip until matching ' ..
        'EGROUP, then flip this to assert_not + text_payload == ""')
    t.assert_not(resp.parse_error, resp.parse_error)
    t.assert_equals(resp.text_payload, '')
end

core_g.test_group_unknown_fields_print = function()
    -- ProtobufInput.GroupUnknownFields_Print.TextFormatOutput
    -- TARGET text:   1004 {\n  1: 321\n}\n  (groups render as submessages)
    -- CURRENT: blocked by same SGROUP-skip issue as the Drop variant.
    -- Group bytes are captured verbatim (SGROUP..EGROUP); the renderer
    -- recurses through them and emits "1004 { 1: 321 }".
    local resp = pb_to_text_print_unknowns(GROUP_UNKNOWN)
    t.assert_not_equals(resp.parse_error, nil,
        'blocked on SGROUP support in wire.skip_field; once decoder ' ..
        'accepts groups and pb.text renders unknowns, expect ' ..
        'text_payload containing "1004 {" and "1: 321"')
    t.assert_not(resp.parse_error, resp.parse_error)
    t.assert_str_contains(resp.text_payload, '1004 {')
    t.assert_str_contains(resp.text_payload, '1: 321')
end

core_g.test_repeated_unknown_fields_drop = function()
    -- ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput
    -- Payload prepends a group, so the same SGROUP issue blocks decode.
    -- TARGET: parse_error == nil, text_payload == ''.
    -- Payload starts with the same group bytes; after SGROUP-skip the
    -- repeated_int32 unknowns drop cleanly too.
    local resp = pb_to_text(REPEATED_UNKNOWN)
    t.assert_not_equals(resp.parse_error, nil,
        'blocked on SGROUP-skip in wire.lua (group bytes come first)')
    t.assert_not(resp.parse_error, resp.parse_error)
    t.assert_equals(resp.text_payload, '')
end

core_g.test_repeated_unknown_fields_print = function()
    -- ProtobufInput.RepeatedUnknownFields_Print.TextFormatOutput
    -- TARGET text:
    --   1004 {\n  1: 321\n}\n1011: 1\n1011: 2\n1011: 3\n
    -- CURRENT: blocked by SGROUP-skip, then by unknown-field rendering.
    -- Group bytes followed by three repeated-int32 entries. Each varint
    -- becomes its own "<id>: <value>" line.
    local resp = pb_to_text_print_unknowns(REPEATED_UNKNOWN)
    t.assert_not_equals(resp.parse_error, nil,
        'two-stage fix: (1) accept SGROUP in wire.skip_field, ' ..
        '(2) render captured _unknown_fields from pb.text. ' ..
        'Then assert_str_contains for "1011: 1", "1011: 2", "1011: 3"')
    t.assert_not(resp.parse_error, resp.parse_error)
    t.assert_str_contains(resp.text_payload, '1004 {')
    t.assert_str_contains(resp.text_payload, '1: 321')
    t.assert_str_contains(resp.text_payload, '1011: 1')
    t.assert_str_contains(resp.text_payload, '1011: 2')
    t.assert_str_contains(resp.text_payload, '1011: 3')
end

-- ---------------------------------------------------------------------------