~bigbes/tarantool

tarantool-protobuf

76140b72a95bd6de74a34a7db0d735c0fd6721e7 — Eugene Blikh 3 months ago 1bbd8f3
json: emit unbroken base64 for bytes fields ({nowrap = true})

Canonical proto3 JSON expects RFC 4648 unwrapped base64, but Tarantool's
`digest.base64_encode` defaults to RFC 2045 MIME-style 76-char line
wrapping. Any `bytes` payload past ~57 bytes used to land in the JSON
string with an embedded `\n`, which breaks every spec-compliant
consumer (grpc-gateway, protojson, protobuf-go's JSON, ...).

Pass `{nowrap = true}` at the two encode sites: the per-field bytes
encoder and the `google.protobuf.Any` opaque-fallback `value` encoder.

Surfaced by tarantool-etcd's `TestJSONGatewayBytesUnwrapped` over Range
responses whose `value` is >=57 bytes; its reference grpc-gateway never
emits the wrapped form.
3 files changed, 28 insertions(+), 2 deletions(-)

M runtime/pb/json.lua
M test/any_fieldmask_test.lua
M test/json_test.lua
M runtime/pb/json.lua => runtime/pb/json.lua +2 -2
@@ 573,7 573,7 @@ local function encode_scalar(proto_type, v)
        return tostring(v)
    end
    if proto_type == 'uint32' then return v end
    if proto_type == 'bytes' then return digest.base64_encode(v) end
    if proto_type == 'bytes' then return digest.base64_encode(v, {nowrap = true}) end
    if proto_type == 'float' or proto_type == 'double' then
        return v  -- handled by encode_json_number
    end


@@ 740,7 740,7 @@ local function any_to_json(v)
        -- forcing every embedded payload into a Wkt shape).
        local obj = {}
        if type_url ~= '' then obj['@type'] = type_url end
        if bytes ~= '' then obj.value = digest.base64_encode(bytes) end
        if bytes ~= '' then obj.value = digest.base64_encode(bytes, {nowrap = true}) end
        return obj
    end
    local inner = desc.decode and desc.decode(bytes)

M test/any_fieldmask_test.lua => test/any_fieldmask_test.lua +11 -0
@@ 127,6 127,17 @@ gaj.test_json_opaque_fallback_when_type_unregistered = function()
    t.assert_equals(back.extension.value, '\x01\x02\x03')
end

gaj.test_json_opaque_value_base64_unwrapped = function()
    -- Same canonical-base64 rule as plain bytes fields: the Any opaque
    -- fallback must not emit MIME-style line wraps inside the JSON string.
    local opaque = {type_url = 'type.opaque/Big', value = string.rep('A', 64)}
    local enc = pb.json.encode(hello.Event_descriptor, {extension = opaque})
    t.assert_not_str_contains(enc, '\n')
    t.assert_not_str_contains(enc, '\\n')
    local back = pb.json.decode(hello.Event_descriptor, enc)
    t.assert_equals(back.extension.value, opaque.value)
end

-- ---------------------------------------------------------------------------
-- End-to-end: Event message round-trip with extension + update_mask.
-- ---------------------------------------------------------------------------

M test/json_test.lua => test/json_test.lua +15 -0
@@ 43,6 43,21 @@ g.test_bytes_base64 = function()
    t.assert_equals(p2.avatar, p.avatar)
end

g.test_bytes_base64_unwrapped_long_payload = function()
    -- Canonical proto3 JSON requires RFC 4648 base64 with no line wrapping.
    -- Tarantool's digest.base64_encode defaults to MIME-style 76-char wrap,
    -- so any payload past ~57 bytes used to emit a `\n` inside the JSON
    -- string and break grpc-gateway / protojson consumers.
    local p = {avatar = string.rep('A', 64)}
    local enc = pb.json.encode(hello.Person_descriptor, p)
    t.assert_not_str_contains(enc, '\n', 'no raw newline anywhere in JSON output')
    t.assert_not_str_contains(enc, '\\n', 'no escaped newline in base64 token')
    local obj = reparse(enc)
    t.assert_not_str_contains(obj.avatar, '\n', 'base64 token is a single line')
    local p2 = pb.json.decode(hello.Person_descriptor, enc)
    t.assert_equals(p2.avatar, p.avatar)
end

g.test_repeated_packed_scalar = function()
    local p = {lucky_numbers = {1, 2, 3}}
    local obj = reparse(pb.json.encode(hello.Person_descriptor, p))