~bigbes/tarantool

tarantool-protobuf

b1273f1bd5ffa459ac5ff714a6dd75d405b8d43c — Eugene Blikh 3 months ago 96ed328
json: canonical lowerCamelCase + NullValue WKT descriptor

Two unrelated JSON-decoder gaps captured under the conformance triage:

1. to_camel's gsub pattern '_(%w)' didn't match consecutive underscores
   and didn't drop trailing underscores, so proto names like
   __field_name13 / field__name4_ / field_name17__ generated JSON keys
   that didn't match what protoc produces. The fix strips trailing _+
   and collapses '_+%w' to a capitalized letter; a leading underscore
   thus capitalizes the next character, matching the spec.

2. pb.wkt didn't export a descriptor for google.protobuf.NullValue, so
   any enum field whose type is NullValue (oneof_null_value, or the
   implicit one inside Value) crashed decode_enum with "attempt to
   index a nil value." Adds a minimal {by_name, by_value} descriptor.

Drops 3 entries from test/conformance/known_failures.txt
(FieldNameInSnakeCase, FieldNameWithDoubleUnderscores,
NullValueInOtherOneofOldFormat). Adds 4 regression tests covering
double underscore, leading underscore, trailing underscore, and the
NullValue enum decode path.
M runtime/pb/json.lua => runtime/pb/json.lua +6 -2
@@ 33,9 33,13 @@ local INT64_FAMILY = {int64=true, uint64=true, sint64=true, fixed64=true, sfixed
-- Helpers
-- ---------------------------------------------------------------------------

-- snake_case -> camelCase (proto field naming convention for JSON).
-- snake_case -> lowerCamelCase per the proto3 JSON spec. Multi-underscore
-- runs collapse to one capitalized letter; trailing underscores drop. A
-- leading underscore causes the next letter to be capitalized so the JSON
-- name has no leading underscore (e.g. `_field_name3` → `FieldName3`).
local function to_camel(name)
    return (name:gsub('_(%w)', function(c) return c:upper() end))
    name = name:gsub('_+$', '')
    return (name:gsub('_+(%w)', function(c) return c:upper() end))
end

-- Lossless stringification of an integer/cdata for JSON output.

M runtime/pb/wkt.lua => runtime/pb/wkt.lua +11 -0
@@ 254,6 254,17 @@ end
local NULL = box.NULL
M.NULL = NULL

-- google.protobuf.NullValue is a singleton enum (NULL_VALUE = 0). It shows
-- up as the type of Value's null_value field and as a standalone enum
-- field type (e.g. oneof_null_value in the conformance test message). We
-- export a minimal descriptor so the codec layer can route enum decode
-- through enum.by_name lookups instead of crashing on a nil descriptor.
M.NullValue_descriptor = {
    name = 'google.protobuf.NullValue',
    by_name = {NULL_VALUE = 0},
    by_value = {[0] = 'NULL_VALUE'},
}

local STRUCT_MT = {__pb_kind = 'struct'}
local LIST_MT   = {__pb_kind = 'list'}


M test/conformance/known_failures.txt => test/conformance/known_failures.txt +0 -3
@@ 18,15 18,12 @@
#     because of the global JSON-output skip.
#
# Re-generate this file after fixes via `just conformance-refresh-failures`.
Recommended.Proto3.JsonInput.FieldNameWithDoubleUnderscores.ProtobufOutput
Recommended.Proto3.JsonInput.NullValueInOtherOneofOldFormat.Validator
Required.Proto3.JsonInput.AllFieldAcceptNull.JsonOutput
Required.Proto3.JsonInput.AnyNested.JsonOutput
Required.Proto3.JsonInput.AnyWithInt32ValueWrapper.JsonOutput
Required.Proto3.JsonInput.AnyWithStruct.JsonOutput
Required.Proto3.JsonInput.AnyWithValueForInteger.JsonOutput
Required.Proto3.JsonInput.AnyWithValueForJsonObject.JsonOutput
Required.Proto3.JsonInput.FieldNameInSnakeCase.ProtobufOutput
Required.Proto3.JsonInput.Int64FieldMaxValueNotQuoted.JsonOutput
Required.Proto3.JsonInput.Int64FieldMaxValueNotQuoted.ProtobufOutput
Required.Proto3.JsonInput.Int64FieldMinValueNotQuoted.JsonOutput

M test/conformance_test.lua => test/conformance_test.lua +48 -0
@@ 689,6 689,54 @@ core_g.test_null_value_field_emits_null_value_member = function()
    t.assert_equals(resp.protobuf_payload, '\x92\x13\x02\x08\x00')
end

-- =========================================================================
-- Fix 10: lower-camelCase mapping for JSON field names. The proto3 spec
-- collapses runs of underscores (`__`) and drops trailing underscores; a
-- leading underscore causes capitalization of the next letter so the
-- generated JSON name has no leading underscore.
-- =========================================================================

core_g.test_json_decode_camel_case_drops_double_underscore = function()
    -- field 404 = field__name4_ → canonical JSON name "fieldName4".
    -- tag = (404<<3)|0 = 3232 → varint 0xa0 0x19. Plus value varint.
    local resp = json_to_pb([[{"fieldName4": 4}]])
    t.assert_not(resp.parse_error, resp.parse_error)
    t.assert_equals(resp.protobuf_payload, '\xa0\x19\x04')
end

core_g.test_json_decode_camel_case_leading_underscore_capitalizes = function()
    -- field 403 = _field_name3 → canonical "FieldName3".
    -- tag = (403<<3)|0 = 3224 → varint 0x98 0x19.
    local resp = json_to_pb([[{"FieldName3": 3}]])
    t.assert_not(resp.parse_error, resp.parse_error)
    t.assert_equals(resp.protobuf_payload, '\x98\x19\x03')
end

core_g.test_json_decode_camel_case_trailing_underscore_drops = function()
    -- field 417 = field_name17__ → canonical "fieldName17".
    -- tag = (417<<3)|0 = 3336 → varint 0x88 0x1a.
    local resp = json_to_pb([[{"fieldName17": 17}]])
    t.assert_not(resp.parse_error, resp.parse_error)
    t.assert_equals(resp.protobuf_payload, '\x88\x1a\x11')
end

-- =========================================================================
-- Fix 11: google.protobuf.NullValue descriptor exposed so enum fields
-- whose type is NullValue (e.g. oneof_null_value, or Value.null_value)
-- can be decoded from JSON without indexing a nil descriptor.
-- =========================================================================

core_g.test_oneof_null_value_decodes_from_json = function()
    -- oneof_null_value is the singleton enum NullValue (NULL_VALUE=0). With
    -- the descriptor in place, decode_enum can resolve "NULL_VALUE" → 0.
    -- Field 120, tag (120<<3)|0 = 960 → varint 0xc0 0x07. Value 0 elides.
    -- Empty payload is the expected output for a 0-valued enum in a oneof
    -- — except this is a oneof branch with explicit presence, so it WILL
    -- emit. Either way, we want no parse_error.
    local resp = json_to_pb([[{"oneofNullValue": "NULL_VALUE"}]])
    t.assert_not(resp.parse_error, resp.parse_error)
end

core_g.test_oneof_merge_still_clears_sibling_branches = function()
    -- The post-fix merge code must still clear oneof siblings: setting
    -- oneof_uint32 first, then merging two oneof_nested_message entries,