From b1273f1bd5ffa459ac5ff714a6dd75d405b8d43c Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Sat, 16 May 2026 00:57:54 +0300 Subject: [PATCH] json: canonical lowerCamelCase + NullValue WKT descriptor Two unrelated JSON-decoder gaps captured under the conformance triage: 1. to_camel's gsub pattern '_(%w)' didn't match consecutive underscores and didn't drop trailing underscores, so proto names like __field_name13 / field__name4_ / field_name17__ generated JSON keys that didn't match what protoc produces. The fix strips trailing _+ and collapses '_+%w' to a capitalized letter; a leading underscore thus capitalizes the next character, matching the spec. 2. pb.wkt didn't export a descriptor for google.protobuf.NullValue, so any enum field whose type is NullValue (oneof_null_value, or the implicit one inside Value) crashed decode_enum with "attempt to index a nil value." Adds a minimal {by_name, by_value} descriptor. Drops 3 entries from test/conformance/known_failures.txt (FieldNameInSnakeCase, FieldNameWithDoubleUnderscores, NullValueInOtherOneofOldFormat). Adds 4 regression tests covering double underscore, leading underscore, trailing underscore, and the NullValue enum decode path. --- runtime/pb/json.lua | 8 +++-- runtime/pb/wkt.lua | 11 +++++++ test/conformance/known_failures.txt | 3 -- test/conformance_test.lua | 48 +++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 5 deletions(-) diff --git a/runtime/pb/json.lua b/runtime/pb/json.lua index a23cf69a03a36b39a39525f332e6ed9ae9d33315..13016175ac5d82f815b8b147ce4e41bff1e66fc9 100644 --- a/runtime/pb/json.lua +++ b/runtime/pb/json.lua @@ -33,9 +33,13 @@ local INT64_FAMILY = {int64=true, uint64=true, sint64=true, fixed64=true, sfixed -- Helpers -- --------------------------------------------------------------------------- --- snake_case -> camelCase (proto field naming convention for JSON). +-- snake_case -> lowerCamelCase per the proto3 JSON spec. Multi-underscore +-- runs collapse to one capitalized letter; trailing underscores drop. A +-- leading underscore causes the next letter to be capitalized so the JSON +-- name has no leading underscore (e.g. `_field_name3` → `FieldName3`). local function to_camel(name) - return (name:gsub('_(%w)', function(c) return c:upper() end)) + name = name:gsub('_+$', '') + return (name:gsub('_+(%w)', function(c) return c:upper() end)) end -- Lossless stringification of an integer/cdata for JSON output. diff --git a/runtime/pb/wkt.lua b/runtime/pb/wkt.lua index 690fbc07f18b6df9b3fcca7d46f7c3e8935aae3f..b652bfbe8cb395db2aa096fa4fdd94ffc32c1a49 100644 --- a/runtime/pb/wkt.lua +++ b/runtime/pb/wkt.lua @@ -254,6 +254,17 @@ end local NULL = box.NULL M.NULL = NULL +-- google.protobuf.NullValue is a singleton enum (NULL_VALUE = 0). It shows +-- up as the type of Value's null_value field and as a standalone enum +-- field type (e.g. oneof_null_value in the conformance test message). We +-- export a minimal descriptor so the codec layer can route enum decode +-- through enum.by_name lookups instead of crashing on a nil descriptor. +M.NullValue_descriptor = { + name = 'google.protobuf.NullValue', + by_name = {NULL_VALUE = 0}, + by_value = {[0] = 'NULL_VALUE'}, +} + local STRUCT_MT = {__pb_kind = 'struct'} local LIST_MT = {__pb_kind = 'list'} diff --git a/test/conformance/known_failures.txt b/test/conformance/known_failures.txt index 26abafd630c15ede928a5110d0c276fb5cba7390..a7dc8ab6c68a6aa12e3ac72e630dfd5943b8c248 100644 --- a/test/conformance/known_failures.txt +++ b/test/conformance/known_failures.txt @@ -18,15 +18,12 @@ # because of the global JSON-output skip. # # Re-generate this file after fixes via `just conformance-refresh-failures`. -Recommended.Proto3.JsonInput.FieldNameWithDoubleUnderscores.ProtobufOutput -Recommended.Proto3.JsonInput.NullValueInOtherOneofOldFormat.Validator Required.Proto3.JsonInput.AllFieldAcceptNull.JsonOutput Required.Proto3.JsonInput.AnyNested.JsonOutput Required.Proto3.JsonInput.AnyWithInt32ValueWrapper.JsonOutput Required.Proto3.JsonInput.AnyWithStruct.JsonOutput Required.Proto3.JsonInput.AnyWithValueForInteger.JsonOutput Required.Proto3.JsonInput.AnyWithValueForJsonObject.JsonOutput -Required.Proto3.JsonInput.FieldNameInSnakeCase.ProtobufOutput Required.Proto3.JsonInput.Int64FieldMaxValueNotQuoted.JsonOutput Required.Proto3.JsonInput.Int64FieldMaxValueNotQuoted.ProtobufOutput Required.Proto3.JsonInput.Int64FieldMinValueNotQuoted.JsonOutput diff --git a/test/conformance_test.lua b/test/conformance_test.lua index 73b60354c5b38cd23deeb8df5c05187cd25f33a5..58f8a8c7d078715ba664ecd1d2bd74e076911819 100644 --- a/test/conformance_test.lua +++ b/test/conformance_test.lua @@ -689,6 +689,54 @@ core_g.test_null_value_field_emits_null_value_member = function() t.assert_equals(resp.protobuf_payload, '\x92\x13\x02\x08\x00') end +-- ========================================================================= +-- Fix 10: lower-camelCase mapping for JSON field names. The proto3 spec +-- collapses runs of underscores (`__`) and drops trailing underscores; a +-- leading underscore causes capitalization of the next letter so the +-- generated JSON name has no leading underscore. +-- ========================================================================= + +core_g.test_json_decode_camel_case_drops_double_underscore = function() + -- field 404 = field__name4_ → canonical JSON name "fieldName4". + -- tag = (404<<3)|0 = 3232 → varint 0xa0 0x19. Plus value varint. + local resp = json_to_pb([[{"fieldName4": 4}]]) + t.assert_not(resp.parse_error, resp.parse_error) + t.assert_equals(resp.protobuf_payload, '\xa0\x19\x04') +end + +core_g.test_json_decode_camel_case_leading_underscore_capitalizes = function() + -- field 403 = _field_name3 → canonical "FieldName3". + -- tag = (403<<3)|0 = 3224 → varint 0x98 0x19. + local resp = json_to_pb([[{"FieldName3": 3}]]) + t.assert_not(resp.parse_error, resp.parse_error) + t.assert_equals(resp.protobuf_payload, '\x98\x19\x03') +end + +core_g.test_json_decode_camel_case_trailing_underscore_drops = function() + -- field 417 = field_name17__ → canonical "fieldName17". + -- tag = (417<<3)|0 = 3336 → varint 0x88 0x1a. + local resp = json_to_pb([[{"fieldName17": 17}]]) + t.assert_not(resp.parse_error, resp.parse_error) + t.assert_equals(resp.protobuf_payload, '\x88\x1a\x11') +end + +-- ========================================================================= +-- Fix 11: google.protobuf.NullValue descriptor exposed so enum fields +-- whose type is NullValue (e.g. oneof_null_value, or Value.null_value) +-- can be decoded from JSON without indexing a nil descriptor. +-- ========================================================================= + +core_g.test_oneof_null_value_decodes_from_json = function() + -- oneof_null_value is the singleton enum NullValue (NULL_VALUE=0). With + -- the descriptor in place, decode_enum can resolve "NULL_VALUE" → 0. + -- Field 120, tag (120<<3)|0 = 960 → varint 0xc0 0x07. Value 0 elides. + -- Empty payload is the expected output for a 0-valued enum in a oneof + -- — except this is a oneof branch with explicit presence, so it WILL + -- emit. Either way, we want no parse_error. + local resp = json_to_pb([[{"oneofNullValue": "NULL_VALUE"}]]) + t.assert_not(resp.parse_error, resp.parse_error) +end + core_g.test_oneof_merge_still_clears_sibling_branches = function() -- The post-fix merge code must still clear oneof siblings: setting -- oneof_uint32 first, then merging two oneof_nested_message entries,