~bigbes/tarantool-protobuf: wire: truncate varints to 32 bits in int32/uint32/sint32/enum decode

8 files changed, 53 insertions(+), 90 deletions(-)

M cmd/protoc-gen-tarantool/internal/gen/inline.go
M examples/expected/full/conformance/conformance_pb.lua
M examples/expected/full/hello/hello_pb.lua
M examples/expected/full/protobuf_test_messages/proto3/test_messages_proto3_pb.lua
M runtime/pb/codec.lua
M runtime/pb/lazy.lua
M runtime/pb/wire.lua
M test/conformance/known_failures.txt

M cmd/protoc-gen-tarantool/internal/gen/inline.go => cmd/protoc-gen-tarantool/internal/gen/inline.go +4 -4

@@ 268,7 268,7 @@ func emitInlineDecodeFieldBody(w *writer, f *protogen.Field, file *protogen.File
 	case f.Enum != nil:
 		w.line("            local u")
 		w.line("            u, pos = wire.decode_varint(buf, pos)")
-		w.line("            result.%s = tonumber(u)", fname)
+		w.line("            result.%s = wire.varint_to_int32(u)", fname)
 	default:
 		st := scalarName(f.Desc.Kind())
 		w.line("            local val")


@@ 306,12 306,12 @@ func emitInlineDecodeRepeated(w *writer, f *protogen.Field, fname string, file *
 		w.line("                while p2 <= lim do")
 		w.line("                    local u")
 		w.line("                    u, p2 = wire.decode_varint(payload, p2)")
-		w.line("                    list[#list + 1] = tonumber(u)")
+		w.line("                    list[#list + 1] = wire.varint_to_int32(u)")
 		w.line("                end")
 		w.line("            else")
 		w.line("                local u")
 		w.line("                u, pos = wire.decode_varint(buf, pos)")
-		w.line("                list[#list + 1] = tonumber(u)")
+		w.line("                list[#list + 1] = wire.varint_to_int32(u)")
 		w.line("            end")
 	default:
 		st := scalarName(f.Desc.Kind())


@@ 542,7 542,7 @@ func emitMapDecode(w *writer, dst string, f *protogen.Field, file *protogen.File
 	case f.Enum != nil:
 		w.line("                    local _u")
 		w.line("                    _u, _ep = wire.decode_varint(payload, _ep)")
-		w.line("                    %s = tonumber(_u)", dst)
+		w.line("                    %s = wire.varint_to_int32(_u)", dst)
 	default:
 		st := scalarName(f.Desc.Kind())
 		w.line("                    %s, _ep = wire.decode_%s(payload, _ep)", dst, st)

M examples/expected/full/conformance/conformance_pb.lua => examples/expected/full/conformance/conformance_pb.lua +2 -2

@@ 404,7 404,7 @@ function M.ConformanceRequest_decode(buf)
         elseif id == 3 then
             local u
             u, pos = wire.decode_varint(buf, pos)
-            result.requested_output_format = tonumber(u)
+            result.requested_output_format = wire.varint_to_int32(u)
         elseif id == 4 then
             local val
             val, pos = wire.decode_string(buf, pos)


@@ 412,7 412,7 @@ function M.ConformanceRequest_decode(buf)
         elseif id == 5 then
             local u
             u, pos = wire.decode_varint(buf, pos)
-            result.test_category = tonumber(u)
+            result.test_category = wire.varint_to_int32(u)
         elseif id == 6 then
             local payload
             payload, pos = wire.decode_len(buf, pos)

M examples/expected/full/hello/hello_pb.lua => examples/expected/full/hello/hello_pb.lua +1 -1

@@ 871,7 871,7 @@ function M.Person_decode(buf)
         elseif id == 4 then
             local u
             u, pos = wire.decode_varint(buf, pos)
-            result.status = tonumber(u)
+            result.status = wire.varint_to_int32(u)
         elseif id == 5 then
             local payload
             payload, pos = wire.decode_len(buf, pos)

M examples/expected/full/protobuf_test_messages/proto3/test_messages_proto3_pb.lua => examples/expected/full/protobuf_test_messages/proto3/test_messages_proto3_pb.lua +16 -16

@@ 1978,15 1978,15 @@ function M.TestAllTypesProto3_decode(buf)
         elseif id == 21 then
             local u
             u, pos = wire.decode_varint(buf, pos)
-            result.optional_nested_enum = tonumber(u)
+            result.optional_nested_enum = wire.varint_to_int32(u)
         elseif id == 22 then
             local u
             u, pos = wire.decode_varint(buf, pos)
-            result.optional_foreign_enum = tonumber(u)
+            result.optional_foreign_enum = wire.varint_to_int32(u)
         elseif id == 23 then
             local u
             u, pos = wire.decode_varint(buf, pos)
-            result.optional_aliased_enum = tonumber(u)
+            result.optional_aliased_enum = wire.varint_to_int32(u)
         elseif id == 24 then
             local val
             val, pos = wire.decode_string(buf, pos)


@@ 2260,12 2260,12 @@ function M.TestAllTypesProto3_decode(buf)
                 while p2 <= lim do
                     local u
                     u, p2 = wire.decode_varint(payload, p2)
-                    list[#list + 1] = tonumber(u)
+                    list[#list + 1] = wire.varint_to_int32(u)
                 end
             else
                 local u
                 u, pos = wire.decode_varint(buf, pos)
-                list[#list + 1] = tonumber(u)
+                list[#list + 1] = wire.varint_to_int32(u)
             end
         elseif id == 52 then
             local list = result.repeated_foreign_enum


@@ 2277,12 2277,12 @@ function M.TestAllTypesProto3_decode(buf)
                 while p2 <= lim do
                     local u
                     u, p2 = wire.decode_varint(payload, p2)
-                    list[#list + 1] = tonumber(u)
+                    list[#list + 1] = wire.varint_to_int32(u)
                 end
             else
                 local u
                 u, pos = wire.decode_varint(buf, pos)
-                list[#list + 1] = tonumber(u)
+                list[#list + 1] = wire.varint_to_int32(u)
             end
         elseif id == 54 then
             local list = result.repeated_string_piece


@@ 2527,12 2527,12 @@ function M.TestAllTypesProto3_decode(buf)
                 while p2 <= lim do
                     local u
                     u, p2 = wire.decode_varint(payload, p2)
-                    list[#list + 1] = tonumber(u)
+                    list[#list + 1] = wire.varint_to_int32(u)
                 end
             else
                 local u
                 u, pos = wire.decode_varint(buf, pos)
-                list[#list + 1] = tonumber(u)
+                list[#list + 1] = wire.varint_to_int32(u)
             end
         elseif id == 89 then
             local list = result.unpacked_int32


@@ 2765,12 2765,12 @@ function M.TestAllTypesProto3_decode(buf)
                 while p2 <= lim do
                     local u
                     u, p2 = wire.decode_varint(payload, p2)
-                    list[#list + 1] = tonumber(u)
+                    list[#list + 1] = wire.varint_to_int32(u)
                 end
             else
                 local u
                 u, pos = wire.decode_varint(buf, pos)
-                list[#list + 1] = tonumber(u)
+                list[#list + 1] = wire.varint_to_int32(u)
             end
         elseif id == 56 then
             local map = result.map_int32_int32


@@ 3129,7 3129,7 @@ function M.TestAllTypesProto3_decode(buf)
                 elseif eid == 2 then
                     local _u
                     _u, _ep = wire.decode_varint(payload, _ep)
-                    _val = tonumber(_u)
+                    _val = wire.varint_to_int32(_u)
                 else
                     _ep = wire.skip_field(payload, _ep, ewt)
                 end


@@ 3150,7 3150,7 @@ function M.TestAllTypesProto3_decode(buf)
                 elseif eid == 2 then
                     local _u
                     _u, _ep = wire.decode_varint(payload, _ep)
-                    _val = tonumber(_u)
+                    _val = wire.varint_to_int32(_u)
                 else
                     _ep = wire.skip_field(payload, _ep, ewt)
                 end


@@ 3263,7 3263,7 @@ function M.TestAllTypesProto3_decode(buf)
         elseif id == 119 then
             local u
             u, pos = wire.decode_varint(buf, pos)
-            result.oneof_enum = tonumber(u)
+            result.oneof_enum = wire.varint_to_int32(u)
             result.oneof_uint32 = nil
             result.oneof_nested_message = nil
             result.oneof_string = nil


@@ 3276,7 3276,7 @@ function M.TestAllTypesProto3_decode(buf)
         elseif id == 120 then
             local u
             u, pos = wire.decode_varint(buf, pos)
-            result.oneof_null_value = tonumber(u)
+            result.oneof_null_value = wire.varint_to_int32(u)
             result.oneof_uint32 = nil
             result.oneof_nested_message = nil
             result.oneof_string = nil


@@ 3493,7 3493,7 @@ function M.TestAllTypesProto3_decode(buf)
         elseif id == 307 then
             local u
             u, pos = wire.decode_varint(buf, pos)
-            result.optional_null_value = tonumber(u)
+            result.optional_null_value = wire.varint_to_int32(u)
         elseif id == 308 then
             local payload
             payload, pos = wire.decode_len(buf, pos)

M runtime/pb/codec.lua => runtime/pb/codec.lua +10 -8

@@ 475,6 475,7 @@ local function build_repeated_reader(f)
         local WIRE_LEN = wire.WIRE_LEN
         local decode_len_fn = wire.decode_len
         local decode_varint_fn = wire.decode_varint
+        local varint_to_int32 = wire.varint_to_int32
         return function(buf, pos, wt, result)
             local list = result[fname]
             if list == nil then list = {}; result[fname] = list end


@@ 484,13 485,13 @@ local function build_repeated_reader(f)
                 local n = #list
                 while p <= lim do
                     local u, np2 = decode_varint_fn(payload, p)
-                    n = n + 1; list[n] = tonumber(u)
+                    n = n + 1; list[n] = varint_to_int32(u)
                     p = np2
                 end
                 return np
             end
             local u, np = decode_varint_fn(buf, pos)
-            list[#list + 1] = tonumber(u)
+            list[#list + 1] = varint_to_int32(u)
             return np
         end
     end


@@ 541,17 542,18 @@ local function build_reader(f)
 
     if kind == 'enum' then
         local decode_varint_fn = wire.decode_varint
+        local varint_to_int32 = wire.varint_to_int32
         if siblings then
             return function(buf, pos, wt, result)
                 local u, np = decode_varint_fn(buf, pos)
-                result[fname] = tonumber(u)
+                result[fname] = varint_to_int32(u)
                 for i = 1, #siblings do result[siblings[i]] = nil end
                 return np
             end
         end
         return function(buf, pos, wt, result)
             local u, np = decode_varint_fn(buf, pos)
-            result[fname] = tonumber(u)
+            result[fname] = varint_to_int32(u)
             return np
         end
     end


@@ 669,7 671,7 @@ local function decode_one(field, buf, pos)
         return scalar[field.proto_type].decode(buf, pos)
     elseif kind == 'enum' then
         local u, np = wire.decode_varint(buf, pos)
-        return tonumber(u), np
+        return wire.varint_to_int32(u), np
     elseif kind == 'message' then
         local payload, np = wire.decode_len(buf, pos)
         return decode_msg(field.message, payload), np


@@ 796,11 798,11 @@ decode_message = function(desc, buf)
                         while p2 <= lim do
                             local u, np2 = wire.decode_varint(payload, p2)
                             p2 = np2
-                            list[#list + 1] = tonumber(u)
+                            list[#list + 1] = wire.varint_to_int32(u)
                         end
                     else
                         local u, np = wire.decode_varint(buf, pos)
-                        list[#list + 1] = tonumber(u)
+                        list[#list + 1] = wire.varint_to_int32(u)
                         pos = np
                     end
                 elseif kind == 'message' then


@@ 818,7 820,7 @@ decode_message = function(desc, buf)
                 elseif kind == 'enum' then
                     local u, np = wire.decode_varint(buf, pos)
                     pos = np
-                    result[f.name] = tonumber(u)
+                    result[f.name] = wire.varint_to_int32(u)
                 elseif kind == 'message' then
                     local payload, np = wire.decode_len(buf, pos)
                     pos = np

M runtime/pb/lazy.lua => runtime/pb/lazy.lua +3 -3

@@ 119,7 119,7 @@ local function read_singular(field, bytes, val_start)
         return v
     elseif kind == 'enum' then
         local u = wire.decode_varint(bytes, val_start)
-        return tonumber(u)
+        return wire.varint_to_int32(u)
     elseif kind == 'message' then
         local payload = wire.decode_len(bytes, val_start)
         if field.message.decode ~= nil then


@@ 282,13 282,13 @@ local function decode_map_entry(field, bytes, val_start)
             if key_field.kind == 'scalar' then
                 key, p = wire.TYPE_INFO[key_field.proto_type].decode(bytes, p)
             elseif key_field.kind == 'enum' then
-                local u; u, p = wire.decode_varint(bytes, p); key = tonumber(u)
+                local u; u, p = wire.decode_varint(bytes, p); key = wire.varint_to_int32(u)
             end
         elseif eid == 2 then
             if val_field.kind == 'scalar' then
                 val, p = wire.TYPE_INFO[val_field.proto_type].decode(bytes, p)
             elseif val_field.kind == 'enum' then
-                local u; u, p = wire.decode_varint(bytes, p); val = tonumber(u)
+                local u; u, p = wire.decode_varint(bytes, p); val = wire.varint_to_int32(u)
             elseif val_field.kind == 'message' then
                 local payload; payload, p = wire.decode_len(bytes, p)
                 if val_field.message.decode ~= nil then

M runtime/pb/wire.lua => runtime/pb/wire.lua +17 -5

@@ 39,6 39,20 @@ local function to_int64(v)
 end
 M.to_int64 = to_int64
 
+-- Truncate a uint64 varint payload to proto3 int32/uint32 Lua numbers.
+-- Per spec, int32/uint32/enum/sint32 fields must keep only the low 32 bits
+-- of an over-range varint; int32/sint32 additionally sign-extend from bit 31.
+local function varint_to_uint32(u)
+    return tonumber(bit.band(u, 0xFFFFFFFF))
+end
+local function varint_to_int32(u)
+    local n = tonumber(bit.band(u, 0xFFFFFFFF))
+    if n >= 0x80000000 then n = n - 0x100000000 end
+    return n
+end
+M.varint_to_uint32 = varint_to_uint32
+M.varint_to_int32  = varint_to_int32
+
 -- ---------------------------------------------------------------------------
 -- Varint
 -- ---------------------------------------------------------------------------


@@ 322,7 336,7 @@ local function decode_int32(buf, pos)
     if b == nil then error("truncated varint at offset " .. pos, 0) end
     if b < 0x80 then return b, pos + 1 end  -- 0..127 fits int32 directly
     local u, np = decode_varint(buf, pos)
-    return tonumber(INT64(u)), np  -- truncated to int32 range via int64 sign-extension
+    return varint_to_int32(u), np
 end
 local function decode_int64(buf, pos)
     local b = buf:byte(pos)


@@ 336,9 350,7 @@ local function decode_uint32(buf, pos)
     if b == nil then error("truncated varint at offset " .. pos, 0) end
     if b < 0x80 then return b, pos + 1 end
     local u, np = decode_varint(buf, pos)
-    local n = tonumber(UINT64(u))
-    if n < 0 then n = n + 0x100000000 end
-    return n, np
+    return varint_to_uint32(u), np
 end
 local function decode_uint64(buf, pos)
     local b = buf:byte(pos)


@@ 352,7 364,7 @@ local function decode_sint32(buf, pos)
     if b == nil then error("truncated varint at offset " .. pos, 0) end
     if b < 0x80 then return zigzag_decode32(b), pos + 1 end
     local u, np = decode_varint(buf, pos)
-    return zigzag_decode32(tonumber(u)), np
+    return zigzag_decode32(varint_to_uint32(u)), np
 end
 local function decode_sint64(buf, pos)
     local b = buf:byte(pos)

M test/conformance/known_failures.txt => test/conformance/known_failures.txt +0 -51

@@ 31,42 31,6 @@ Recommended.Proto3.ProtobufInput.RejectInvalidUtf8.String.Oneof
 Recommended.Proto3.ProtobufInput.RejectInvalidUtf8.String.Repeated
 Recommended.Proto3.ProtobufInput.RejectInvalidUtf8.String.Singular
 Recommended.Proto3.ProtobufInput.ValidDataOneofBinary.MESSAGE.Merge.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.ENUM.PackedInput.DefaultOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.ENUM.PackedInput.PackedOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.ENUM.PackedInput.UnpackedOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.ENUM.UnpackedInput.DefaultOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.ENUM.UnpackedInput.PackedOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.ENUM.UnpackedInput.UnpackedOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT32.PackedInput.DefaultOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT32.PackedInput.PackedOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT32.PackedInput.UnpackedOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT32.UnpackedInput.DefaultOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT32.UnpackedInput.PackedOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT32.UnpackedInput.UnpackedOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT32.PackedInput.DefaultOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT32.PackedInput.PackedOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT32.PackedInput.UnpackedOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT32.UnpackedInput.DefaultOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT32.UnpackedInput.PackedOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT32.UnpackedInput.UnpackedOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.DefaultOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.PackedOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.UnpackedOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.DefaultOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.PackedOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.UnpackedOutput.ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.ENUM[4].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.ENUM[5].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.INT32[6].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.INT32[7].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.INT32[8].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.INT32[9].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.SINT32[4].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.UINT32[5].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.UINT32[6].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.UINT32[7].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.UINT32[8].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.UINT32[9].ProtobufOutput
 Required.Proto3.JsonInput.AllFieldAcceptNull.JsonOutput
 Required.Proto3.JsonInput.AllFieldAcceptNull.ProtobufOutput
 Required.Proto3.JsonInput.Any.ProtobufOutput


@@ 116,9 80,6 @@ Required.Proto3.ProtobufInput.PrematureEofInsideUnknownValue.FLOAT
 Required.Proto3.ProtobufInput.PrematureEofInsideUnknownValue.SFIXED32
 Required.Proto3.ProtobufInput.PrematureEofInsideUnknownValue.SFIXED64
 Required.Proto3.ProtobufInput.RepeatedScalarMessageMerge.ProtobufOutput
-Required.Proto3.ProtobufInput.RepeatedScalarSelectsLast.ENUM.ProtobufOutput
-Required.Proto3.ProtobufInput.RepeatedScalarSelectsLast.INT32.ProtobufOutput
-Required.Proto3.ProtobufInput.RepeatedScalarSelectsLast.UINT32.ProtobufOutput
 Required.Proto3.ProtobufInput.UnknownWireType6_Field1_Version0
 Required.Proto3.ProtobufInput.UnknownWireType6_Field1_Version1
 Required.Proto3.ProtobufInput.UnknownWireType6_Field1_Version2


@@ 144,16 105,4 @@ Required.Proto3.ProtobufInput.UnknownWireType7_Field3_Version1
 Required.Proto3.ProtobufInput.UnknownWireType7_Field3_Version2
 Required.Proto3.ProtobufInput.UnknownWireType7_Field3_Version3
 Required.Proto3.ProtobufInput.ValidDataOneof.MESSAGE.Merge.ProtobufOutput
-Required.Proto3.ProtobufInput.ValidDataRepeated.ENUM.PackedInput.ProtobufOutput
-Required.Proto3.ProtobufInput.ValidDataRepeated.ENUM.UnpackedInput.ProtobufOutput
-Required.Proto3.ProtobufInput.ValidDataRepeated.INT32.PackedInput.ProtobufOutput
-Required.Proto3.ProtobufInput.ValidDataRepeated.INT32.UnpackedInput.ProtobufOutput
-Required.Proto3.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.ProtobufOutput
-Required.Proto3.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.ProtobufOutput
-Required.Proto3.ProtobufInput.ValidDataScalar.ENUM[4].ProtobufOutput
-Required.Proto3.ProtobufInput.ValidDataScalar.ENUM[5].ProtobufOutput
-Required.Proto3.ProtobufInput.ValidDataScalar.INT32[8].ProtobufOutput
-Required.Proto3.ProtobufInput.ValidDataScalar.INT32[9].ProtobufOutput
-Required.Proto3.ProtobufInput.ValidDataScalar.UINT32[8].ProtobufOutput
-Required.Proto3.ProtobufInput.ValidDataScalar.UINT32[9].ProtobufOutput
 Required.Proto3.TimestampProtoNegativeNanos.JsonOutput