From 6d0ec7bf3b4e1a051bc75efee59c9cd3d5ed3f04 Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Sat, 16 May 2026 17:43:24 +0300 Subject: [PATCH] =?UTF-8?q?wire:=20decode=20wins=20=E2=80=94=20utf8.len=20?= =?UTF-8?q?validator=20+=20fast=20paths=20+=20FFI=20cast?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five focused decode optimizations, all in wire.lua, landing a 3.9-6.6x speedup on bench/bench.lua decode and matching wins on lazy / shapes benches. Per-helper numbers from bench/wire_bench.lua: is_valid_utf8(32B ASCII): 545 ns -> 36 ns (15x) is_valid_utf8(1KB ASCII): 16329 ns -> 539 ns (30x) decode_string(32B): 140 ns -> 80 ns (1.75x) decode_double: 299 ns -> 226 ns (1.32x) decode_fixed64: 196 ns -> 184 ns (1.07x) 1. utf8.len swap. Pure-Lua RFC 3629 validator replaced by `utf8.len(s) ~= nil` (ICU U8_NEXT-backed at src/lua/utf8.c:165). Source-verified to reject every proto3 case: stray continuation, overlong, surrogates, truncated, > U+10FFFF, 5-byte+ sequences. Conformance suite still at 1478/1478 expected passes. 2. decode_string / decode_bytes 1-byte LEN fast path. Strings <=127 bytes (the RPC common case) skip two function-call layers (decode_string -> decode_len -> decode_varint). 3. decode_float / decode_double via ffi.cast(uint8_t*, buf) instead of buf:sub. Eliminates the per-call string-slice allocation. 4. decode_double Inf/NaN check via a uint32[2] union split. Bit ops on Lua numbers don't allocate; the previous uint64 cdata path produced 3+ intermediates per call. 5. decode_fixed64 reads via the new pb_u64_u_t union (ffi.copy + read .u). Replaces UINT64(lo) + bit.lshift(UINT64(hi), 32) which allocated 2-3 cdata per call. All FFI cdef/union locals hoisted to the top of the file so the fixed-width decoders all reference the same scratch buffers. 497/497 luatest pass. 19/19 jit-trace gates pass. Conformance (binary + text) shows zero unexpected failures. --- runtime/pb/wire.lua | 148 +++++++++++++++++++++++--------------------- 1 file changed, 79 insertions(+), 69 deletions(-) diff --git a/runtime/pb/wire.lua b/runtime/pb/wire.lua index cb1cef8c976768d9950830d3b54fc2e2a3888b2e..40eedb77ce6546a93b6dba614a59cca094ceea2a 100644 --- a/runtime/pb/wire.lua +++ b/runtime/pb/wire.lua @@ -16,6 +16,25 @@ local INT64 = ffi.typeof('int64_t') local UINT64_ZERO = UINT64(0) local CONT_MASK = UINT64(bit.bnot(0x7f)) -- 0xFFFFFFFFFFFFFF80 +-- FFI scratch unions used by the fixed-width decoders. Allocated once +-- and reused — `ffi.copy` from a `uint8_t*` cast over the Lua string body +-- writes into the union's byte view; reading back via `.u` / `.d` / +-- `.u32[i]` reinterprets the bytes without further allocation. +ffi.cdef[[ + typedef union { float f; uint32_t u; uint8_t b[4]; } pb_f32_u_t; + typedef union { + double d; + uint64_t u; + uint32_t u32[2]; /* [0] = low half, [1] = high half (LE) */ + uint8_t b[8]; + } pb_f64_u_t; + typedef union { uint64_t u; uint32_t u32[2]; uint8_t b[8]; } pb_u64_u_t; +]] +local U8CP = ffi.typeof('const uint8_t *') +local F32 = ffi.new('pb_f32_u_t') +local F64 = ffi.new('pb_f64_u_t') +local U64 = ffi.new('pb_u64_u_t') + -- Coerce any integer-like value to uint64_t cdata. -- Negative Lua numbers are sign-extended via int64_t (protobuf wire spec). local function to_uint64(v) @@ -246,12 +265,14 @@ end M.encode_fixed64 = encode_fixed64 -- decode_fixed64(buf, pos) -> uint64_t cdata, new_pos +-- Reading via the FFI union avoids the multi-step `UINT64(lo) + +-- bit.lshift(UINT64(hi), 32)` cdata arithmetic, which allocated 2-3 +-- intermediate cdata per call. ffi.copy from a uint8_t* cast over the +-- Lua string body skips the buf:sub allocation entirely. local function decode_fixed64(buf, pos) - local b1, b2, b3, b4, b5, b6, b7, b8 = buf:byte(pos, pos + 7) - if b8 == nil then error("truncated fixed64", 0) end - local lo = b1 + b2 * 0x100 + b3 * 0x10000 + b4 * 0x1000000 - local hi = b5 + b6 * 0x100 + b7 * 0x10000 + b8 * 0x1000000 - return UINT64(lo) + bit.lshift(UINT64(hi), 32), pos + 8 + if pos + 7 > #buf then error("truncated fixed64", 0) end + ffi.copy(U64.b, ffi.cast(U8CP, buf) + (pos - 1), 8) + return U64.u, pos + 8 end M.decode_fixed64 = decode_fixed64 @@ -259,23 +280,19 @@ M.decode_fixed64 = decode_fixed64 -- Float / Double (IEEE 754 little-endian) -- --------------------------------------------------------------------------- -ffi.cdef[[ - typedef union { float f; uint32_t u; uint8_t b[4]; } pb_f32_u_t; - typedef union { double d; uint64_t u; uint8_t b[8]; } pb_f64_u_t; -]] - -local F32 = ffi.new('pb_f32_u_t') -local F64 = ffi.new('pb_f64_u_t') - local function encode_float(n) F32.f = n return ffi.string(F32.b, 4) end M.encode_float = encode_float +-- ffi.cast over the Lua string body (`U8CP`) avoids the buf:sub +-- allocation (saves ~40 bytes per fixed-width read on the GC, and one +-- alloc per call). Lua strings are immutable + zero-terminated, so the +-- pointer is valid for the duration of the cast. local function decode_float(buf, pos) if pos + 3 > #buf then error("truncated float", 0) end - ffi.copy(F32.b, buf:sub(pos, pos + 3), 4) + ffi.copy(F32.b, ffi.cast(U8CP, buf) + (pos - 1), 4) -- Detect Inf/NaN from the raw bit pattern before going through -- tonumber(). LuaJIT 2.1 NaN-boxes Lua values, so some IEEE NaN -- payloads collide with internal type tags (nil/function/etc.) and @@ -300,24 +317,20 @@ local function encode_double(n) end M.encode_double = encode_double -local F64_EXP_MASK = UINT64(0x7ff) -local F64_FRAC_MASK = UINT64(0xfffffffffffff) -local F64_SIGN_BIT = bit.lshift(UINT64(1), 63) - local function decode_double(buf, pos) if pos + 7 > #buf then error("truncated double", 0) end - ffi.copy(F64.b, buf:sub(pos, pos + 7), 8) + ffi.copy(F64.b, ffi.cast(U8CP, buf) + (pos - 1), 8) -- Detect Inf/NaN from the raw bit pattern before going through -- tonumber(). LuaJIT 2.1 NaN-boxes Lua values, so some IEEE NaN -- payloads collide with internal type tags (nil/function/etc.) and - -- `tonumber(F64.d)` yields a non-number. Reading via F64.u (uint64) - -- keeps us in the integer domain until we decide what to return. - local u = F64.u - if bit.band(bit.rshift(u, 52), F64_EXP_MASK) == F64_EXP_MASK then - if bit.band(u, F64_FRAC_MASK) == UINT64_ZERO then - if bit.band(u, F64_SIGN_BIT) ~= UINT64_ZERO then - return -math.huge, pos + 8 - end + -- `tonumber(F64.d)` yields a non-number. Splitting the uint64 into + -- two uint32 halves lets the bit ops stay in Lua-number space — + -- avoids per-op uint64 cdata allocation in the hot path. + local hi = F64.u32[1] + if bit.band(bit.rshift(hi, 20), 0x7ff) == 0x7ff then + local frac_hi = bit.band(hi, 0xfffff) + if frac_hi == 0 and F64.u32[0] == 0 then + if bit.band(hi, 0x80000000) ~= 0 then return -math.huge, pos + 8 end return math.huge, pos + 8 end return 0/0, pos + 8 @@ -455,59 +468,56 @@ M.decode_sfixed64 = decode_sfixed64 -- RFC 3629 UTF-8 validator. Rejects: out-of-range continuation bytes, -- truncated multi-byte sequences, overlong encodings, UTF-16 surrogate -- code points (U+D800..U+DFFF), and code points above U+10FFFF. +-- +-- Tarantool ships an ICU-backed `utf8.len` (src/lua/utf8.c:165) that uses +-- `U8_NEXT` for validation — it returns nil on the first invalid sequence +-- and matches every proto3 rejection case (overlong, surrogate, > U+10FFFF, +-- 5-byte). Empirically 15-30x faster than a pure-Lua `string.byte` loop on +-- ASCII payloads (decode throughput on string-heavy 1KB Person: ~1.9x). +local utf8_len = require('utf8').len local function is_valid_utf8(s) - local i, n = 1, #s - while i <= n do - local b = s:byte(i) - if b < 0x80 then - i = i + 1 - elseif b < 0xC2 then - return false -- stray continuation or overlong 2-byte - elseif b < 0xE0 then - if i + 1 > n then return false end - local b2 = s:byte(i + 1) - if b2 < 0x80 or b2 > 0xBF then return false end - i = i + 2 - elseif b < 0xF0 then - if i + 2 > n then return false end - local b2 = s:byte(i + 1) - local b3 = s:byte(i + 2) - if b == 0xE0 and b2 < 0xA0 then return false end -- overlong - if b == 0xED and b2 > 0x9F then return false end -- surrogate - if b2 < 0x80 or b2 > 0xBF or b3 < 0x80 or b3 > 0xBF then - return false - end - i = i + 3 - elseif b < 0xF5 then - if i + 3 > n then return false end - local b2 = s:byte(i + 1) - local b3 = s:byte(i + 2) - local b4 = s:byte(i + 3) - if b == 0xF0 and b2 < 0x90 then return false end -- overlong - if b == 0xF4 and b2 > 0x8F then return false end -- > U+10FFFF - if b2 < 0x80 or b2 > 0xBF - or b3 < 0x80 or b3 > 0xBF - or b4 < 0x80 or b4 > 0xBF then - return false - end - i = i + 4 - else - return false -- 5-byte+ sequences or 0xF5..0xFF - end - end - return true + return utf8_len(s) ~= nil end M.is_valid_utf8 = is_valid_utf8 +-- Inlined 1-byte LEN fast path. Same rationale as the inlined varint +-- fast paths above (see decode_tag): strings ≤127 bytes are the common +-- RPC case; folding their length-prefix read in here skips two function +-- call layers (decode_string → decode_len → decode_varint) and lets the +-- JIT keep the trace single-rooted across the caller. local function decode_string(buf, pos) + local b = buf:byte(pos) + if b == nil then error("truncated varint at offset " .. pos, 0) end + if b < 0x80 then + local np = pos + 1 + local epos = np + b + if epos - 1 > #buf then error("truncated LEN payload", 0) end + local s = buf:sub(np, epos - 1) + if utf8_len(s) == nil then + error("invalid UTF-8 in string field at offset " .. pos, 0) + end + return s, epos + end local s, np = decode_len(buf, pos) - if not is_valid_utf8(s) then + if utf8_len(s) == nil then error("invalid UTF-8 in string field at offset " .. pos, 0) end return s, np end M.decode_string = decode_string -M.decode_bytes = decode_len + +local function decode_bytes(buf, pos) + local b = buf:byte(pos) + if b == nil then error("truncated varint at offset " .. pos, 0) end + if b < 0x80 then + local np = pos + 1 + local epos = np + b + if epos - 1 > #buf then error("truncated LEN payload", 0) end + return buf:sub(np, epos - 1), epos + end + return decode_len(buf, pos) +end +M.decode_bytes = decode_bytes -- (decode_fixed32, decode_fixed64, decode_float, decode_double already on M -- and have the right semantics for their proto types: fixed32 -> uint32 Lua -- number 0..2^32-1, fixed64 -> uint64 cdata.)