~bigbes/tarantool

tarantool-protobuf

89cc50084e0fd47c6a9bddfe749a5a5284730849 — Eugene Blikh 3 months ago 8014163 
codec: specialize writers for repeated fields (+20–80% encode across sizes)

Extends compile_writers to handle repeated scalar (packed and unpacked),
repeated message, and repeated enum (packed and unpacked). Each writer
knows its tag bytes, encoder function, and packed/unpacked shape; the
encode_message loop calls them directly without rediscovering the field
shape on every iteration.

Combined with the singular-field writers (previous commit), this lifts
runtime-mode encode throughput uniformly:

  runtime/10B   encode:  5.0 →  9.1 MB/s  (+82%)
  runtime/100B  encode:   48 →   82 MB/s  (+70%)
  runtime/1KB   encode:   50 →   69 MB/s  (+39%)
  runtime/10KB  encode:   87 →  101 MB/s  (+16%)
  runtime/100KB encode:   90 →  111 MB/s  (+22%)

  full mode encode: +10% on small sizes, +10% on large (small bonus from
                    nested encode_msg calls going through writers too)
  decode: small uplift on full mode 100B (323 → 340 MB/s), noise on rest
  alloc/op: unchanged (bench-compare clean)
  bridges across 10 jit-trace runs: 7 → 3 (-57%; total -89% from baseline)

Falls through to encode_field only for map fields and oneof branches —
both keep their existing dispatch path.
patch browse .tar.gz .zip 
1 files changed, 95 insertions(+), 3 deletions(-)

M runtime/pb/codec.lua

M runtime/pb/codec.lua => runtime/pb/codec.lua +95 -3
@@ 224,15 224,107 @@ end
--
-- Shapes that fall through to encode_field (no writer set):
--   - map fields (need `pairs()` over user data; can't be helped)
--   - repeated fields (TODO — common shape, worth specializing)
--   - fields inside a oneof (active-branch dispatch happens in encode_message)
-- ---------------------------------------------------------------------------

local function build_repeated_writer(f)
    local fname = f.name
    local kind  = f.kind

    if kind == 'scalar' then
        local handler = scalar[f.proto_type]
        if not handler then return nil end
        local encode_value = handler.encode

        if f.packed and handler.packable then
            local tag_bytes = wire.encode_tag(f.id, wire.WIRE_LEN)
            return function(data, out)
                local v = data[fname]
                if v == nil then return end
                local nv = #v
                if nv == 0 then return end
                local parts = {}
                for i = 1, nv do parts[i] = encode_value(v[i]) end
                local payload = table.concat(parts)
                local n = #out
                out[n + 1] = tag_bytes
                out[n + 2] = wire.encode_len(payload)
            end
        end

        -- Unpacked repeated scalar (also covers string/bytes — LEN
        -- wire type, which is non-packable by proto3 rules).
        local tag_bytes = wire.encode_tag(f.id, handler.wire)
        return function(data, out)
            local v = data[fname]
            if v == nil then return end
            local nv = #v
            if nv == 0 then return end
            local n = #out
            for i = 1, nv do
                n = n + 1; out[n] = tag_bytes
                n = n + 1; out[n] = encode_value(v[i])
            end
        end
    end

    if kind == 'message' then
        local sub_desc  = f.message
        local tag_bytes = wire.encode_tag(f.id, wire.WIRE_LEN)
        return function(data, out)
            local v = data[fname]
            if v == nil then return end
            local nv = #v
            if nv == 0 then return end
            local n = #out
            for i = 1, nv do
                n = n + 1; out[n] = tag_bytes
                n = n + 1; out[n] = wire.encode_len(encode_msg(sub_desc, v[i]))
            end
        end
    end

    if kind == 'enum' then
        local enum_desc = f.enum
        -- proto3 default: repeated enums are packed unless explicitly disabled.
        if f.packed ~= false then
            local tag_bytes = wire.encode_tag(f.id, wire.WIRE_LEN)
            return function(data, out)
                local v = data[fname]
                if v == nil then return end
                local nv = #v
                if nv == 0 then return end
                local parts = {}
                for i = 1, nv do
                    parts[i] = wire.encode_varint(encode_enum_value(enum_desc, v[i]))
                end
                local n = #out
                out[n + 1] = tag_bytes
                out[n + 2] = wire.encode_len(table.concat(parts))
            end
        end
        local tag_bytes = wire.encode_tag(f.id, wire.WIRE_VARINT)
        return function(data, out)
            local v = data[fname]
            if v == nil then return end
            local nv = #v
            if nv == 0 then return end
            local n = #out
            for i = 1, nv do
                n = n + 1; out[n] = tag_bytes
                n = n + 1; out[n] = wire.encode_varint(encode_enum_value(enum_desc, v[i]))
            end
        end
    end

    return nil
end

local function build_writer(f)
    -- Maps and oneof branches keep going through encode_field.
    if f.kind == 'map' or f.oneof then return nil end
    -- Repeated fields fall through for now.
    if f.repeated then return nil end

    if f.repeated then return build_repeated_writer(f) end

    local fname  = f.name
    local kind   = f.kind