From afbbf143a7da90873ddcc53abaf6f424023c0705 Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Mon, 25 May 2026 09:32:59 +0300 Subject: [PATCH] ci: conformance suite with Garage S3 image cache (7lf) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two new srht.bigb.es manifests: .builds/conformance.yml — auto-submits on push to master/tags. Pulls the conformance Docker image from s3://docker-cache/ keyed by sha256(docker/conformance.Dockerfile); on cache miss, builds it and uploads. Runs both 'just conformance' and the C-runtime variant against the cached image. .builds/conformance-image-rebuild.yml — manual-submit only (allow-refs: []). Force-rebuilds with --no-cache --pull and replaces the S3 object. Use when the cached image needs refresh without a Dockerfile change (base-image CVE, host docker upgrade, corrupted upload). Garage S3 specifics: - addressing_style = path (Garage doesn't do virtual-host) - signature_version = s3v4 (Garage rejects SigV2) - region = garage (matches phoebe-lab/garage/config/garage.toml.tmpl) - head-object for cache-hit check (s3 ls returns exit 0 on missing keys; head-object 404s cleanly) - Endpoint https://s3.bigb.es works from the build worker because srht-build-1 (192.168.88.128) is in Authelia's *.bigb.es LAN-bypass range (192.168.88.0/24) Credentials come from two file secrets registered at https://builds.srht.bigb.es/secrets, mounted as ~/.s3-cache-key-id and ~/.s3-cache-key-secret. Loaded under 'set +x' and persisted to ~/.buildenv via printf %q so subsequent tasks pick them up without re-tracing. The bucket 'docker-cache' and its bound key must exist on Garage with read+write permission for the supplied access key — first push will fail at the smoke-test 'aws s3 ls' line otherwise. (7lf) --- .beads/issues.jsonl | 2 +- .builds/conformance-image-rebuild.yml | 68 +++++++++++++++ .builds/conformance.yml | 114 ++++++++++++++++++++++++++ 3 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 .builds/conformance-image-rebuild.yml create mode 100644 .builds/conformance.yml diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 16881044240eeca291287404e3267087e260ab0e..be701014d8f03b753b8b9a850988d3e00d7c97a1 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -39,7 +39,7 @@ {"_type":"issue","id":"tarantool-protobuf-wky","title":"C accel: build and packaging support for C modules","description":"Add build-system and packaging support for C acceleration. Needs Justfile targets, rockspec support for compiled modules, platform naming, local dev build, CI matrix integration, and fallback when the C module is absent. Generated C backend must not make pure-Lua install impossible unless explicitly selected.","notes":"Architecture finalized in docs/c-accel.md. Concrete deliverables: (1) Justfile recipe 'just build-c' that builds the C runtime module (mirror bench/c_accel/Makefile auto-detection of TT_INC). (2) Rockspec optionally builds the C module — install on host without cc must succeed and produce a pure-Lua install. (3) runtime/pb/init.lua does pcall(require, 'pb.c_runtime') only when os.getenv('PB_ENABLE_C') == '1'. (4) Sourcehut CI build manifest gains one extra job that runs the full test+conformance suite with PB_ENABLE_C=1 set — single .build.yml, both modes covered on every push to master. (5) CI also runs one variant without the C module compiled to confirm pure-Lua install works.","status":"closed","priority":2,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-17T16:32:52Z","created_by":"Eugene Blikh","updated_at":"2026-05-19T04:24:13Z","started_at":"2026-05-19T04:10:45Z","closed_at":"2026-05-19T04:24:13Z","close_reason":"Build and packaging scaffolding landed: docs/specs/c_accel_build_packaging.md spec, runtime/pb/init.lua PB_ENABLE_C=1 opt-in pcall hook (exposed as pb.c_runtime field), Justfile build-c/clean-c recipes (stub erroring cleanly until bd-ra6 lands runtime/pb/c/), Justfile lua_cpath constant + LUA_CPATH on test/test-one, .gitignore for compiled .so/.dylib, sourcehut .builds/{pure-lua,c-enabled}.yml manifests (ubuntu/noble, parallel jobs via separate files since sourcehut has no matrix), .sourcehut/conformance.yml for manual conformance triggers. Rockspec changes deferred to first C-source landing (described in spec). All 745 tests pass with PB_ENABLE_C unset and PB_ENABLE_C=1 (silent fallback verified).","dependencies":[{"issue_id":"tarantool-protobuf-wky","depends_on_id":"tarantool-protobuf-pf6","type":"blocks","created_at":"2026-05-17T19:33:11Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-801","title":"Release: tagged releases + rockspec + Go plugin binaries","description":"Once the CI pipeline is green, ship tagged releases. Rockspec for the Lua runtime (publishable via tt rocks). Pre-built protoc-gen-tarantool binaries for darwin/arm64, darwin/amd64, linux/amd64, linux/arm64. Sourcecraft has 'sc release' tooling (see sc-release skill) for the release workflow itself.","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:16Z","created_by":"Eugene Blikh","updated_at":"2026-05-17T15:47:16Z","labels":["release"],"dependencies":[{"issue_id":"tarantool-protobuf-801","depends_on_id":"tarantool-protobuf-1eu","type":"blocks","created_at":"2026-05-17T18:47:20Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-1eu","title":"Release: Sourcecraft.dev project + CI pipeline","description":"Set up the canonical sourcecraft.dev project for the repo and a CI pipeline. Matrix: Tarantool 2.11 (CE+EE) and 3.x (CE+EE), Linux + macOS. Targets to run: just gen, just test, just bench-compare (alloc regression gate), just conformance (gated on the cached Docker image — see related CI wire-up bead). The repo lives under ~/data/home which by convention publishes to sourcecraft.dev (not github).","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:15Z","created_by":"Eugene Blikh","updated_at":"2026-05-17T15:47:15Z","labels":["ci","release"],"dependencies":[{"issue_id":"tarantool-protobuf-1eu","depends_on_id":"tarantool-protobuf-7lf","type":"blocks","created_at":"2026-05-17T18:47:20Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} -{"_type":"issue","id":"tarantool-protobuf-7lf","title":"CI: wire up conformance suite with cached Docker image","description":"The Docker image build (docker/conformance.Dockerfile) is the long pole at ~10-15 min on a clean cache. A registry push from a scheduled job would let CI runs reuse a warm image. Today the conformance suite runs locally via 'just conformance' but isn't gated on pushes. Goal: every push to master runs the binary+JSON and text-format suites; PRs run the same. Pre-requisite for M8 sourcecraft setup (this defines what the CI pipeline runs).","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:15Z","created_by":"Eugene Blikh","updated_at":"2026-05-17T15:47:15Z","labels":["ci","conformance"],"dependency_count":0,"dependent_count":1,"comment_count":0} +{"_type":"issue","id":"tarantool-protobuf-7lf","title":"CI: wire up conformance suite with cached Docker image","description":"The Docker image build (docker/conformance.Dockerfile) is the long pole at ~10-15 min on a clean cache. A registry push from a scheduled job would let CI runs reuse a warm image. Today the conformance suite runs locally via 'just conformance' but isn't gated on pushes. Goal: every push to master runs the binary+JSON and text-format suites; PRs run the same. Pre-requisite for M8 sourcecraft setup (this defines what the CI pipeline runs).","status":"in_progress","priority":2,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:15Z","created_by":"Eugene Blikh","updated_at":"2026-05-25T06:32:38Z","started_at":"2026-05-25T06:32:38Z","labels":["ci","conformance"],"dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-0an","title":"Profile the 1 KB decode cliff in hello.Person","description":"bench/bench.lua shows hello.Person decode dropping 260 MB/s @ 100 B to 128 MB/s @ 1 KB — per-MB throughput halves at the payload-shape transition (scalars-only -\u003e emails[] + nested address + packed lucky_numbers). starwing's C decoder scales smoothly across the same transition. Smoking gun: a trace abort or side-trace stitching failure at the array/nested branch. Capture jit.dump=tbim over the transition, identify the abort, fix it. Likely small change once located. Prereq for several decoder fixes — pin down what we're actually hitting first.","notes":"2026-05-17 inspection vs starwing/lua-protobuf 0.5.3 on Tarantool 3.8.0: reproduced starwing hello.Person encode at 75/688/888/1391/1431 MB/s and decode at 11.5/101.6/378/698.7/702.8 MB/s for 10B/100B/1KB/10KB/100KB. Current full mode is encode 30/257/287/541/643 MB/s and decode 30.7/247.5/128.8/186.4/184.3 MB/s. So we beat starwing on tiny decode, but lose 2.9-3.8x once Person switches to repeated strings + nested Address + packed ints. jit_trace passes including multi-byte varint paths; no fatal abort or bridge smoking gun. Focused probes on 930B Person: generated full decode ~6.9-7.5 us/op; scan tags+skip_field ~3.6 us/op; scan tags/lengths only ~2.1 us/op; hand order-specialized decoder with same string extraction/Address decode/lucky parse ~4.3 us/op. Patching decode_string to skip UTF-8 improves only ~16% (7.1 -\u003e 6.1 us/op), so UTF-8 is material but not the main cliff. Likely cause: per-field Lua dispatch/function-call/table growth overhead repeated across ~30 LEN/VARINT fields; C starwing parses from pointer/buffer and lands near our tag-only scan cost. Existing follow-ups bgu (ffi.string string decode), gcy (inline nested decode), and 0u1 (typed decoder byte fast paths) are relevant, but the largest remaining gap likely needs generated ordered/specialized decode loops or ptr+offset readers that avoid decode_tag/decode_len helper dispatch and substring slicing.","status":"closed","priority":2,"issue_type":"bug","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:12Z","created_by":"Eugene Blikh","updated_at":"2026-05-17T16:02:54Z","started_at":"2026-05-17T15:56:33Z","closed_at":"2026-05-17T16:02:54Z","close_reason":"Profiled against starwing and local focused probes. No fatal JIT abort found; cliff attributed mainly to repeated Lua tag/field dispatch plus LEN/string slicing/validation. Follow-up fast-path work captured in tarantool-protobuf-4kj and existing decoder tasks.","labels":["decoder","investigation","perf"],"dependency_count":0,"dependent_count":3,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-b12","title":"Encoder: parallel-array iteration for proto2 extensions","description":"Mirror the desc.oneofs_list finalize-time pattern: build desc.extensions_list once in pb.finalize_message, iterate with 'for i=1,#' in encode instead of walking the descriptor's hash table on every call. The 4-7x proto2 gap (vs 2-4x proto3 in bench/starwing_bench.lua) is partly explained by hash-walked extensions on the hot path. Expected: brings proto2 BenchPayload encode closer to proto3 Person encode at the same shape.","status":"closed","priority":2,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:12Z","created_by":"Eugene Blikh","updated_at":"2026-05-24T19:00:34Z","started_at":"2026-05-24T18:59:15Z","closed_at":"2026-05-24T19:00:34Z","close_reason":"Already fixed in commit 2656c97 (2026-05-17, same day issue filed). register_extension appends to extensions_list (parallel array); codec.encode_message, text.emit_message, json.encode_message all iterate via 'for i=1,#elist'. Inline-mode codegen also walks the list via 'for _i = N+1, #_elist' for runtime-registered extensions past the statically-known set. No pairs() over extensions remains anywhere on the hot path. The issue was never closed in bd.","labels":["encoder","perf","proto2"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-bgu","title":"Decoder: ffi.string(ptr+off, len) for string/bytes decode","description":"Replace buf:sub(npos, npos+len-1) in decode_string/decode_bytes with ffi.string against a cached ffi.cast('const uint8_t*', buf). Removes method-call dispatch and lets the JIT fuse the read with surrounding code. Hits hardest on repeated-string fields (26 emails strings in 1KB Person -\u003e 26 sub calls today). Watch for LuaJIT string-intern collision behavior to stay identical to sub().","status":"closed","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:10Z","created_by":"Eugene Blikh","updated_at":"2026-05-24T04:20:11Z","closed_at":"2026-05-24T04:20:11Z","close_reason":"Spike (bench/c_accel/bgu_probe.lua, removed) measured ffi.string vs buf:sub for the 1-byte LEN string fast path on Tarantool 3.8.0/LuaJIT 2.1.0-beta3 / Apple M-series. Results (ns/call):\n\n size buf:sub per-call-cast amortized/26 amortized/4\n 10B 89.7 280.5 (3.1x) 224.9 (2.5x) 246.2 (2.7x)\n 32B 107.6 299.4 (2.8x) 243.8 (2.3x) 260.9 (2.4x)\n 80B 154.3 362.4 (2.3x) 300.1 (1.9x) 314.9 (2.0x)\n\nV1 (per-call ffi.cast(U8CP, buf) inside decode_string): 2.3-3.1x slower.\nV2 (cast hoisted to caller, threaded as ptr arg, amortized over 26 emails — the Person 1KB peak shape): still 1.9-2.5x slower. Cast-once-amortize-many fails because 'ptr + np - 1' mints a fresh cdata wrapper per call that the JIT cannot sink across the decode_string frame — same root cause as a6n's failed top-of-_decode cast. The amortization helps a little (V2 \u003c V1) but does not close the gap.\n\nConclusion: bgu's premise (ffi.string lets the JIT fuse the string read with surrounding code) does not hold on the current LuaJIT fork — the cdata allocation cost dominates any reduction in dispatch. Closing as superseded by a6n-decode-inline-len-fast-path-2026-05-23. Revival criteria: a measurable cdata-sink win on a future LuaJIT (likely needs upstream allocation-sink improvements for pointer-arith cdata), OR an API redesign where the decoder works on a cdata buffer end-to-end (not a Lua string).","labels":["decoder","perf","wire"],"dependencies":[{"issue_id":"tarantool-protobuf-bgu","depends_on_id":"tarantool-protobuf-0an","type":"blocks","created_at":"2026-05-17T18:47:21Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} diff --git a/.builds/conformance-image-rebuild.yml b/.builds/conformance-image-rebuild.yml new file mode 100644 index 0000000000000000000000000000000000000000..d105cfc3079c46b72c36f2f704280c4d8f04ce5d --- /dev/null +++ b/.builds/conformance-image-rebuild.yml @@ -0,0 +1,68 @@ +# Force-rebuild the conformance Docker image and upload it to S3 under the +# current Dockerfile-content hash. Use when the cached image needs refresh +# without a Dockerfile change (base-image CVE pickup, host docker upgrade, +# corrupted upload). Auto-submit is disabled — invoke manually: +# +# hut --instance srht.bigb.es builds submit .builds/conformance-image-rebuild.yml +# +# Normal CI uses .builds/conformance.yml, which lazy-fills the same S3 key +# on the first push after a Dockerfile change. +image: ubuntu/noble +packages: + - curl + - ca-certificates + - docker.io + - awscli + - zstd +secrets: + - a42011da-a362-40da-9db6-1bf1f9be8be3 # ~/.s3-cache-key-id + - c0cf5f2d-5d63-4d50-a324-5eaecaa992dd # ~/.s3-cache-key-secret +sources: + - https://git.srht.bigb.es/~bigbes/tarantool-protobuf +environment: + AWS_DEFAULT_REGION: garage + AWS_ENDPOINT_URL: https://s3.bigb.es + S3_BUCKET: docker-cache + S3_PREFIX: tarantool-protobuf/conformance-images +submitter: + git.sr.ht: + enabled: false + allow-refs: [] +tasks: + - aws_credentials: | + mkdir -p ~/.aws + cat > ~/.aws/config <> ~/.buildenv + set -x + - docker_start: | + sudo systemctl start docker + sudo usermod -aG docker build + sudo chmod 666 /var/run/docker.sock + docker version + - rebuild: | + cd tarantool-protobuf + HASH=$(sha256sum docker/conformance.Dockerfile | cut -c1-16) + OBJECT_KEY="$S3_PREFIX/$HASH.tar.zst" + S3_URI="s3://$S3_BUCKET/$OBJECT_KEY" + echo "Image hash: $HASH" + echo "S3 URI: $S3_URI" + docker build --no-cache --pull \ + -t tarantool-protobuf-conformance:latest \ + -f docker/conformance.Dockerfile docker/ + docker save tarantool-protobuf-conformance:latest \ + | zstd -T0 -3 \ + | aws --endpoint-url="$AWS_ENDPOINT_URL" s3 cp - "$S3_URI" + aws --endpoint-url="$AWS_ENDPOINT_URL" s3api head-object \ + --bucket "$S3_BUCKET" --key "$OBJECT_KEY" diff --git a/.builds/conformance.yml b/.builds/conformance.yml new file mode 100644 index 0000000000000000000000000000000000000000..728fbc4aff80c7f8f011d36065623237950ec639 --- /dev/null +++ b/.builds/conformance.yml @@ -0,0 +1,114 @@ +image: ubuntu/noble +packages: + - curl + - ca-certificates + - gnupg + - protobuf-compiler + - cmake + - build-essential + - docker.io + - awscli + - zstd +secrets: + - a42011da-a362-40da-9db6-1bf1f9be8be3 # ~/.s3-cache-key-id + - c0cf5f2d-5d63-4d50-a324-5eaecaa992dd # ~/.s3-cache-key-secret +sources: + - https://git.srht.bigb.es/~bigbes/tarantool-protobuf +environment: + GOPATH: /home/build/go + GOROOT: /home/build/.local/go + GO_VERSION: "1.26.3" + PATH: /home/build/.local/go/bin:/home/build/.local/bin:/home/build/go/bin:/usr/local/bin:/usr/bin:/bin + AWS_DEFAULT_REGION: garage + AWS_ENDPOINT_URL: https://s3.bigb.es + S3_BUCKET: docker-cache + S3_PREFIX: tarantool-protobuf/conformance-images +submitter: + git.sr.ht: + enabled: true + allow-refs: + - refs/heads/master + - "refs/tags/*" +tasks: + - aws_credentials: | + mkdir -p ~/.aws + cat > ~/.aws/config <> ~/.buildenv + set -x + aws --endpoint-url="$AWS_ENDPOINT_URL" s3 ls "s3://$S3_BUCKET/" >/dev/null + - install_go: | + mkdir -p ~/.local + curl -sSL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" \ + | tar -xz -C ~/.local + go version + - install_tarantool: | + curl -L https://tarantool.io/release/3/installer.sh | sudo bash + sudo apt-get install -y tarantool tarantool-dev tt + - install_just: | + mkdir -p ~/.local/bin + curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh \ + | bash -s -- --to ~/.local/bin + - install_luatest: | + cd tarantool-protobuf + tt rocks install luatest + - docker_start: | + sudo systemctl start docker + sudo usermod -aG docker build + sudo chmod 666 /var/run/docker.sock + docker version + - conformance_image: | + cd tarantool-protobuf + HASH=$(sha256sum docker/conformance.Dockerfile | cut -c1-16) + OBJECT_KEY="$S3_PREFIX/$HASH.tar.zst" + S3_URI="s3://$S3_BUCKET/$OBJECT_KEY" + echo "Image hash: $HASH" + echo "S3 URI: $S3_URI" + if aws --endpoint-url="$AWS_ENDPOINT_URL" s3api head-object \ + --bucket "$S3_BUCKET" --key "$OBJECT_KEY" >/dev/null 2>&1; then + echo "Cache HIT — pulling cached image" + aws --endpoint-url="$AWS_ENDPOINT_URL" s3 cp "$S3_URI" - \ + | unzstd \ + | docker load + else + echo "Cache MISS — building from scratch and uploading" + docker build -t tarantool-protobuf-conformance:latest \ + -f docker/conformance.Dockerfile docker/ + docker save tarantool-protobuf-conformance:latest \ + | zstd -T0 -3 \ + | aws --endpoint-url="$AWS_ENDPOINT_URL" s3 cp - "$S3_URI" + fi + docker image inspect tarantool-protobuf-conformance:latest >/dev/null + - codegen: | + cd tarantool-protobuf + just gen + - conformance: | + cd tarantool-protobuf + docker run --rm -v "$(pwd):/work" -w /work \ + tarantool-protobuf-conformance:latest + - conformance_c: | + cd tarantool-protobuf + docker run --rm -v "$(pwd):/work" -w /work -e PB_ENABLE_C=1 \ + --entrypoint bash tarantool-protobuf-conformance:latest -c ' + set -e + make -C runtime/pb/c clean >/dev/null + make -C runtime/pb/c >/dev/null + conformance_test_runner --enforce_recommended \ + --failure_list test/conformance/known_failures.txt \ + --text_format_failure_list test/conformance/known_failures_text.txt \ + /usr/bin/tarantool cmd/conformance-runner.lua + rc=$? + make -C runtime/pb/c clean >/dev/null + exit $rc + '