From 817601a0f3f968376c474805bd0b1b28981f63cb Mon Sep 17 00:00:00 2001 From: John Dvorak Date: Sun, 2 Aug 2026 20:26:36 -0700 Subject: [PATCH] ci: run complex-query cold-traffic benchmarks in the benchmark job The prod-gating measurement (complex-query-bench: tuple-to-userset, chains, defeasible, comparators, OWA, nested fusion, mixed unions over cold traffic with binary parity) was committed but never ran in CI. The benchmark job now runs it alongside the possibilistic baseline compare, uploads the results artifact, and surfaces both in the run summary. Also: refreshed the committed .rigor-baseline.json (engine unchanged since the tag; the runner's canonical baseline lives in its artifact store), and added npm run benchmark:complex-query for local runs. --- .gitea/workflows/ci.yaml | 14 ++++- .rigor-baseline.json | 114 +++++++++++++++++++-------------------- package.json | 3 +- 3 files changed, 72 insertions(+), 59 deletions(-) diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index fd2aeee..5b17054 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -70,6 +70,10 @@ jobs: run: node --expose-gc scripts/benchmark.js --json > bench-results.json continue-on-error: true + - name: Run complex-query benchmarks (cold traffic, normal vs binary) + run: node --expose-gc benchmarks/complex-query-bench.js --size=25000 --samples=10000 > complex-query-results.json + continue-on-error: true + - name: Save baseline (on tag) if: startsWith(github.ref, 'refs/tags/v') run: node --expose-gc scripts/benchmark.js --save @@ -87,7 +91,9 @@ jobs: uses: actions/upload-artifact@v3 with: name: bench-results - path: bench-results.json + path: | + bench-results.json + complex-query-results.json retention-days: 30 - name: Benchmark summary @@ -99,6 +105,12 @@ jobs: cat bench-results.json >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY fi + if [ -f complex-query-results.json ]; then + echo '### Complex-Query Results (cold traffic, normal vs binary)' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + cat complex-query-results.json >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + fi publish: runs-on: ubuntu-latest diff --git a/.rigor-baseline.json b/.rigor-baseline.json index 46f4194..2623e31 100644 --- a/.rigor-baseline.json +++ b/.rigor-baseline.json @@ -1,140 +1,140 @@ { "version": 1, - "generated": "2026-08-02T19:51:19.208Z", + "generated": "2026-08-03T03:25:46.644Z", "actions": { "check[direct-hit]": { - "mostPlausible": 0.25810395981996936, + "mostPlausible": 0.2641342114909732, "alphaCuts": { "p50": { - "lower": 0.16434574601481775, - "upper": 0.40535024830520205 + "lower": 0.11993100000017193, + "upper": 0.6825795028928235 }, "p95": { - "lower": 0.10394999999999016, - "upper": 1.2158777106838974 + "lower": 0.11993100000017193, + "upper": 0.6825795028928235 }, "p99": { - "lower": 0.10394999999999016, - "upper": 2.040950000000002 + "lower": 0.11993100000017193, + "upper": 0.7088692690070402 } } }, "check[union-ttu]": { - "mostPlausible": 0.3581109933665416, + "mostPlausible": 0.5572143564747223, "alphaCuts": { "p50": { - "lower": 0.07645000000001517, - "upper": 0.35811139703768785 + "lower": 0.10743099999989908, + "upper": 0.5572148030094048 }, "p95": { - "lower": 0.07645000000001517, - "upper": 0.35811139703768785 + "lower": 0.10743099999989908, + "upper": 0.5572148030094048 }, "p99": { - "lower": 0.07645000000001517, - "upper": 0.35811139703768785 + "lower": 0.10743099999989908, + "upper": 0.5572148030094048 } } }, "check[denied-miss]": { - "mostPlausible": 0.21718442918988012, + "mostPlausible": 0.17001072644302176, "alphaCuts": { "p50": { - "lower": 0.18384889559816292, - "upper": 0.2565642603295502 + "lower": 0.10983099999997875, + "upper": 0.27403866354448264 }, "p95": { - "lower": 0.13274999999997988, - "upper": 0.3601500000000823 + "lower": 0.10983099999997875, + "upper": 0.27403866354448264 }, "p99": { - "lower": 0.13274999999997988, - "upper": 0.3601500000000823 + "lower": 0.10983099999997875, + "upper": 0.2792968083762517 } } }, "check[include-meta]": { - "mostPlausible": 0.9937660048895368, + "mostPlausible": 0.4313421276490078, "alphaCuts": { "p50": { - "lower": 0.9937660048895368, - "upper": 0.9937663566453343 + "lower": 0.2319310000000246, + "upper": 0.9209354557840623 }, "p95": { - "lower": 0.9937660048895368, - "upper": 0.9937663566453343 + "lower": 0.2319310000000246, + "upper": 0.9209354557840623 }, "p99": { - "lower": 0.9937660048895368, - "upper": 0.9937663566453343 + "lower": 0.2319310000000246, + "upper": 0.9491636618587536 } } }, "check[overlay-on-top]": { - "mostPlausible": 0.3485105914857449, + "mostPlausible": 2.4209884301909703, "alphaCuts": { "p50": { - "lower": 0.26044999999998425, - "upper": 0.3881187132654935 + "lower": 2.4209884301909703, + "upper": 2.4209887680862785 }, "p95": { - "lower": 0.26044999999998425, - "upper": 0.5286391152207875 + "lower": 2.4209884301909703, + "upper": 2.4209887680862785 }, "p99": { - "lower": 0.26044999999998425, - "upper": 0.5374790233000701 + "lower": 2.4209884301909703, + "upper": 2.4209887680862785 } } }, "check[binary-direct]": { - "mostPlausible": 0.2660225524868295, + "mostPlausible": 0.24005599110530157, "alphaCuts": { "p50": { - "lower": 0.21220996174732032, - "upper": 0.3334807973509898 + "lower": 0.154731000000304, + "upper": 0.2756826494555559 }, "p95": { - "lower": 0.16554999999993186, - "upper": 0.5780022226626057 + "lower": 0.154731000000304, + "upper": 0.41012851350897955 }, "p99": { - "lower": 0.16554999999993186, - "upper": 0.8219500000000078 + "lower": 0.154731000000304, + "upper": 0.41896614930102694 } } }, "snapshot[build-binary]": { - "mostPlausible": 4.119453383803492, + "mostPlausible": 7.446251599999954, "alphaCuts": { "p50": { - "lower": 3.638457178151043, - "upper": 4.664037328289517 + "lower": 5.3102783288306075, + "upper": 9.582224244110606 }, "p95": { - "lower": 3.1707499999999906, - "upper": 6.309350037482752 + "lower": 5.3102783288306075, + "upper": 9.582224244110606 }, "p99": { - "lower": 3.1707499999999906, - "upper": 7.547749999999944 + "lower": 5.225227783447299, + "upper": 9.667235344457687 } } }, "snapshot[restore-binary]": { - "mostPlausible": 6.906283652168588, + "mostPlausible": 9.853481541731316, "alphaCuts": { "p50": { - "lower": 3.2581500000001067, - "upper": 6.906283919208314 + "lower": 7.845275634169648, + "upper": 12.375747204692138 }, "p95": { - "lower": 3.2581500000001067, - "upper": 6.906283919208314 + "lower": 4.7317310000000745, + "upper": 21.54997911145655 }, "p99": { - "lower": 3.2581500000001067, - "upper": 6.906283919208314 + "lower": 4.7317310000000745, + "upper": 25.40483299999982 } } } diff --git a/package.json b/package.json index 1ce1130..04ede95 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,8 @@ "benchmark:memory": "node benchmarks/memory-breakdown.js", "benchmark:multi-hop": "node benchmarks/multi-hop-rule-bench.js", "benchmark:save": "node --expose-gc scripts/benchmark.js --save", - "benchmark:complex": "node --expose-gc benchmarks/complex-query-bench.js" + "benchmark:complex": "node --expose-gc benchmarks/complex-query-bench.js", + "benchmark:complex-query": "node --expose-gc benchmarks/complex-query-bench.js" }, "keywords": [ "zanzibar",