Arbiter/core
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ci: run complex-query cold-traffic benchmarks in the benchmark job
CI / test (push) Successful in 5m42s
Details
CI / benchmark (push) Successful in 42s
Details
CI / publish (push) Has been skipped
Details

Browse Source 
The prod-gating measurement (complex-query-bench: tuple-to-userset,
chains, defeasible, comparators, OWA, nested fusion, mixed unions over
cold traffic with binary parity) was committed but never ran in CI. The
benchmark job now runs it alongside the possibilistic baseline compare,
uploads the results artifact, and surfaces both in the run summary.

Also: refreshed the committed .rigor-baseline.json (engine unchanged
since the tag; the runner's canonical baseline lives in its artifact
store), and added npm run benchmark:complex-query for local runs.
This commit is contained in:
John Dvorak
2026-08-02 20:26:36 -07:00
parent 3444a8b6aa
commit 817601a0f3
3 changed files with 72 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitea/workflows/ci.yaml
+13 -1
View File
@@ -70,6 +70,10 @@ jobs:
run: node --expose-gc scripts/benchmark.js --json > bench-results.json run: node --expose-gc scripts/benchmark.js --json > bench-results.json
continue-on-error: true continue-on-error: true
- name: Run complex-query benchmarks (cold traffic, normal vs binary)
run: node --expose-gc benchmarks/complex-query-bench.js --size=25000 --samples=10000 > complex-query-results.json
continue-on-error: true
- name: Save baseline (on tag) - name: Save baseline (on tag)
if: startsWith(github.ref, 'refs/tags/v') if: startsWith(github.ref, 'refs/tags/v')
run: node --expose-gc scripts/benchmark.js --save run: node --expose-gc scripts/benchmark.js --save
@@ -87,7 +91,9 @@ jobs:
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
name: bench-results name: bench-results
path: bench-results.json path: |
bench-results.json
complex-query-results.json
retention-days: 30 retention-days: 30
- name: Benchmark summary - name: Benchmark summary
@@ -99,6 +105,12 @@ jobs:
cat bench-results.json >> $GITHUB_STEP_SUMMARY cat bench-results.json >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY
fi fi
if [ -f complex-query-results.json ]; then
echo '### Complex-Query Results (cold traffic, normal vs binary)' >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
cat complex-query-results.json >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
fi
publish: publish:
runs-on: ubuntu-latest runs-on: ubuntu-latest
.rigor-baseline.json
+57 -57
View File
@@ -1,140 +1,140 @@
{ {
"version": 1, "version": 1,
"generated": "2026-08-02T19:51:19.208Z", "generated": "2026-08-03T03:25:46.644Z",
"actions": { "actions": {
"check[direct-hit]": { "check[direct-hit]": {
"mostPlausible": 0.25810395981996936, "mostPlausible": 0.2641342114909732,
"alphaCuts": { "alphaCuts": {
"p50": { "p50": {
"lower": 0.16434574601481775, "lower": 0.11993100000017193,
"upper": 0.40535024830520205 "upper": 0.6825795028928235
}, },
"p95": { "p95": {
"lower": 0.10394999999999016, "lower": 0.11993100000017193,
"upper": 1.2158777106838974 "upper": 0.6825795028928235
}, },
"p99": { "p99": {
"lower": 0.10394999999999016, "lower": 0.11993100000017193,
"upper": 2.040950000000002 "upper": 0.7088692690070402
} }
} }
}, },
"check[union-ttu]": { "check[union-ttu]": {
"mostPlausible": 0.3581109933665416, "mostPlausible": 0.5572143564747223,
"alphaCuts": { "alphaCuts": {
"p50": { "p50": {
"lower": 0.07645000000001517, "lower": 0.10743099999989908,
"upper": 0.35811139703768785 "upper": 0.5572148030094048
}, },
"p95": { "p95": {
"lower": 0.07645000000001517, "lower": 0.10743099999989908,
"upper": 0.35811139703768785 "upper": 0.5572148030094048
}, },
"p99": { "p99": {
"lower": 0.07645000000001517, "lower": 0.10743099999989908,
"upper": 0.35811139703768785 "upper": 0.5572148030094048
} }
} }
}, },
"check[denied-miss]": { "check[denied-miss]": {
"mostPlausible": 0.21718442918988012, "mostPlausible": 0.17001072644302176,
"alphaCuts": { "alphaCuts": {
"p50": { "p50": {
"lower": 0.18384889559816292, "lower": 0.10983099999997875,
"upper": 0.2565642603295502 "upper": 0.27403866354448264
}, },
"p95": { "p95": {
"lower": 0.13274999999997988, "lower": 0.10983099999997875,
"upper": 0.3601500000000823 "upper": 0.27403866354448264
}, },
"p99": { "p99": {
"lower": 0.13274999999997988, "lower": 0.10983099999997875,
"upper": 0.3601500000000823 "upper": 0.2792968083762517
} }
} }
}, },
"check[include-meta]": { "check[include-meta]": {
"mostPlausible": 0.9937660048895368, "mostPlausible": 0.4313421276490078,
"alphaCuts": { "alphaCuts": {
"p50": { "p50": {
"lower": 0.9937660048895368, "lower": 0.2319310000000246,
"upper": 0.9937663566453343 "upper": 0.9209354557840623
}, },
"p95": { "p95": {
"lower": 0.9937660048895368, "lower": 0.2319310000000246,
"upper": 0.9937663566453343 "upper": 0.9209354557840623
}, },
"p99": { "p99": {
"lower": 0.9937660048895368, "lower": 0.2319310000000246,
"upper": 0.9937663566453343 "upper": 0.9491636618587536
} }
} }
}, },
"check[overlay-on-top]": { "check[overlay-on-top]": {
"mostPlausible": 0.3485105914857449, "mostPlausible": 2.4209884301909703,
"alphaCuts": { "alphaCuts": {
"p50": { "p50": {
"lower": 0.26044999999998425, "lower": 2.4209884301909703,
"upper": 0.3881187132654935 "upper": 2.4209887680862785
}, },
"p95": { "p95": {
"lower": 0.26044999999998425, "lower": 2.4209884301909703,
"upper": 0.5286391152207875 "upper": 2.4209887680862785
}, },
"p99": { "p99": {
"lower": 0.26044999999998425, "lower": 2.4209884301909703,
"upper": 0.5374790233000701 "upper": 2.4209887680862785
} }
} }
}, },
"check[binary-direct]": { "check[binary-direct]": {
"mostPlausible": 0.2660225524868295, "mostPlausible": 0.24005599110530157,
"alphaCuts": { "alphaCuts": {
"p50": { "p50": {
"lower": 0.21220996174732032, "lower": 0.154731000000304,
"upper": 0.3334807973509898 "upper": 0.2756826494555559
}, },
"p95": { "p95": {
"lower": 0.16554999999993186, "lower": 0.154731000000304,
"upper": 0.5780022226626057 "upper": 0.41012851350897955
}, },
"p99": { "p99": {
"lower": 0.16554999999993186, "lower": 0.154731000000304,
"upper": 0.8219500000000078 "upper": 0.41896614930102694
} }
} }
}, },
"snapshot[build-binary]": { "snapshot[build-binary]": {
"mostPlausible": 4.119453383803492, "mostPlausible": 7.446251599999954,
"alphaCuts": { "alphaCuts": {
"p50": { "p50": {
"lower": 3.638457178151043, "lower": 5.3102783288306075,
"upper": 4.664037328289517 "upper": 9.582224244110606
}, },
"p95": { "p95": {
"lower": 3.1707499999999906, "lower": 5.3102783288306075,
"upper": 6.309350037482752 "upper": 9.582224244110606
}, },
"p99": { "p99": {
"lower": 3.1707499999999906, "lower": 5.225227783447299,
"upper": 7.547749999999944 "upper": 9.667235344457687
} }
} }
}, },
"snapshot[restore-binary]": { "snapshot[restore-binary]": {
"mostPlausible": 6.906283652168588, "mostPlausible": 9.853481541731316,
"alphaCuts": { "alphaCuts": {
"p50": { "p50": {
"lower": 3.2581500000001067, "lower": 7.845275634169648,
"upper": 6.906283919208314 "upper": 12.375747204692138
}, },
"p95": { "p95": {
"lower": 3.2581500000001067, "lower": 4.7317310000000745,
"upper": 6.906283919208314 "upper": 21.54997911145655
}, },
"p99": { "p99": {
"lower": 3.2581500000001067, "lower": 4.7317310000000745,
"upper": 6.906283919208314 "upper": 25.40483299999982
} }
} }
} }
package.json
+2 -1
View File
@@ -49,7 +49,8 @@
"benchmark:memory": "node benchmarks/memory-breakdown.js", "benchmark:memory": "node benchmarks/memory-breakdown.js",
"benchmark:multi-hop": "node benchmarks/multi-hop-rule-bench.js", "benchmark:multi-hop": "node benchmarks/multi-hop-rule-bench.js",
"benchmark:save": "node --expose-gc scripts/benchmark.js --save", "benchmark:save": "node --expose-gc scripts/benchmark.js --save",
"benchmark:complex": "node --expose-gc benchmarks/complex-query-bench.js" "benchmark:complex": "node --expose-gc benchmarks/complex-query-bench.js",
"benchmark:complex-query": "node --expose-gc benchmarks/complex-query-bench.js"
}, },
"keywords": [ "keywords": [
"zanzibar", "zanzibar",