feat: tuple_to_userset intermediate reachability; ratio-based benchmark gate
- ChainRule._expandRuleFromSrc now expands tuple_to_userset configs: from a source node the reachable set is the objects sharing an intermediate with the source (src ->computed-> intermediate ->tupleset-> object, direction aware, weakest-link combined). Lets a TTU evidence serve as an intermediate condition step in a chain. - scripts/benchmark.js: ratio-based self-calibration. Comparing each action's RATIO to a cheap reference action (default check[direct-hit]) cancels machine-load swings that scale all actions proportionally, so the gate only fails on code regressions that shift a single action's ratio. The reference is still checked absolutely with a loose bound. Verified: stable across runs, and a simulated union-ttu slowdown is caught (+76.8% ratio). Tests: chain-condition-step intermediate TTU expansion.
This commit is contained in:
+44
-11
@@ -207,17 +207,50 @@ if (SAVE) {
|
||||
const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8'))
|
||||
const regResult = detectRegressions(current, baseline)
|
||||
if (!AS_JSON) console.log(formatRegressions(regResult, 'pretty'))
|
||||
// The lib flags a regression whenever current > baseline.p95/p99, which is
|
||||
// tighter than run-to-run noise at sub-ms scales (baseline alphaCuts capture
|
||||
// per-run sample spread, not cross-run variance). Require a minimum change
|
||||
// before treating a high-severity flag as a CI failure, so the gate fails on
|
||||
// real regressions (>10% past the p99) instead of ~3-5% noise.
|
||||
const MIN_HIGH_REGRESSION_PERCENT = Number(process.env.MIN_HIGH_REGRESSION_PERCENT ?? 10)
|
||||
const critical = regResult.regressions.filter(
|
||||
r => r.severity === 'high' && r.changePercent >= MIN_HIGH_REGRESSION_PERCENT
|
||||
)
|
||||
if (critical.length > 0) {
|
||||
console.error(`❌ ${critical.length} critical regression(s): ${critical.map(r => r.name).join(', ')}`)
|
||||
|
||||
// ── Ratio-based self-calibration ─────────────────────────────────────
|
||||
// Absolute timings swing with machine load (a shared runner at load 19
|
||||
// shifted every action +15..+50%). Comparing each action's RATIO to a cheap
|
||||
// reference action cancels the load: load scales all actions proportionally,
|
||||
// while a code regression shifts only the affected action's ratio. The
|
||||
// reference action itself is still gate-checked absolutely with a loose
|
||||
// bound (a regression of the reference would otherwise mask every ratio).
|
||||
const RATIO_REFERENCE = process.env.BENCH_RATIO_REFERENCE || 'check[direct-hit]'
|
||||
const RATIO_PERCENT = Number(process.env.RATIO_REGRESSION_PERCENT ?? 15)
|
||||
const REFERENCE_PERCENT = Number(process.env.REFERENCE_REGRESSION_PERCENT ?? 30)
|
||||
|
||||
const currentRef = current.actions?.[RATIO_REFERENCE]?.mostPlausible
|
||||
const baselineRef = baseline.actions?.[RATIO_REFERENCE]?.mostPlausible
|
||||
const ratioViolations = []
|
||||
let referenceViolation = false
|
||||
|
||||
if (currentRef > 0 && baselineRef > 0) {
|
||||
if (currentRef > baselineRef * (1 + REFERENCE_PERCENT / 100)) {
|
||||
referenceViolation = true
|
||||
ratioViolations.push(
|
||||
`${RATIO_REFERENCE} (reference) baseline=${baselineRef.toFixed(4)} → current=${currentRef.toFixed(4)} (+${(((currentRef / baselineRef) - 1) * 100).toFixed(1)}%)`
|
||||
)
|
||||
}
|
||||
for (const [name, cur] of Object.entries(current.actions)) {
|
||||
if (name === RATIO_REFERENCE) continue
|
||||
const base = baseline.actions?.[name]
|
||||
if (!base || !(base.mostPlausible > 0)) continue
|
||||
const curRatio = cur.mostPlausible / currentRef
|
||||
const baseRatio = base.mostPlausible / baselineRef
|
||||
if (curRatio > baseRatio * (1 + RATIO_PERCENT / 100)) {
|
||||
ratioViolations.push(
|
||||
`${name} ratio ${curRatio.toFixed(3)} → baseline ${baseRatio.toFixed(3)} (+${(((curRatio / baseRatio) - 1) * 100).toFixed(1)}%)`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (ratioViolations.length > 0) {
|
||||
console.error(`❌ ${ratioViolations.length} ratio regression(s):`)
|
||||
for (const v of ratioViolations) console.error(` - ${v}`)
|
||||
if (referenceViolation) {
|
||||
console.error(' (the reference action regressed absolutely — ratios may be unreliable)')
|
||||
}
|
||||
process.exit(1)
|
||||
}
|
||||
} else if (!SAVE) {
|
||||
|
||||
Reference in New Issue
Block a user