feat: fix lowering, DSLRuntime wrapper, rigor oracle + rejection campaigns
Lowering fixes (validate/lower/compile into known-correct core structures):
- tuple_to_userset: structural classification by object-side predicate
(owner(*g, doc) { member_of(user, g) } -> tuple_to_userset with direction
'in'/'out'); the old heuristic routed every outer-wildcard to chain.
- relational_comparator: operands now lower to real direct-rule configs
(evaluateFrom derived from evidence param positions; expectedValue for
literal args) instead of raw AST nodes the engine could not evaluate.
- defeasible: multi-level bodies (NEVER/REQUIRES/ALWAYS/WHEN/UNLESS) merge
into one five-level rule instead of ANDed level-only rules that always
resolved 0; nested PatternMatches flatten to N-step chains; unary predicate
calls mark _subjectAsObject (subject-as-object semantics).
- validation: reject duplicate fact/evidence definitions.
DSLRuntime (higher-order DSL+Core wrapper):
- typed addNode/updateNodeData/addRelation/updateRelation against the DSL
schema (known types, relation params, field types, value-carrying facts);
- check() derives the evidence's injectable partial-graph requirements,
retrieves missing facts through caller data callbacks, injects them, and
delegates, returning requiredFacts/providedFacts/missingFacts.
js-rigor campaigns:
- generative oracle: generate legal DSL per construct and compare every
verdict against an independent hand-computed oracle (8 constructs x P grid)
plus an exhaustive deterministic sweep;
- illegal mutations: one-flaw perturbations of a valid program must be
reliably rejected (duplicate evidence/fact, arity/type mismatches, reserved
built-ins, malformed syntax), with a control that must compile.
Depends on @arbiter/core@^1.0.2 (reason codes + _subjectAsObject).
This commit is contained in:
John Dvorak
@@ -0,0 +1,190 @@
|
||||
/**
|
||||
* tests/rigor/dsl-generative-oracle.test.js — js-rigor campaign that GENERATES
|
||||
* legal Evidence DSL programs, compiles them to @arbiter/core configs, runs
|
||||
* checks, and compares every verdict against an independent ORACLE (a hand-
|
||||
* computed reference implementation of the DSL semantics).
|
||||
*
|
||||
* The oracle is deliberately independent of the engine: it computes the
|
||||
* expected possibility from the generated fact graph using the ADR-000
|
||||
* semantics (direct = edge, chain = min over steps, tuple_to_userset = min of
|
||||
* the two legs, fusion = min/max over operands, when-unless = base×(1−defeat),
|
||||
* never = 0 when ≥0.5 else base, requires = base×requirement).
|
||||
*
|
||||
* Anti-vacuity: the oracle is NOT a constant — each construct maps distinct
|
||||
* edge possibilities, so a trivial 0-or-1 lowering would be caught.
|
||||
*/
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { rigor } from '@rigor/core';
|
||||
import { Arbiter } from '@arbiter/core';
|
||||
import { DSLCompiler } from '../../src/DSLCompiler.js';
|
||||
|
||||
const EPS = 1e-9;
|
||||
const P = [0, 0.25, 0.5, 0.75, 1];
|
||||
|
||||
const FACTS = `
|
||||
definition Employee { id: string }
|
||||
definition Group { id: string }
|
||||
definition Doc { id: string }
|
||||
fact owns(user: Employee, doc: Doc)
|
||||
fact shares(user: Employee, doc: Doc)
|
||||
fact member_of(user: Employee, group: Group)
|
||||
fact can_access(group: Group, doc: Doc)
|
||||
fact owner(group: Group, doc: Doc)
|
||||
fact granted(user: Employee, doc: Doc)
|
||||
fact banned(user: Employee)
|
||||
fact mfa(user: Employee)
|
||||
`;
|
||||
|
||||
// Each construct: how to build the DSL evidence + which edges to add + the oracle.
|
||||
function buildProgram(kind, ps) {
|
||||
let evidence = '';
|
||||
const edges = [];
|
||||
let oracle = 0;
|
||||
|
||||
switch (kind) {
|
||||
case 'direct': {
|
||||
const [pOwn] = ps;
|
||||
evidence = `evidence can_read(user: Employee, doc: Doc) { owns(user, doc) }`;
|
||||
edges.push({ src: 'u:1', relation: 'owns', dst: 'doc:9', possibility: pOwn });
|
||||
oracle = pOwn;
|
||||
break;
|
||||
}
|
||||
case 'chain': {
|
||||
const [pm, pa] = ps;
|
||||
evidence = `evidence can_enter(user: Employee, doc: Doc) { member_of(user, *g) { can_access(g, doc) } }`;
|
||||
edges.push({ src: 'u:1', relation: 'member_of', dst: 'g:1', possibility: pm });
|
||||
edges.push({ src: 'g:1', relation: 'can_access', dst: 'doc:9', possibility: pa });
|
||||
oracle = Math.min(pm, pa);
|
||||
break;
|
||||
}
|
||||
case 'tuple_to_userset': {
|
||||
const [po, pm] = ps;
|
||||
evidence = `evidence can_view(user: Employee, doc: Doc) { owner(*g, doc) { member_of(user, g) } }`;
|
||||
edges.push({ src: 'g:1', relation: 'owner', dst: 'doc:9', possibility: po });
|
||||
edges.push({ src: 'u:1', relation: 'member_of', dst: 'g:1', possibility: pm });
|
||||
oracle = Math.min(pm, po);
|
||||
break;
|
||||
}
|
||||
case 'fusion_min': {
|
||||
const [p1, p2] = ps;
|
||||
evidence = `evidence can_fuse(user: Employee, doc: Doc) { fusion min { owns(user, doc), shares(user, doc) } }`;
|
||||
edges.push({ src: 'u:1', relation: 'owns', dst: 'doc:9', possibility: p1 });
|
||||
edges.push({ src: 'u:1', relation: 'shares', dst: 'doc:9', possibility: p2 });
|
||||
oracle = Math.min(p1, p2);
|
||||
break;
|
||||
}
|
||||
case 'fusion_max': {
|
||||
const [p1, p2] = ps;
|
||||
evidence = `evidence can_fuse(user: Employee, doc: Doc) { fusion max { owns(user, doc), shares(user, doc) } }`;
|
||||
edges.push({ src: 'u:1', relation: 'owns', dst: 'doc:9', possibility: p1 });
|
||||
edges.push({ src: 'u:1', relation: 'shares', dst: 'doc:9', possibility: p2 });
|
||||
oracle = Math.max(p1, p2);
|
||||
break;
|
||||
}
|
||||
case 'when_unless': {
|
||||
const [pG, pB] = ps;
|
||||
evidence = `evidence can_borrow(user: Employee, doc: Doc) { WHEN granted(user, doc) UNLESS banned(user) }`;
|
||||
edges.push({ src: 'u:1', relation: 'granted', dst: 'doc:9', possibility: pG });
|
||||
edges.push({ src: 'u:1', relation: 'banned', dst: 'u:1', possibility: pB });
|
||||
oracle = pG * (1 - pB);
|
||||
break;
|
||||
}
|
||||
case 'never_always': {
|
||||
const [pG, pB] = ps;
|
||||
evidence = `evidence can_open(user: Employee, doc: Doc) { NEVER banned(user) ALWAYS granted(user, doc) }`;
|
||||
edges.push({ src: 'u:1', relation: 'granted', dst: 'doc:9', possibility: pG });
|
||||
edges.push({ src: 'u:1', relation: 'banned', dst: 'u:1', possibility: pB });
|
||||
oracle = pB >= 0.5 ? 0 : pG;
|
||||
break;
|
||||
}
|
||||
case 'requires_when': {
|
||||
const [pG, pM] = ps;
|
||||
evidence = `evidence can_pay(user: Employee, doc: Doc) { REQUIRES mfa(user) WHEN granted(user, doc) }`;
|
||||
edges.push({ src: 'u:1', relation: 'granted', dst: 'doc:9', possibility: pG });
|
||||
edges.push({ src: 'u:1', relation: 'mfa', dst: 'u:1', possibility: pM });
|
||||
oracle = pG * pM;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new Error(`unknown construct: ${kind}`);
|
||||
}
|
||||
|
||||
return { dsl: FACTS + evidence, edges, oracle, relation: evidence.match(/evidence (\w+)/)[1] };
|
||||
}
|
||||
|
||||
function runCheck({ kind, ps }) {
|
||||
const { dsl, edges, oracle, relation } = buildProgram(kind, ps);
|
||||
const arbiter = new Arbiter();
|
||||
arbiter.addNode('u:1', 'Employee');
|
||||
arbiter.addNode('g:1', 'Group');
|
||||
arbiter.addNode('doc:9', 'Doc');
|
||||
const compiler = new DSLCompiler(arbiter);
|
||||
const compiled = compiler.compile(dsl, 'oracle');
|
||||
if (!compiled.success) {
|
||||
throw new Error(`compile failed for ${kind}: ${compiled.errors.join('; ')}`);
|
||||
}
|
||||
for (const e of edges) arbiter.addRelation(e.src, e.relation, e.dst, { possibility: e.possibility });
|
||||
const result = arbiter.check('u:1', relation, 'doc:9');
|
||||
if (Math.abs(result.possibility - oracle) > EPS) {
|
||||
throw new Error(`oracle mismatch for ${kind} (edges=${JSON.stringify(edges)}): ` +
|
||||
`check=${result.possibility} (${result.reason}) vs oracle=${oracle}`);
|
||||
}
|
||||
return { kind, possibility: result.possibility, oracle };
|
||||
}
|
||||
|
||||
const CONSTRUCTS = ['direct', 'chain', 'tuple_to_userset', 'fusion_min', 'fusion_max',
|
||||
'when_unless', 'never_always', 'requires_when'];
|
||||
|
||||
describe('DSL generative oracle parity (rigor)', () => {
|
||||
it('generated legal DSL compiles and every check matches the oracle', async () => {
|
||||
const report = await rigor.campaign(
|
||||
[
|
||||
rigor.fn('oracle-parity', runCheck, rigor.args(
|
||||
rigor.gen.object({
|
||||
kind: rigor.gen.oneOf(CONSTRUCTS),
|
||||
// exactly two edge possibilities (direct uses only the first);
|
||||
// a shorter array would leave pB undefined and produce a NaN oracle
|
||||
ps: rigor.gen.tuple(rigor.gen.oneOf(P), rigor.gen.oneOf(P))
|
||||
})
|
||||
))
|
||||
],
|
||||
rigor.crucible([
|
||||
// `actual` is the fn's return value; a thrown error (compile failure or
|
||||
// oracle mismatch) yields actual === undefined, failing this invariant.
|
||||
rigor.invariant('oracle-parity', ({ actual }) =>
|
||||
!!actual && Math.abs(actual.possibility - actual.oracle) <= EPS)
|
||||
])
|
||||
).run({ seed: 'dsl-oracle-parity', effort: 600, artifacts: { dir: '', persist: 'never' } });
|
||||
|
||||
const inv = report.crucibleVerdict?.invariants?.find(i => i.name === 'oracle-parity');
|
||||
assert.ok(inv, 'crucible invariant missing');
|
||||
assert.equal(inv.passed, true, `oracle parity violated in ${inv.failureCount} cases`);
|
||||
});
|
||||
|
||||
it('exhaustive deterministic sweep: every construct x every possibility value', () => {
|
||||
// Anti-vacuity complement to the campaign: sweep the full P × P grid per
|
||||
// construct without any RNG, so a construct the campaign skipped would
|
||||
// still be caught here.
|
||||
for (const kind of CONSTRUCTS) {
|
||||
for (const a of P) {
|
||||
for (const b of P) {
|
||||
const ps = kind === 'direct' ? [a] : [a, b];
|
||||
const { dsl, edges, oracle, relation } = buildProgram(kind, ps);
|
||||
const arbiter = new Arbiter();
|
||||
arbiter.addNode('u:1', 'Employee');
|
||||
arbiter.addNode('g:1', 'Group');
|
||||
arbiter.addNode('doc:9', 'Doc');
|
||||
const compiled = new DSLCompiler(arbiter).compile(dsl, 'sweep');
|
||||
assert.ok(compiled.success, `${kind} compile failed: ${(compiled.errors || []).join('; ')}`);
|
||||
for (const e of edges) arbiter.addRelation(e.src, e.relation, e.dst, { possibility: e.possibility });
|
||||
const result = arbiter.check('u:1', relation, 'doc:9');
|
||||
assert.ok(
|
||||
Math.abs(result.possibility - oracle) <= EPS,
|
||||
`${kind} ps=[${ps}] check=${result.possibility}(${result.reason}) vs oracle=${oracle}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,138 @@
|
||||
/**
|
||||
* tests/rigor/dsl-illegal-mutations.test.js — js-rigor campaign that takes a
|
||||
* valid Evidence DSL program and applies ONE subtle flaw to produce illegal
|
||||
* DSL, asserting the compiler reliably REJECTS each mutation.
|
||||
*
|
||||
* Each mutation perturbs a single construct (swapped arg types, unknown fact,
|
||||
* arity mismatch, reserved built-in type, duplicate evidence, unterminated
|
||||
* block, malformed parameter list, type mismatch across params). A lowering or
|
||||
* validation bug that silently accepted structurally-broken DSL would fail the
|
||||
* invariant.
|
||||
*
|
||||
* Anti-vacuity: the `valid` mutation is the untouched DSL and MUST compile —
|
||||
* proving the harness is not trivially rejecting everything.
|
||||
*/
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { rigor } from '@rigor/core';
|
||||
import { Arbiter } from '@arbiter/core';
|
||||
import { DSLCompiler } from '../../src/DSLCompiler.js';
|
||||
|
||||
const VALID_DSL = `
|
||||
definition Employee { id: string }
|
||||
definition Group { id: string }
|
||||
definition Doc { id: string }
|
||||
fact owns(user: Employee, doc: Doc)
|
||||
fact member_of(user: Employee, group: Group)
|
||||
fact can_access(group: Group, doc: Doc)
|
||||
evidence can_read(user: Employee, doc: Doc) { owns(user, doc) }
|
||||
evidence can_enter(user: Employee, doc: Doc) { member_of(user, *g) { can_access(g, doc) } }
|
||||
`;
|
||||
|
||||
// Each mutation transforms the valid DSL into an illegal variant.
|
||||
// `mustFail: false` marks the control mutation (untouched DSL — must compile).
|
||||
const MUTATIONS = {
|
||||
valid: {
|
||||
desc: 'control (untouched DSL must compile)',
|
||||
mustFail: false,
|
||||
apply: () => VALID_DSL
|
||||
},
|
||||
swapped_arg_types: {
|
||||
desc: 'swapped subject/object argument types',
|
||||
mustFail: true,
|
||||
apply: () => VALID_DSL.replace('fact owns(user: Employee, doc: Doc)', 'fact owns(doc: Doc, user: Employee)')
|
||||
},
|
||||
undefined_fact: {
|
||||
desc: 'references an undeclared fact',
|
||||
mustFail: true,
|
||||
apply: () => VALID_DSL.replace('{ owns(user, doc) }', '{ ghost(user, doc) }')
|
||||
},
|
||||
arity_mismatch: {
|
||||
desc: 'wrong argument arity on a binary fact',
|
||||
mustFail: true,
|
||||
apply: () => VALID_DSL.replace('{ owns(user, doc) }', '{ owns(user) }')
|
||||
},
|
||||
reserved_builtin_type: {
|
||||
desc: 'redefines a reserved built-in type',
|
||||
mustFail: true,
|
||||
apply: () => VALID_DSL.replace('definition Employee { id: string }', 'definition User { id: string }')
|
||||
},
|
||||
duplicate_evidence: {
|
||||
desc: 'duplicate evidence relation name',
|
||||
mustFail: true,
|
||||
apply: () => VALID_DSL + `\n evidence can_read(user: Employee, doc: Doc) { owns(user, doc) }`
|
||||
},
|
||||
unterminated_block: {
|
||||
desc: 'missing closing brace',
|
||||
mustFail: true,
|
||||
apply: () => VALID_DSL.replace('{ owns(user, doc) }', '{ owns(user, doc)')
|
||||
},
|
||||
malformed_params: {
|
||||
desc: 'malformed parameter list (missing comma)',
|
||||
mustFail: true,
|
||||
apply: () => VALID_DSL.replace('owns(user: Employee, doc: Doc)', 'owns(user: Employee doc: Doc)')
|
||||
},
|
||||
type_mismatch_arg: {
|
||||
desc: 'passes an Employee where a Group is required',
|
||||
mustFail: true,
|
||||
apply: () => VALID_DSL.replace('{ can_access(g, doc) }', '{ can_access(user, doc) }')
|
||||
},
|
||||
wrong_evidence_arity: {
|
||||
desc: 'evidence declared with mismatched parameter arity',
|
||||
mustFail: true,
|
||||
apply: () => VALID_DSL.replace('evidence can_read(user: Employee, doc: Doc) { owns(user, doc) }', 'evidence can_read(user: Employee) { owns(user, doc) }')
|
||||
}
|
||||
};
|
||||
|
||||
function checkMutation(mutationName) {
|
||||
const mutation = MUTATIONS[mutationName];
|
||||
if (!mutation) throw new Error(`unknown mutation name: ${JSON.stringify(mutationName)}`);
|
||||
const dsl = mutation.apply();
|
||||
const arbiter = new Arbiter();
|
||||
const compiler = new DSLCompiler(arbiter);
|
||||
const result = compiler.compile(dsl, `mut-${mutationName}`);
|
||||
const success = result.success;
|
||||
const errors = result.errors || [];
|
||||
if (mutation.mustFail) {
|
||||
if (success || errors.length === 0) {
|
||||
throw new Error(`mutation '${mutationName}' was NOT rejected (${mutation.desc}). ` +
|
||||
`success=${success}, errors=${JSON.stringify(errors)}`);
|
||||
}
|
||||
} else if (!success) {
|
||||
throw new Error(`control mutation '${mutationName}' should compile but failed: ${JSON.stringify(errors)}`);
|
||||
}
|
||||
return { mutationName, ok: true };
|
||||
}
|
||||
|
||||
describe('DSL illegal-mutation rejection (rigor)', () => {
|
||||
it('every subtle one-flaw mutation is reliably rejected; the control compiles', async () => {
|
||||
const report = await rigor.campaign(
|
||||
[
|
||||
rigor.fn('reject-mutation', checkMutation, rigor.args(
|
||||
rigor.gen.oneOf(Object.keys(MUTATIONS))
|
||||
))
|
||||
],
|
||||
rigor.crucible([
|
||||
// `actual` is the fn's return; a contract violation (a must-fail
|
||||
// mutation that compiled, a control that failed, or an unknown name)
|
||||
// throws → actual undefined → this invariant fails.
|
||||
rigor.invariant('rejection-contract', ({ actual }) =>
|
||||
!!actual && actual.ok === true)
|
||||
])
|
||||
).run({ seed: 'dsl-illegal-mutations', effort: 400, artifacts: { dir: '', persist: 'never' } });
|
||||
|
||||
const inv = report.crucibleVerdict?.invariants?.find(i => i.name === 'rejection-contract');
|
||||
assert.ok(inv, 'crucible invariant missing');
|
||||
assert.equal(inv.passed, true, `rejection contract violated in ${inv.failureCount} cases`);
|
||||
});
|
||||
|
||||
it('every mutation kind is exercised (no vacuous pass)', () => {
|
||||
const seen = new Set();
|
||||
for (const name of Object.keys(MUTATIONS)) {
|
||||
// deterministic probe of each kind
|
||||
seen.add(name);
|
||||
checkMutation(name);
|
||||
}
|
||||
assert.equal(seen.size, Object.keys(MUTATIONS).length);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user