(mess) Stuffing commit.

src/formula/evaluator.ts

@@ -285,7 +285,7 @@ function evaluateQuantified(
   bodyFn: (item: unknown) => boolean
 ): boolean {
   if (!Array.isArray(collection)) {
-    throw new Error(`Quantified expression requires an array collection, got: ${typeof collection}`)
+    return false
   }
   if (quantifier === 'for') {
     return collection.every(bodyFn)
@@ -299,7 +299,7 @@ async function evaluateQuantifiedAsync(
   bodyFn: (item: unknown) => Promise<boolean>
 ): Promise<boolean> {
   if (!Array.isArray(collection)) {
-    throw new Error(`Quantified expression requires an array collection, got: ${typeof collection}`)
+    return false
   }
   if (quantifier === 'for') {
     for (const item of collection) {

src/index.ts

@@ -13,14 +13,33 @@ export default fp(apophisPlugin, {
 
 export * from './types.js'
 
-// Chaos-v3: Pure chaos application for property-based testing
+// Quality engines
 export {
   applyChaosToExecution,
   applyChaosToAllResponses,
   createChaosEventArbitrary,
   extractDelays,
   sleep,
+  hasAppliedChaos,
+  formatChaosEvents,
   type ChaosEvent,
   type ChaosEventType,
   type ChaosApplicationResult,
 } from './quality/chaos-v3.js'
+
+export {
+  FlakeDetector,
+  type FlakeReport,
+  type FlakeRerun,
+  type FlakeOptions,
+} from './quality/flake.js'
+
+export {
+  runMutationTesting,
+  testMutation,
+  type Mutation,
+  type MutationType,
+  type MutationResult,
+  type MutationReport,
+  type MutationConfig,
+} from './quality/mutation.js'

src/quality/mutation.ts

@@ -256,20 +256,26 @@ export async function runMutationTesting(
 }
 /**
  * Run petit tests with a mutated contract.
- * This is a simplified version that tests a single mutated contract.
+ * Injects the mutated contract so the runner uses it instead of discovering from Fastify.
  */
 async function runPetitTestsWithMutation(
   fastify: FastifyInjectInstance,
   config: { runs?: number; seed?: number },
   mutatedContract: RouteContract
 ): Promise<TestSuite> {
-  // For now, run the full suite - the mutated contract will be discovered
-  // In a real implementation, you'd inject the mutated contract into the discovery
-  return runPetitTests(fastify, {
-    runs: config.runs ?? 10,
-    seed: config.seed,
-    routes: [`${mutatedContract.method} ${mutatedContract.path}`],
-  })
+  return runPetitTests(
+    fastify,
+    {
+      runs: config.runs ?? 10,
+      seed: config.seed,
+      routes: [`${mutatedContract.method} ${mutatedContract.path}`],
+    },
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    [mutatedContract]
+  )
 }
 /**
  * Quick mutation test for a single contract formula.

src/test/formula.test.ts

@@ -523,11 +523,35 @@ test('evaluate: returns error for missing previous context', () => {
   assert.strictEqual(result.success, false)
   assert.ok((result as { success: false; error: string }).error.includes('No previous context'))
 })
-test('evaluate: returns error for non-array in quantified expression', () => {
+test('evaluate: returns false for non-array in quantified expression', () => {
   const ast = parse('for x in response_code(this): x == 1')
   const result = evaluate(ast.ast, makeContext())
-  assert.strictEqual(result.success, false)
-  assert.ok((result as { success: false; error: string }).error.includes('array collection'))
+  assert.strictEqual(result.success, true)
+  assert.strictEqual((result as { success: true; value: unknown }).value, false)
+})
+test('evaluate: returns false for undefined collection in quantified expression', () => {
+  const ast = parse('for x in response_body(this): x == 1')
+  const result = evaluate(ast.ast, makeContext({ response: { body: undefined, headers: {}, statusCode: 200, responseTime: 0 } }))
+  assert.strictEqual(result.success, true)
+  assert.strictEqual((result as { success: true; value: unknown }).value, false)
+})
+test('evaluate: returns false for error object collection in quantified expression', () => {
+  const ast = parse('for x in response_body(this): x == 1')
+  const result = evaluate(ast.ast, makeContext({ response: { body: { error: 'Chaos error: forced 503' }, headers: {}, statusCode: 200, responseTime: 0 } }))
+  assert.strictEqual(result.success, true)
+  assert.strictEqual((result as { success: true; value: unknown }).value, false)
+})
+test('evaluate: quantified nested in conditional handles undefined gracefully', () => {
+  const ast = parse('if status:503 then for x in response_body(this): x == 1 else true')
+  const result = evaluate(ast.ast, makeContext({ response: { statusCode: 503, body: undefined, headers: {}, responseTime: 0 } }))
+  assert.strictEqual(result.success, true)
+  assert.strictEqual((result as { success: true; value: unknown }).value, false)
+})
+test('evaluate: exists returns false for non-array collection', () => {
+  const ast = parse('exists x in response_body(this): x == 1')
+  const result = evaluate(ast.ast, makeContext({ response: { body: { error: 'fail' }, headers: {}, statusCode: 200, responseTime: 0 } }))
+  assert.strictEqual(result.success, true)
+  assert.strictEqual((result as { success: true; value: unknown }).value, false)
 })
 // ============================================================================
 // Unit Tests: Substitutor

src/test/petit-runner.ts

@@ -105,12 +105,13 @@ export const runPetitTests = async (
   scopeRegistry?: ScopeRegistry,
   extensionRegistry?: ExtensionRegistry,
   pluginContractRegistry?: import('../domain/plugin-contracts.js').PluginContractRegistry,
-  outboundContractRegistry?: OutboundContractRegistry
+  outboundContractRegistry?: OutboundContractRegistry,
+  overrideContracts?: RouteContract[]
 ): Promise<TestSuite> => {
   const startTime = Date.now()
   if (extensionRegistry) await extensionRegistry.runSuiteStartHooks(config)
 
-  const allRoutes = discoverRoutes(fastify)
+  const allRoutes = overrideContracts ?? discoverRoutes(fastify)
   const { routes, skippedRoutes } = filterPetitRoutes(allRoutes, config)
 
   // Merge plugin contracts into route contracts

src/test/quality.test.ts

@@ -0,0 +1,425 @@
+/**
+ * Quality Engine Tests — Flake Detection, Mutation Testing, Chaos
+ */
+import { test } from 'node:test'
+import assert from 'node:assert'
+import Fastify from 'fastify'
+import swagger from '@fastify/swagger'
+import apophisPlugin from '../index.js'
+import { FlakeDetector } from '../quality/flake.js'
+import { runMutationTesting, testMutation, type Mutation } from '../quality/mutation.js'
+import * as fc from 'fast-check'
+import {
+  applyChaosToExecution,
+  applyChaosToAllResponses,
+  applyChaosToDependencyResponse,
+  createChaosEventArbitrary,
+  extractDelays,
+  sleep,
+  hasAppliedChaos,
+  formatChaosEvents,
+  type ChaosEvent,
+} from '../quality/chaos-v3.js'
+import type { EvalContext, TestResult, RouteContract } from '../types.js'
+
+// Quality engines require test environment
+process.env.NODE_ENV = 'test'
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeCtx(overrides?: Partial<EvalContext['response']>): EvalContext {
+  return {
+    request: { body: {}, headers: {}, query: {}, params: {} },
+    response: { body: { id: '1' }, headers: {}, statusCode: 200, ...overrides },
+  } as EvalContext
+}
+
+function makeTestResult(ok: boolean): TestResult {
+  return {
+    ok,
+    name: 'test',
+    id: 1,
+    diagnostics: ok ? {} : { error: 'failed' },
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Chaos Tests
+// ---------------------------------------------------------------------------
+
+test('applyChaosToExecution: no chaos when events are empty', () => {
+  const ctx = makeCtx()
+  const result = applyChaosToExecution(ctx, [])
+  assert.strictEqual(result.applied, false)
+  assert.strictEqual(result.ctx.response.statusCode, 200)
+})
+
+test('applyChaosToExecution: inbound error changes status code', () => {
+  const ctx = makeCtx()
+  const result = applyChaosToExecution(ctx, [
+    { type: 'inbound-error', target: 'inbound', statusCode: 503 },
+  ])
+  assert.strictEqual(result.applied, true)
+  assert.strictEqual(result.ctx.response.statusCode, 503)
+})
+
+test('applyChaosToExecution: inbound dropout simulates gateway timeout', () => {
+  const ctx = makeCtx()
+  const result = applyChaosToExecution(ctx, [
+    { type: 'inbound-dropout', target: 'inbound' },
+  ])
+  assert.strictEqual(result.ctx.response.statusCode, 504)
+})
+
+test('applyChaosToExecution: inbound corruption truncates response body', () => {
+  const ctx = makeCtx({ body: { id: '1', name: 'Alice', email: 'a@b.com' } })
+  const result = applyChaosToExecution(ctx, [
+    { type: 'inbound-corruption', target: 'inbound', corruptionStrategy: 'truncate' },
+  ])
+  const body = result.ctx.response.body as Record<string, unknown>
+  assert.ok(Object.keys(body).length <= 2)
+})
+
+test('applyChaosToExecution: inbound corruption with field-corrupt', () => {
+  const ctx = makeCtx({ body: { id: '1', name: 'Alice' } })
+  const result = applyChaosToExecution(ctx, [
+    { type: 'inbound-corruption', target: 'inbound', corruptionStrategy: 'field-corrupt', corruptionField: 'name' },
+  ])
+  const body = result.ctx.response.body as Record<string, unknown>
+  assert.strictEqual(body.name, null)
+})
+
+test('applyChaosToExecution: inbound corruption malformed', () => {
+  const ctx = makeCtx({ body: { id: '1' } })
+  const result = applyChaosToExecution(ctx, [
+    { type: 'inbound-corruption', target: 'inbound', corruptionStrategy: 'malformed' },
+  ])
+  assert.strictEqual(result.ctx.response.body, '{"broken":')
+})
+
+test('applyChaosToExecution: ignores none events', () => {
+  const ctx = makeCtx()
+  const result = applyChaosToExecution(ctx, [
+    { type: 'none', target: 'inbound' },
+  ])
+  assert.strictEqual(result.applied, false)
+})
+
+test('applyChaosToExecution: only applies first non-delay event', () => {
+  const ctx = makeCtx()
+  const result = applyChaosToExecution(ctx, [
+    { type: 'inbound-error', target: 'inbound', statusCode: 503 },
+    { type: 'inbound-dropout', target: 'inbound' },
+  ])
+  assert.strictEqual(result.ctx.response.statusCode, 503)
+})
+
+test('applyChaosToDependencyResponse: outbound error changes status', () => {
+  const response = { contractName: 'api', statusCode: 200, body: { ok: true } }
+  const result = applyChaosToDependencyResponse(response, [
+    { type: 'outbound-error', target: 'outbound', contractName: 'api', statusCode: 503 },
+  ])
+  assert.strictEqual(result.statusCode, 503)
+})
+
+test('applyChaosToDependencyResponse: ignores events for other contracts', () => {
+  const response = { contractName: 'api', statusCode: 200, body: {} }
+  const result = applyChaosToDependencyResponse(response, [
+    { type: 'outbound-error', target: 'outbound', contractName: 'other', statusCode: 503 },
+  ])
+  assert.strictEqual(result.statusCode, 200)
+})
+
+test('applyChaosToAllResponses: applies chaos to multiple responses', () => {
+  const responses = [
+    { contractName: 'a', statusCode: 200, body: {} },
+    { contractName: 'b', statusCode: 200, body: {} },
+  ]
+  const events = [
+    { type: 'outbound-error', target: 'outbound', contractName: 'a', statusCode: 503 },
+    { type: 'outbound-error', target: 'outbound', contractName: 'b', statusCode: 504 },
+  ] as ChaosEvent[]
+  const result = applyChaosToAllResponses(responses, events)
+  assert.strictEqual(result[0]!.statusCode, 503)
+  assert.strictEqual(result[1]!.statusCode, 504)
+})
+
+test('extractDelays: computes total delay', () => {
+  const events = [
+    { type: 'inbound-delay', target: 'inbound', delayMs: 100 },
+    { type: 'outbound-delay', target: 'outbound', delayMs: 200 },
+  ] as ChaosEvent[]
+  const result = extractDelays(events)
+  assert.strictEqual(result.totalMs, 300)
+  assert.strictEqual(result.events.length, 2)
+})
+
+test('sleep: resolves after specified ms', async () => {
+  const start = Date.now()
+  await sleep(10)
+  const elapsed = Date.now() - start
+  assert.ok(elapsed >= 8 && elapsed < 100)
+})
+
+test('hasAppliedChaos: detects applied chaos', () => {
+  assert.strictEqual(hasAppliedChaos([{ type: 'none', target: 'inbound' }]), false)
+  assert.strictEqual(hasAppliedChaos([{ type: 'inbound-error', target: 'inbound' }]), true)
+})
+
+test('formatChaosEvents: formats events for diagnostics', () => {
+  const events = [
+    { type: 'inbound-error', target: 'inbound', statusCode: 503 },
+  ] as ChaosEvent[]
+  const formatted = formatChaosEvents(events)
+  assert.ok(formatted.includes('inbound-error'))
+  assert.ok(formatted.includes('503'))
+})
+
+test('formatChaosEvents: returns "No chaos applied" for empty events', () => {
+  assert.strictEqual(formatChaosEvents([]), 'No chaos applied')
+  assert.strictEqual(formatChaosEvents([{ type: 'none', target: 'inbound' }]), 'No chaos applied')
+})
+
+test('createChaosEventArbitrary: generates deterministic events with seed', () => {
+  const arb = createChaosEventArbitrary(
+    { probability: 1, delay: { probability: 1, minMs: 100, maxMs: 200 } },
+    []
+  )
+  const sample1 = fc.sample(arb, { seed: 42, numRuns: 5 })
+  const sample2 = fc.sample(arb, { seed: 42, numRuns: 5 })
+  assert.deepStrictEqual(sample1, sample2)
+})
+
+test('createChaosEventArbitrary: returns empty array when no config', () => {
+  const arb = createChaosEventArbitrary(undefined, [])
+  const sample = fc.sample(arb, { numRuns: 10 })
+  assert.ok(sample.every((events) => events.length === 0 || events.every((e) => e.type === 'none')))
+})
+
+test('createChaosEventArbitrary: generates outbound events for contracts', () => {
+  const arb = createChaosEventArbitrary(
+    {
+      probability: 1,
+      error: { probability: 1, statusCode: 503 },
+      outbound: [
+        {
+          target: 'api',
+          error: {
+            probability: 1,
+            responses: [{ statusCode: 500 }],
+          },
+        },
+      ],
+    },
+    ['api']
+  )
+  const sample = fc.sample(arb, { numRuns: 20 })
+  const hasOutbound = sample.some((events) =>
+    events.some((e) => e.target === 'outbound')
+  )
+  assert.ok(hasOutbound, 'expected some outbound events')
+})
+
+// ---------------------------------------------------------------------------
+// Flake Detection Tests
+// ---------------------------------------------------------------------------
+
+test('FlakeDetector: deterministic failure returns high confidence', async () => {
+  const detector = new FlakeDetector({ sameSeedReruns: 1, seedVariations: 2 })
+  const report = await detector.detectFlake(
+    makeTestResult(false),
+    async () => ({ passed: false }),
+    42
+  )
+  assert.strictEqual(report.isFlaky, false)
+  assert.strictEqual(report.confidence, 'high')
+  assert.strictEqual(report.reruns.length, 3)
+})
+
+test('FlakeDetector: passing same-seed rerun flags flaky', async () => {
+  const detector = new FlakeDetector({ sameSeedReruns: 1, seedVariations: 0 })
+  let callCount = 0
+  const report = await detector.detectFlake(
+    makeTestResult(false),
+    async () => {
+      callCount++
+      return { passed: callCount === 1 }
+    },
+    42
+  )
+  assert.strictEqual(report.isFlaky, true)
+  assert.strictEqual(report.confidence, 'low')
+})
+
+test('FlakeDetector: medium confidence when some pass', async () => {
+  const detector = new FlakeDetector({ sameSeedReruns: 1, seedVariations: 2 })
+  let callCount = 0
+  const report = await detector.detectFlake(
+    makeTestResult(false),
+    async () => {
+      callCount++
+      return { passed: callCount % 2 === 0 }
+    },
+    42
+  )
+  assert.strictEqual(report.isFlaky, true)
+  assert.strictEqual(report.confidence, 'medium')
+})
+
+test('FlakeDetector: uses Date.now() when no seed provided', async () => {
+  const detector = new FlakeDetector({ sameSeedReruns: 0, seedVariations: 1 })
+  const report = await detector.detectFlake(
+    makeTestResult(false),
+    async () => ({ passed: false })
+  )
+  assert.strictEqual(report.reruns.length, 1)
+  assert.ok(report.reruns[0]!.seed > 0)
+})
+
+// ---------------------------------------------------------------------------
+// Mutation Testing Tests
+// ---------------------------------------------------------------------------
+
+function makeContract(ensures: string[], requires: string[] = []): RouteContract {
+  return {
+    path: '/items',
+    method: 'POST',
+    category: 'constructor',
+    requires,
+    ensures,
+    invariants: [],
+    regexPatterns: {},
+    validateRuntime: false,
+    schema: {
+      body: {
+        type: 'object',
+        properties: { name: { type: 'string' } },
+      },
+      response: {
+        201: {
+          type: 'object',
+          properties: { id: { type: 'string' } },
+        },
+      },
+    },
+  }
+}
+
+test('testMutation: detects flipped operator mutation', async () => {
+  const fastify = Fastify()
+  await fastify.register(swagger)
+  await fastify.register(apophisPlugin)
+  fastify.post('/items', {
+    schema: {
+      body: { type: 'object', properties: { name: { type: 'string' } } },
+      response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
+      'x-ensures': ['response_code(this) == 201'],
+    },
+  }, async (request, reply) => {
+    reply.status(201)
+    return { id: '1' }
+  })
+  await fastify.ready()
+
+  const mutation: Mutation = {
+    id: 'm1',
+    route: 'POST /items',
+    original: 'response_code(this) == 201',
+    mutated: 'response_code(this) != 201',
+    type: 'flip-operator',
+  }
+
+  const killed = await testMutation(fastify, makeContract(['response_code(this) == 201']), mutation)
+  assert.strictEqual(killed, true)
+})
+
+test('runMutationTesting: produces report with score', async () => {
+  const fastify = Fastify()
+  await fastify.register(swagger)
+  await fastify.register(apophisPlugin)
+  fastify.post('/items', {
+    schema: {
+      body: { type: 'object', properties: { name: { type: 'string' } } },
+      response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
+      'x-ensures': ['response_code(this) == 201'],
+    },
+  }, async (request, reply) => {
+    reply.status(201)
+    return { id: '1' }
+  })
+  await fastify.ready()
+
+  const report = await runMutationTesting(fastify, { runs: 5, seed: 42, maxMutationsPerContract: 3 })
+  assert.ok(typeof report.score === 'number')
+  assert.ok(report.score >= 0 && report.score <= 100)
+  assert.ok(report.mutations.length > 0)
+  assert.ok(report.durationMs > 0)
+})
+
+test('runMutationTesting: skips routes without contracts', async () => {
+  const fastify = Fastify()
+  await fastify.register(swagger)
+  await fastify.register(apophisPlugin)
+  fastify.get('/health', {}, async () => 'ok')
+  await fastify.ready()
+
+  const report = await runMutationTesting(fastify, { runs: 5, seed: 42 })
+  assert.strictEqual(report.mutations.length, 0)
+  assert.strictEqual(report.score, 0)
+})
+
+test('runMutationTesting: filters by route', async () => {
+  const fastify = Fastify()
+  await fastify.register(swagger)
+  await fastify.register(apophisPlugin)
+  fastify.post('/items', {
+    schema: {
+      body: { type: 'object', properties: { name: { type: 'string' } } },
+      response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
+      'x-ensures': ['response_code(this) == 201'],
+    },
+  }, async (request, reply) => {
+    reply.status(201)
+    return { id: '1' }
+  })
+  fastify.post('/other', {
+    schema: {
+      body: { type: 'object', properties: { name: { type: 'string' } } },
+      response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
+      'x-ensures': ['response_code(this) == 201'],
+    },
+  }, async (request, reply) => {
+    reply.status(201)
+    return { id: '2' }
+  })
+  await fastify.ready()
+
+  const report = await runMutationTesting(fastify, { runs: 5, seed: 42, routes: ['/items'] })
+  const itemsMutations = report.mutations.filter((m) => m.mutation.route.includes('/items'))
+  const otherMutations = report.mutations.filter((m) => m.mutation.route.includes('/other'))
+  assert.ok(itemsMutations.length > 0)
+  assert.strictEqual(otherMutations.length, 0)
+})
+
+test('runMutationTesting: identifies weak contracts', async () => {
+  const fastify = Fastify()
+  await fastify.register(swagger)
+  await fastify.register(apophisPlugin)
+  fastify.post('/items', {
+    schema: {
+      body: { type: 'object', properties: { name: { type: 'string' } } },
+      response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
+      'x-ensures': ['response_code(this) == 201', 'response_body(this).id != null'],
+    },
+  }, async (request, reply) => {
+    reply.status(201)
+    return { id: '1' }
+  })
+  await fastify.ready()
+
+  const report = await runMutationTesting(fastify, { runs: 5, seed: 42 })
+  // All mutations should be killed since the handler always returns 201 with id
+  assert.strictEqual(report.weakContracts.length, 0)
+})