/** * Quality Engine Tests — Flake Detection, Mutation Testing, Chaos */ import { test } from 'node:test' import assert from 'node:assert' import Fastify from 'fastify' import swagger from '@fastify/swagger' import apophisPlugin from '../index.js' import { FlakeDetector } from '../quality/flake.js' import { runMutationTesting, testMutation, type Mutation } from '../quality/mutation.js' import * as fc from 'fast-check' import { applyChaosToExecution, applyChaosToAllResponses, applyChaosToDependencyResponse, createChaosEventArbitrary, extractDelays, sleep, hasAppliedChaos, formatChaosEvents, type ChaosEvent, } from '../quality/chaos-v3.js' import type { EvalContext, TestResult, RouteContract } from '../types.js' // Quality engines require test environment process.env.NODE_ENV = 'test' // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- function makeCtx(overrides?: Partial): EvalContext { return { request: { body: {}, headers: {}, query: {}, params: {} }, response: { body: { id: '1' }, headers: {}, statusCode: 200, ...overrides }, } as EvalContext } function makeTestResult(ok: boolean): TestResult { return { ok, name: 'test', id: 1, diagnostics: ok ? {} : { error: 'failed' }, } } // --------------------------------------------------------------------------- // Chaos Tests // --------------------------------------------------------------------------- test('applyChaosToExecution: no chaos when events are empty', () => { const ctx = makeCtx() const result = applyChaosToExecution(ctx, []) assert.strictEqual(result.applied, false) assert.strictEqual(result.ctx.response.statusCode, 200) }) test('applyChaosToExecution: inbound error changes status code', () => { const ctx = makeCtx() const result = applyChaosToExecution(ctx, [ { type: 'inbound-error', target: 'inbound', statusCode: 503 }, ]) assert.strictEqual(result.applied, true) assert.strictEqual(result.ctx.response.statusCode, 503) }) test('applyChaosToExecution: inbound dropout simulates gateway timeout', () => { const ctx = makeCtx() const result = applyChaosToExecution(ctx, [ { type: 'inbound-dropout', target: 'inbound' }, ]) assert.strictEqual(result.ctx.response.statusCode, 504) }) test('applyChaosToExecution: inbound corruption truncates response body', () => { const ctx = makeCtx({ body: { id: '1', name: 'Alice', email: 'a@b.com' } }) const result = applyChaosToExecution(ctx, [ { type: 'inbound-corruption', target: 'inbound', corruptionStrategy: 'truncate' }, ]) const body = result.ctx.response.body as Record assert.ok(Object.keys(body).length <= 2) }) test('applyChaosToExecution: inbound corruption with field-corrupt', () => { const ctx = makeCtx({ body: { id: '1', name: 'Alice' } }) const result = applyChaosToExecution(ctx, [ { type: 'inbound-corruption', target: 'inbound', corruptionStrategy: 'field-corrupt', corruptionField: 'name' }, ]) const body = result.ctx.response.body as Record assert.strictEqual(body.name, null) }) test('applyChaosToExecution: inbound corruption malformed', () => { const ctx = makeCtx({ body: { id: '1' } }) const result = applyChaosToExecution(ctx, [ { type: 'inbound-corruption', target: 'inbound', corruptionStrategy: 'malformed' }, ]) assert.strictEqual(result.ctx.response.body, '{"broken":') }) test('applyChaosToExecution: ignores none events', () => { const ctx = makeCtx() const result = applyChaosToExecution(ctx, [ { type: 'none', target: 'inbound' }, ]) assert.strictEqual(result.applied, false) }) test('applyChaosToExecution: only applies first non-delay event', () => { const ctx = makeCtx() const result = applyChaosToExecution(ctx, [ { type: 'inbound-error', target: 'inbound', statusCode: 503 }, { type: 'inbound-dropout', target: 'inbound' }, ]) assert.strictEqual(result.ctx.response.statusCode, 503) }) test('applyChaosToDependencyResponse: outbound error changes status', () => { const response = { contractName: 'api', statusCode: 200, body: { ok: true } } const result = applyChaosToDependencyResponse(response, [ { type: 'outbound-error', target: 'outbound', contractName: 'api', statusCode: 503 }, ]) assert.strictEqual(result.statusCode, 503) }) test('applyChaosToDependencyResponse: ignores events for other contracts', () => { const response = { contractName: 'api', statusCode: 200, body: {} } const result = applyChaosToDependencyResponse(response, [ { type: 'outbound-error', target: 'outbound', contractName: 'other', statusCode: 503 }, ]) assert.strictEqual(result.statusCode, 200) }) test('applyChaosToAllResponses: applies chaos to multiple responses', () => { const responses = [ { contractName: 'a', statusCode: 200, body: {} }, { contractName: 'b', statusCode: 200, body: {} }, ] const events = [ { type: 'outbound-error', target: 'outbound', contractName: 'a', statusCode: 503 }, { type: 'outbound-error', target: 'outbound', contractName: 'b', statusCode: 504 }, ] as ChaosEvent[] const result = applyChaosToAllResponses(responses, events) assert.strictEqual(result[0]!.statusCode, 503) assert.strictEqual(result[1]!.statusCode, 504) }) test('extractDelays: computes total delay', () => { const events = [ { type: 'inbound-delay', target: 'inbound', delayMs: 100 }, { type: 'outbound-delay', target: 'outbound', delayMs: 200 }, ] as ChaosEvent[] const result = extractDelays(events) assert.strictEqual(result.totalMs, 300) assert.strictEqual(result.events.length, 2) }) test('sleep: resolves after specified ms', async () => { const start = Date.now() await sleep(10) const elapsed = Date.now() - start assert.ok(elapsed >= 8 && elapsed < 100) }) test('hasAppliedChaos: detects applied chaos', () => { assert.strictEqual(hasAppliedChaos([{ type: 'none', target: 'inbound' }]), false) assert.strictEqual(hasAppliedChaos([{ type: 'inbound-error', target: 'inbound' }]), true) }) test('formatChaosEvents: formats events for diagnostics', () => { const events = [ { type: 'inbound-error', target: 'inbound', statusCode: 503 }, ] as ChaosEvent[] const formatted = formatChaosEvents(events) assert.ok(formatted.includes('inbound-error')) assert.ok(formatted.includes('503')) }) test('formatChaosEvents: returns "No chaos applied" for empty events', () => { assert.strictEqual(formatChaosEvents([]), 'No chaos applied') assert.strictEqual(formatChaosEvents([{ type: 'none', target: 'inbound' }]), 'No chaos applied') }) test('createChaosEventArbitrary: generates deterministic events with seed', () => { const arb = createChaosEventArbitrary( { probability: 1, delay: { probability: 1, minMs: 100, maxMs: 200 } }, [] ) const sample1 = fc.sample(arb, { seed: 42, numRuns: 5 }) const sample2 = fc.sample(arb, { seed: 42, numRuns: 5 }) assert.deepStrictEqual(sample1, sample2) }) test('createChaosEventArbitrary: returns empty array when no config', () => { const arb = createChaosEventArbitrary(undefined, []) const sample = fc.sample(arb, { numRuns: 10 }) assert.ok(sample.every((events) => events.length === 0 || events.every((e) => e.type === 'none'))) }) test('createChaosEventArbitrary: generates outbound events for contracts', () => { const arb = createChaosEventArbitrary( { probability: 1, error: { probability: 1, statusCode: 503 }, outbound: [ { target: 'api', error: { probability: 1, responses: [{ statusCode: 500 }], }, }, ], }, ['api'] ) const sample = fc.sample(arb, { numRuns: 20 }) const hasOutbound = sample.some((events) => events.some((e) => e.target === 'outbound') ) assert.ok(hasOutbound, 'expected some outbound events') }) // --------------------------------------------------------------------------- // Flake Detection Tests // --------------------------------------------------------------------------- test('FlakeDetector: deterministic failure returns high confidence', async () => { const detector = new FlakeDetector({ sameSeedReruns: 1, seedVariations: 2 }) const report = await detector.detectFlake( makeTestResult(false), async () => ({ passed: false }), 42 ) assert.strictEqual(report.isFlaky, false) assert.strictEqual(report.confidence, 'high') assert.strictEqual(report.reruns.length, 3) }) test('FlakeDetector: passing same-seed rerun flags flaky', async () => { const detector = new FlakeDetector({ sameSeedReruns: 1, seedVariations: 0 }) let callCount = 0 const report = await detector.detectFlake( makeTestResult(false), async () => { callCount++ return { passed: callCount === 1 } }, 42 ) assert.strictEqual(report.isFlaky, true) assert.strictEqual(report.confidence, 'low') }) test('FlakeDetector: medium confidence when some pass', async () => { const detector = new FlakeDetector({ sameSeedReruns: 1, seedVariations: 2 }) let callCount = 0 const report = await detector.detectFlake( makeTestResult(false), async () => { callCount++ return { passed: callCount % 2 === 0 } }, 42 ) assert.strictEqual(report.isFlaky, true) assert.strictEqual(report.confidence, 'medium') }) test('FlakeDetector: uses Date.now() when no seed provided', async () => { const detector = new FlakeDetector({ sameSeedReruns: 0, seedVariations: 1 }) const report = await detector.detectFlake( makeTestResult(false), async () => ({ passed: false }) ) assert.strictEqual(report.reruns.length, 1) assert.ok(report.reruns[0]!.seed > 0) }) // --------------------------------------------------------------------------- // Mutation Testing Tests // --------------------------------------------------------------------------- function makeContract(ensures: string[], requires: string[] = []): RouteContract { return { path: '/items', method: 'POST', category: 'constructor', requires, ensures, invariants: [], regexPatterns: {}, validateRuntime: false, schema: { body: { type: 'object', properties: { name: { type: 'string' } }, }, response: { 201: { type: 'object', properties: { id: { type: 'string' } }, }, }, }, } } test('testMutation: detects flipped operator mutation', async () => { const fastify = Fastify() await fastify.register(swagger) await fastify.register(apophisPlugin) fastify.post('/items', { schema: { body: { type: 'object', properties: { name: { type: 'string' } } }, response: { 201: { type: 'object', properties: { id: { type: 'string' } } } }, 'x-ensures': ['response_code(this) == 201'], }, }, async (request, reply) => { reply.status(201) return { id: '1' } }) await fastify.ready() const mutation: Mutation = { id: 'm1', route: 'POST /items', original: 'response_code(this) == 201', mutated: 'response_code(this) != 201', type: 'flip-operator', } const killed = await testMutation(fastify, makeContract(['response_code(this) == 201']), mutation) assert.strictEqual(killed, true) }) test('runMutationTesting: produces report with score', async () => { const fastify = Fastify() await fastify.register(swagger) await fastify.register(apophisPlugin) fastify.post('/items', { schema: { body: { type: 'object', properties: { name: { type: 'string' } } }, response: { 201: { type: 'object', properties: { id: { type: 'string' } } } }, 'x-ensures': ['response_code(this) == 201'], }, }, async (request, reply) => { reply.status(201) return { id: '1' } }) await fastify.ready() const report = await runMutationTesting(fastify, { runs: 5, seed: 42, maxMutationsPerContract: 3 }) assert.ok(typeof report.score === 'number') assert.ok(report.score >= 0 && report.score <= 100) assert.ok(report.mutations.length > 0) assert.ok(report.durationMs > 0) }) test('runMutationTesting: skips routes without contracts', async () => { const fastify = Fastify() await fastify.register(swagger) await fastify.register(apophisPlugin) fastify.get('/health', {}, async () => 'ok') await fastify.ready() const report = await runMutationTesting(fastify, { runs: 5, seed: 42 }) assert.strictEqual(report.mutations.length, 0) assert.strictEqual(report.score, 0) }) test('runMutationTesting: filters by route', async () => { const fastify = Fastify() await fastify.register(swagger) await fastify.register(apophisPlugin) fastify.post('/items', { schema: { body: { type: 'object', properties: { name: { type: 'string' } } }, response: { 201: { type: 'object', properties: { id: { type: 'string' } } } }, 'x-ensures': ['response_code(this) == 201'], }, }, async (request, reply) => { reply.status(201) return { id: '1' } }) fastify.post('/other', { schema: { body: { type: 'object', properties: { name: { type: 'string' } } }, response: { 201: { type: 'object', properties: { id: { type: 'string' } } } }, 'x-ensures': ['response_code(this) == 201'], }, }, async (request, reply) => { reply.status(201) return { id: '2' } }) await fastify.ready() const report = await runMutationTesting(fastify, { runs: 5, seed: 42, routes: ['/items'] }) const itemsMutations = report.mutations.filter((m) => m.mutation.route.includes('/items')) const otherMutations = report.mutations.filter((m) => m.mutation.route.includes('/other')) assert.ok(itemsMutations.length > 0) assert.strictEqual(otherMutations.length, 0) }) test('runMutationTesting: identifies weak contracts', async () => { const fastify = Fastify() await fastify.register(swagger) await fastify.register(apophisPlugin) fastify.post('/items', { schema: { body: { type: 'object', properties: { name: { type: 'string' } } }, response: { 201: { type: 'object', properties: { id: { type: 'string' } } } }, 'x-ensures': ['response_code(this) == 201', 'response_body(this).id != null'], }, }, async (request, reply) => { reply.status(201) return { id: '1' } }) await fastify.ready() const report = await runMutationTesting(fastify, { runs: 5, seed: 42 }) // All mutations should be killed since the handler always returns 201 with id assert.strictEqual(report.weakContracts.length, 0) })