/** * Mutation Testing Engine for APOPHIS * * Injects synthetic bugs into APOSTL contracts to measure test suite strength. * A "mutation" is a small change to a contract (e.g., flip == to !=, change a number). * If the test suite catches the mutation (fails), the mutation is "killed". * If the test suite passes, the mutation "survives" — indicating a gap in coverage. * * Usage: * const report = await runMutationTesting(fastify, { runs: 10 }) * console.log(`Mutation score: ${report.score}%`) */ import type { FastifyInstance } from 'fastify' import { runPetitTests } from '../test/petit-runner.js' import { discoverRoutes } from '../domain/discovery.js' import type { FastifyInjectInstance, RouteContract, TestConfig, TestSuite } from '../types.js' export interface Mutation { readonly id: string readonly route: string readonly original: string readonly mutated: string readonly type: MutationType } export type MutationType = | 'flip-operator' // == → !=, < → >= | 'change-number' // 200 → 201, 0 → 1 | 'remove-clause' // A && B → A | 'negate-boolean' // true → false | 'swap-variable' // response_body → request_body | 'remove-ensures' // Remove one ensures clause export interface MutationResult { readonly mutation: Mutation readonly killed: boolean readonly error?: string readonly durationMs: number } export interface MutationReport { readonly mutations: MutationResult[] readonly killed: number readonly survived: number readonly score: number // 0-100 readonly durationMs: number readonly weakContracts: string[] // contracts that survived all mutations } export interface MutationConfig { readonly runs?: number readonly seed?: number /** Max mutations per contract (default: 5) */ readonly maxMutationsPerContract?: number /** Only mutate these routes */ readonly routes?: string[] } // ─── Mutation Operators ───────────────────────────────────────────────────── const MUTATION_OPERATORS: Array<(formula: string) => string | null> = [ // Flip equality operator (f) => { if (f.includes('==')) return f.replace('==', '!=') if (f.includes('!=')) return f.replace('!=', '==') return null }, // Flip comparison operator (f) => { if (f.includes('<=')) return f.replace('<=', '>') if (f.includes('>=')) return f.replace('>=', '<') if (f.includes('<') && !f.includes('<=')) return f.replace('<', '>=') if (f.includes('>') && !f.includes('>=')) return f.replace('>', '<=') return null }, // Change status code (f) => { const match = f.match(/status:(\d+)/) if (match && match[1]) { const code = parseInt(match[1], 10) return f.replace(`status:${code}`, `status:${code + 1}`) } return null }, // Change number literal (f) => { const match = f.match(/==\s*(\d+)/) if (match && match[1]) { const num = parseInt(match[1], 10) return f.replace(`== ${num}`, `== ${num + 1}`) } return null }, // Negate boolean (f) => { if (f.includes('== true')) return f.replace('== true', '== false') if (f.includes('== false')) return f.replace('== false', '== true') return null }, // Swap operation header (f) => { if (f.includes('response_body')) return f.replace('response_body', 'request_body') if (f.includes('request_body')) return f.replace('request_body', 'response_body') if (f.includes('response_code')) return f.replace('response_code', 'response_time') return null }, // Remove clause from conjunction (f) => { if (f.includes(' && ')) { const parts = f.split(' && ') if (parts.length > 1) { return parts.slice(1).join(' && ') } } return null }, ] function generateMutations(contract: RouteContract, maxMutations: number): Mutation[] { const mutations: Mutation[] = [] let mutationId = 0 // Collect all formulas const allFormulas = [...contract.ensures, ...contract.requires] for (const formula of allFormulas) { for (const operator of MUTATION_OPERATORS) { if (mutations.length >= maxMutations) break const mutated = operator(formula) if (mutated && mutated !== formula) { mutations.push({ id: `m${mutationId++}`, route: `${contract.method} ${contract.path}`, original: formula, mutated, type: inferMutationType(formula, mutated), }) } } } // Remove ensures clause entirely if (contract.ensures.length > 1 && mutations.length < maxMutations) { mutations.push({ id: `m${mutationId++}`, route: `${contract.method} ${contract.path}`, original: contract.ensures[0]!, mutated: '', type: 'remove-ensures', }) } return mutations } function inferMutationType(original: string, mutated: string): MutationType { if (original.includes('==') && mutated.includes('!=')) return 'flip-operator' if (original.includes('!=') && mutated.includes('==')) return 'flip-operator' if (/\d+/.test(original) && /\d+/.test(mutated)) { const origNum = original.match(/\d+/)?.[0] const mutNum = mutated.match(/\d+/)?.[0] if (origNum !== mutNum) return 'change-number' } if (original.includes('true') && mutated.includes('false')) return 'negate-boolean' if (original.includes('false') && mutated.includes('true')) return 'negate-boolean' if (original.includes(' && ') && !mutated.includes(' && ')) return 'remove-clause' if (original.includes('response_body') && mutated.includes('request_body')) return 'swap-variable' if (original.includes('request_body') && mutated.includes('response_body')) return 'swap-variable' return 'flip-operator' } /** * Apply a mutation to a route contract. */ function applyMutation(contract: RouteContract, mutation: Mutation): RouteContract { if (mutation.type === 'remove-ensures') { return { ...contract, ensures: contract.ensures.filter(f => f !== mutation.original), } } return { ...contract, ensures: contract.ensures.map(f => f === mutation.original ? mutation.mutated : f), requires: contract.requires.map(f => f === mutation.original ? mutation.mutated : f), } } // ─── Mutation Testing Runner ──────────────────────────────────────────────── /** * Run mutation testing against a Fastify instance. * * For each route contract, generates mutations and runs the test suite. * A mutation is "killed" if the test suite detects the contract violation. * * Returns a report with mutation score (percentage killed). */ export async function runMutationTesting( fastify: FastifyInstance, config: MutationConfig = {} ): Promise { const startTime = Date.now() const maxMutations = config.maxMutationsPerContract ?? 5 const results: MutationResult[] = [] const weakContracts: string[] = [] // Discover routes from Fastify const contracts = discoverRoutes(fastify as unknown as FastifyInjectInstance) // Filter routes if specified const targetContracts = config.routes ? contracts.filter(c => config.routes!.includes(c.path)) : contracts for (const contract of targetContracts) { // Skip contracts without any testable clauses if (contract.ensures.length === 0 && contract.requires.length === 0) { continue } const mutations = generateMutations(contract, maxMutations) let contractKilled = 0 for (const mutation of mutations) { const mutationStart = Date.now() // Apply mutation to the route's schema const mutatedContract = applyMutation(contract, mutation) // We need to temporarily mutate the route schema for testing // Since Fastify 5 doesn't expose routes directly, we work with the contract try { // Run test suite with mutated contract // We pass the mutated contract directly to the runner const suite = await runPetitTestsWithMutation( fastify as unknown as FastifyInjectInstance, { runs: config.runs ?? 10, seed: config.seed, }, mutatedContract ) const killed = suite.summary.failed > 0 results.push({ mutation, killed, durationMs: Date.now() - mutationStart, }) if (killed) contractKilled++ } catch (err) { // Error during testing counts as killed results.push({ mutation, killed: true, error: err instanceof Error ? err.message : String(err), durationMs: Date.now() - mutationStart, }) contractKilled++ } } // Track weak contracts (none of their mutations were killed) if (mutations.length > 0 && contractKilled === 0) { weakContracts.push(`${contract.method} ${contract.path}`) } } const killed = results.filter(r => r.killed).length const survived = results.filter(r => !r.killed).length const total = results.length return { mutations: results, killed, survived, score: total > 0 ? Math.round((killed / total) * 100) : 0, durationMs: Date.now() - startTime, weakContracts, } } /** * Run petit tests with a mutated contract. * Injects the mutated contract so the runner uses it instead of discovering from Fastify. */ async function runPetitTestsWithMutation( fastify: FastifyInjectInstance, config: { runs?: number; seed?: number }, mutatedContract: RouteContract ): Promise { return runPetitTests( fastify, { runs: config.runs ?? 10, seed: config.seed, routes: [`${mutatedContract.method} ${mutatedContract.path}`], }, undefined, undefined, undefined, undefined, [mutatedContract] ) } /** * Quick mutation test for a single contract formula. * Returns true if the mutation would be caught. */ export async function testMutation( fastify: FastifyInstance, contract: RouteContract, mutation: Mutation, config: Pick = {} ): Promise { const mutatedContract = applyMutation(contract, mutation) try { const suite = await runPetitTestsWithMutation( fastify as unknown as FastifyInjectInstance, { runs: config.runs ?? 10, seed: config.seed, }, mutatedContract ) return suite.summary.failed > 0 } catch { return true } }