chore: crush git history - reborn from consolidation on 2026-03-10
This commit is contained in:
John Dvorak
@@ -0,0 +1,454 @@
|
||||
/**
|
||||
* Chaos-v3: Pure Chaos Application
|
||||
*
|
||||
* Chaos events are GENERATED by fast-check as part of the test arbitrary,
|
||||
* not picked at runtime. This makes chaos SHRINKABLE — when a test fails,
|
||||
* fast-check finds the minimal chaos event that causes the failure.
|
||||
*
|
||||
* Architecture:
|
||||
* 1. GENERATION: fast-check arbitrary generates ChaosEvent[] alongside requests
|
||||
* 2. APPLICATION: applyChaosToExecution() applies pre-generated events to EvalContext
|
||||
* 3. OUTBOUND: applyChaosToDependencyResponse() corrupts mock runtime responses
|
||||
*
|
||||
* No runtime RNG. No side effects during generation. Pure functions only.
|
||||
*/
|
||||
|
||||
import * as fc from 'fast-check'
|
||||
import type { ChaosConfig, EvalContext } from '../types.js'
|
||||
|
||||
// ============================================================================
|
||||
// Types
|
||||
// ============================================================================
|
||||
export type ChaosEventType =
|
||||
| 'none'
|
||||
| 'inbound-delay'
|
||||
| 'inbound-error'
|
||||
| 'inbound-dropout'
|
||||
| 'inbound-corruption'
|
||||
| 'outbound-delay'
|
||||
| 'outbound-error'
|
||||
| 'outbound-dropout'
|
||||
| 'outbound-corruption'
|
||||
export interface ChaosEvent {
|
||||
readonly type: ChaosEventType
|
||||
readonly target: 'inbound' | 'outbound'
|
||||
/** For outbound events: which dependency contract */
|
||||
readonly contractName?: string
|
||||
readonly delayMs?: number
|
||||
readonly statusCode?: number
|
||||
readonly body?: unknown
|
||||
readonly corruptionStrategy?: 'truncate' | 'malformed' | 'field-corrupt'
|
||||
readonly corruptionField?: string
|
||||
}
|
||||
export interface ChaosApplicationResult {
|
||||
readonly ctx: EvalContext
|
||||
readonly events: ReadonlyArray<ChaosEvent>
|
||||
/** Whether any chaos was actually applied */
|
||||
readonly applied: boolean
|
||||
}
|
||||
// ============================================================================
|
||||
// Inbound chaos event handlers
|
||||
// ============================================================================
|
||||
/**
|
||||
* Each handler receives the current EvalContext and the event,
|
||||
* and returns the modified context. Handlers are pure functions.
|
||||
*/
|
||||
type InboundChaosHandler = (ctx: EvalContext, event: ChaosEvent) => EvalContext
|
||||
|
||||
const inboundHandlers: Record<string, InboundChaosHandler> = {
|
||||
'inbound-delay': (ctx) => ctx,
|
||||
'inbound-error': (ctx, event) => ({
|
||||
...ctx,
|
||||
response: {
|
||||
...ctx.response,
|
||||
statusCode: event.statusCode ?? 500,
|
||||
body: event.body ?? { error: `Chaos error: forced ${event.statusCode ?? 500}` },
|
||||
},
|
||||
}),
|
||||
'inbound-dropout': (ctx, event) => ({
|
||||
...ctx,
|
||||
response: {
|
||||
...ctx.response,
|
||||
statusCode: event.statusCode ?? 504,
|
||||
body: { error: `Chaos dropout: ${event.statusCode ?? 504} Gateway Timeout simulated` },
|
||||
},
|
||||
}),
|
||||
'inbound-corruption': (ctx, event) => applyCorruptionToContext(ctx, event),
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Pure: Apply chaos to inbound execution context
|
||||
// ============================================================================
|
||||
/**
|
||||
* Apply pre-generated chaos events to an EvalContext.
|
||||
* Returns the modified context and metadata about what was applied.
|
||||
*
|
||||
* This is a PURE function: given the same events and context, it always
|
||||
* produces the same result. No RNG, no side effects.
|
||||
*/
|
||||
export function applyChaosToExecution(
|
||||
ctx: EvalContext,
|
||||
events: ReadonlyArray<ChaosEvent>
|
||||
): ChaosApplicationResult {
|
||||
const inboundEvents = events.filter((e) => e.target === 'inbound' && e.type !== 'none')
|
||||
if (inboundEvents.length === 0) {
|
||||
return { ctx, events, applied: false }
|
||||
}
|
||||
// Apply events in order: delay → error → dropout → corruption
|
||||
// Only the first applicable event modifies the context (they're mutually exclusive)
|
||||
let modified = ctx
|
||||
let applied = false
|
||||
for (const event of inboundEvents) {
|
||||
const handler = inboundHandlers[event.type]
|
||||
if (handler) {
|
||||
modified = handler(modified, event)
|
||||
applied = true
|
||||
}
|
||||
// Only apply the first non-delay event
|
||||
if (applied && event.type !== 'inbound-delay') {
|
||||
break
|
||||
}
|
||||
}
|
||||
return { ctx: modified, events, applied }
|
||||
}
|
||||
// ============================================================================
|
||||
// Corruption strategy handlers
|
||||
// ============================================================================
|
||||
|
||||
type CorruptionHandler = (body: unknown, event: ChaosEvent) => unknown
|
||||
|
||||
function truncateBody(body: unknown): unknown {
|
||||
if (typeof body === 'string') {
|
||||
const cutPoint = Math.floor(body.length / 2)
|
||||
return body.slice(0, cutPoint)
|
||||
}
|
||||
if (typeof body === 'object' && body !== null && !Array.isArray(body)) {
|
||||
const entries = Object.entries(body as Record<string, unknown>)
|
||||
const cutPoint = Math.floor(entries.length / 2)
|
||||
const truncated: Record<string, unknown> = {}
|
||||
for (let i = 0; i < cutPoint; i++) {
|
||||
const [k, v] = entries[i]!
|
||||
truncated[k] = v
|
||||
}
|
||||
return truncated
|
||||
}
|
||||
if (Array.isArray(body)) {
|
||||
const cutPoint = Math.max(1, Math.floor(body.length / 2))
|
||||
return body.slice(0, cutPoint)
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
function corruptField(body: unknown, field: string | undefined): unknown {
|
||||
if (typeof body !== 'object' || body === null || Array.isArray(body)) return body
|
||||
if (!field) return body
|
||||
const corrupted = { ...(body as Record<string, unknown>) }
|
||||
if (field in corrupted) {
|
||||
corrupted[field] = null
|
||||
}
|
||||
return corrupted
|
||||
}
|
||||
|
||||
const corruptionHandlers: Record<string, CorruptionHandler> = {
|
||||
'truncate': (body) => truncateBody(body),
|
||||
'malformed': () => '{"broken":',
|
||||
'field-corrupt': (body, event) => corruptField(body, event.corruptionField),
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply corruption to an EvalContext's response body.
|
||||
*/
|
||||
function applyCorruptionToContext(ctx: EvalContext, event: ChaosEvent): EvalContext {
|
||||
const body = ctx.response.body
|
||||
if (body === null || body === undefined) return ctx
|
||||
const handler = event.corruptionStrategy ? corruptionHandlers[event.corruptionStrategy] : undefined
|
||||
const corruptedBody = handler ? handler(body, event) : body
|
||||
if (corruptedBody === body) return ctx
|
||||
return {
|
||||
...ctx,
|
||||
response: {
|
||||
...ctx.response,
|
||||
body: corruptedBody,
|
||||
},
|
||||
}
|
||||
}
|
||||
// ============================================================================
|
||||
// Pure: Apply chaos to dependency responses
|
||||
// ============================================================================
|
||||
export interface DependencyResponse {
|
||||
readonly contractName: string
|
||||
readonly statusCode: number
|
||||
readonly body: unknown
|
||||
}
|
||||
/**
|
||||
* Apply pre-generated chaos events to a dependency response.
|
||||
* Returns the corrupted response.
|
||||
*/
|
||||
// ============================================================================
|
||||
// Outbound chaos event handlers
|
||||
// ============================================================================
|
||||
|
||||
type OutboundChaosHandler = (response: DependencyResponse, event: ChaosEvent) => DependencyResponse
|
||||
|
||||
const outboundHandlers: Record<string, OutboundChaosHandler> = {
|
||||
'outbound-error': (response, event) => ({
|
||||
...response,
|
||||
statusCode: event.statusCode ?? 503,
|
||||
body: event.body ?? { error: 'Dependency failure simulated' },
|
||||
}),
|
||||
'outbound-dropout': (response, event) => ({
|
||||
...response,
|
||||
statusCode: event.statusCode ?? 504,
|
||||
body: { error: 'Gateway timeout simulated' },
|
||||
}),
|
||||
'outbound-corruption': (response, event) =>
|
||||
applyCorruptionToDependencyResponse(response, event),
|
||||
'outbound-delay': (response) => response,
|
||||
}
|
||||
|
||||
export function applyChaosToDependencyResponse(
|
||||
response: DependencyResponse,
|
||||
events: ReadonlyArray<ChaosEvent>
|
||||
): DependencyResponse {
|
||||
const relevantEvents = events.filter(
|
||||
(e) =>
|
||||
e.target === 'outbound' &&
|
||||
e.contractName === response.contractName &&
|
||||
e.type !== 'none'
|
||||
)
|
||||
if (relevantEvents.length === 0) return response
|
||||
let modified = response
|
||||
for (const event of relevantEvents) {
|
||||
const handler = outboundHandlers[event.type]
|
||||
if (handler) {
|
||||
modified = handler(modified, event)
|
||||
}
|
||||
}
|
||||
return modified
|
||||
}
|
||||
function applyCorruptionToDependencyResponse(
|
||||
response: DependencyResponse,
|
||||
event: ChaosEvent
|
||||
): DependencyResponse {
|
||||
const body = response.body
|
||||
if (body === null || body === undefined) return response
|
||||
const handler = event.corruptionStrategy ? corruptionHandlers[event.corruptionStrategy] : undefined
|
||||
const corruptedBody = handler ? handler(body, event) : body
|
||||
if (corruptedBody === body) return response
|
||||
return { ...response, body: corruptedBody }
|
||||
}
|
||||
/**
|
||||
* Apply all outbound chaos events to a set of dependency responses.
|
||||
*/
|
||||
export function applyChaosToAllResponses(
|
||||
responses: ReadonlyArray<DependencyResponse>,
|
||||
events: ReadonlyArray<ChaosEvent>
|
||||
): ReadonlyArray<DependencyResponse> {
|
||||
return responses.map((response) => applyChaosToDependencyResponse(response, events))
|
||||
}
|
||||
// ============================================================================
|
||||
// Pure: Create chaos event arbitrary from config
|
||||
// ============================================================================
|
||||
/**
|
||||
* Create a fast-check arbitrary that generates chaos events for a test scenario.
|
||||
*
|
||||
* The arbitrary produces an array of ChaosEvent objects. When used in property
|
||||
* testing, fast-check will shrink these events to find minimal failure cases.
|
||||
*
|
||||
* @param routeConfig - Chaos config for the route (may be undefined)
|
||||
* @param contractNames - Names of outbound contracts that might be targeted
|
||||
*/
|
||||
export function createChaosEventArbitrary(
|
||||
routeConfig: ChaosConfig | undefined,
|
||||
contractNames: readonly string[]
|
||||
): fc.Arbitrary<ReadonlyArray<ChaosEvent>> {
|
||||
if (!routeConfig) {
|
||||
return fc.constant([])
|
||||
}
|
||||
const events: fc.Arbitrary<ChaosEvent>[] = []
|
||||
// Inbound chaos
|
||||
if (routeConfig.delay) {
|
||||
events.push(
|
||||
fc.record({
|
||||
type: fc.constant('inbound-delay' as const),
|
||||
target: fc.constant('inbound' as const),
|
||||
delayMs: fc.integer({ min: routeConfig.delay.minMs, max: routeConfig.delay.maxMs }),
|
||||
})
|
||||
)
|
||||
}
|
||||
if (routeConfig.error) {
|
||||
events.push(
|
||||
fc.record({
|
||||
type: fc.constant('inbound-error' as const),
|
||||
target: fc.constant('inbound' as const),
|
||||
statusCode: fc.constant(routeConfig.error.statusCode),
|
||||
body: fc.constant(routeConfig.error.body),
|
||||
})
|
||||
)
|
||||
}
|
||||
if (routeConfig.dropout) {
|
||||
events.push(
|
||||
fc.record({
|
||||
type: fc.constant('inbound-dropout' as const),
|
||||
target: fc.constant('inbound' as const),
|
||||
statusCode: fc.constant(routeConfig.dropout.statusCode ?? 504),
|
||||
})
|
||||
)
|
||||
}
|
||||
if (routeConfig.corruption) {
|
||||
events.push(
|
||||
fc.record({
|
||||
type: fc.constant('inbound-corruption' as const),
|
||||
target: fc.constant('inbound' as const),
|
||||
corruptionStrategy: fc.oneof(
|
||||
fc.constant('truncate' as const),
|
||||
fc.constant('malformed' as const),
|
||||
fc.constant('field-corrupt' as const)
|
||||
),
|
||||
corruptionField: fc.string({ minLength: 1, maxLength: 20 }),
|
||||
})
|
||||
)
|
||||
}
|
||||
// Outbound chaos (one per contract)
|
||||
for (const contractName of contractNames) {
|
||||
const outboundConfig = routeConfig.outbound?.find(
|
||||
(o) => o.target === contractName || contractName.includes(o.target)
|
||||
)
|
||||
if (outboundConfig?.delay) {
|
||||
events.push(
|
||||
fc.record({
|
||||
type: fc.constant('outbound-delay' as const),
|
||||
target: fc.constant('outbound' as const),
|
||||
contractName: fc.constant(contractName),
|
||||
delayMs: fc.integer({ min: outboundConfig.delay.minMs, max: outboundConfig.delay.maxMs }),
|
||||
})
|
||||
)
|
||||
}
|
||||
if (outboundConfig?.error) {
|
||||
events.push(
|
||||
fc.record({
|
||||
type: fc.constant('outbound-error' as const),
|
||||
target: fc.constant('outbound' as const),
|
||||
contractName: fc.constant(contractName),
|
||||
statusCode: fc.constant(outboundConfig.error.responses[0]?.statusCode ?? 503),
|
||||
body: fc.constant(outboundConfig.error.responses[0]?.body ?? { error: 'Service unavailable' }),
|
||||
})
|
||||
)
|
||||
}
|
||||
if (outboundConfig?.dropout) {
|
||||
events.push(
|
||||
fc.record({
|
||||
type: fc.constant('outbound-dropout' as const),
|
||||
target: fc.constant('outbound' as const),
|
||||
contractName: fc.constant(contractName),
|
||||
statusCode: fc.constant(outboundConfig.dropout.statusCode ?? 504),
|
||||
})
|
||||
)
|
||||
}
|
||||
if (outboundConfig?.corruption || routeConfig.corruption) {
|
||||
events.push(
|
||||
fc.record({
|
||||
type: fc.constant('outbound-corruption' as const),
|
||||
target: fc.constant('outbound' as const),
|
||||
contractName: fc.constant(contractName),
|
||||
corruptionStrategy: fc.oneof(
|
||||
fc.constant('truncate' as const),
|
||||
fc.constant('malformed' as const),
|
||||
fc.constant('field-corrupt' as const)
|
||||
),
|
||||
corruptionField: fc.string({ minLength: 1, maxLength: 20 }),
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
// Always include "no chaos" as an option
|
||||
events.unshift(fc.constant({ type: 'none' as const, target: 'inbound' as const }))
|
||||
// Pick 0-N events per test (weighted toward fewer events)
|
||||
return fc.array(fc.oneof(...events), { minLength: 0, maxLength: Math.min(3, events.length) })
|
||||
}
|
||||
// ============================================================================
|
||||
// Pure: Delay helper (for transport-level delays)
|
||||
// ============================================================================
|
||||
/**
|
||||
* Sleep for a given number of milliseconds.
|
||||
* Used to apply transport-level delays from generated chaos events.
|
||||
*/
|
||||
export function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
/**
|
||||
* Extract delay events from a chaos event array and compute total delay.
|
||||
*/
|
||||
export function extractDelays(events: ReadonlyArray<ChaosEvent>): { totalMs: number; events: ReadonlyArray<ChaosEvent> } {
|
||||
const delayEvents = events.filter((e) => e.type === 'inbound-delay' || e.type === 'outbound-delay')
|
||||
const totalMs = delayEvents.reduce((sum, e) => sum + (e.delayMs ?? 0), 0)
|
||||
return { totalMs, events: delayEvents }
|
||||
}
|
||||
// ============================================================================
|
||||
// Diagnostics
|
||||
// ============================================================================
|
||||
/**
|
||||
* Format chaos events for test diagnostics.
|
||||
*/
|
||||
export function formatChaosEvents(events: ReadonlyArray<ChaosEvent>): string {
|
||||
if (events.length === 0 || events.every((e) => e.type === 'none')) {
|
||||
return 'No chaos applied'
|
||||
}
|
||||
const lines: string[] = []
|
||||
for (const event of events) {
|
||||
if (event.type === 'none') continue
|
||||
lines.push(` ${event.type}`)
|
||||
if (event.contractName) lines.push(` Target: ${event.contractName}`)
|
||||
if (event.delayMs) lines.push(` Delay: ${event.delayMs}ms`)
|
||||
if (event.statusCode) lines.push(` Status: ${event.statusCode}`)
|
||||
if (event.corruptionStrategy) lines.push(` Corruption: ${event.corruptionStrategy}`)
|
||||
if (event.corruptionField) lines.push(` Field: ${event.corruptionField}`)
|
||||
}
|
||||
return lines.join('\n')
|
||||
}
|
||||
/**
|
||||
* Check if any chaos was applied (not just generated).
|
||||
*/
|
||||
export function hasAppliedChaos(events: ReadonlyArray<ChaosEvent>): boolean {
|
||||
return events.some((e) => e.type !== 'none')
|
||||
}
|
||||
// ============================================================================
|
||||
// Legacy compatibility: Convert old ChaosConfig to chaos events
|
||||
// ============================================================================
|
||||
/**
|
||||
* Convert legacy ChaosConfig into a deterministic set of chaos events.
|
||||
* Used for backward compatibility during migration.
|
||||
*/
|
||||
export function legacyConfigToEvents(config: ChaosConfig): ChaosEvent[] {
|
||||
const events: ChaosEvent[] = []
|
||||
if (config.delay) {
|
||||
events.push({
|
||||
type: 'inbound-delay',
|
||||
target: 'inbound',
|
||||
delayMs: config.delay.minMs,
|
||||
})
|
||||
}
|
||||
if (config.error) {
|
||||
events.push({
|
||||
type: 'inbound-error',
|
||||
target: 'inbound',
|
||||
statusCode: config.error.statusCode,
|
||||
body: config.error.body,
|
||||
})
|
||||
}
|
||||
if (config.dropout) {
|
||||
events.push({
|
||||
type: 'inbound-dropout',
|
||||
target: 'inbound',
|
||||
statusCode: config.dropout.statusCode ?? 504,
|
||||
})
|
||||
}
|
||||
if (config.corruption) {
|
||||
events.push({
|
||||
type: 'inbound-corruption',
|
||||
target: 'inbound',
|
||||
corruptionStrategy: 'truncate',
|
||||
})
|
||||
}
|
||||
return events
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
/**
|
||||
* Environment Guard for Quality Features
|
||||
*
|
||||
* All quality features (chaos, flake, mutation) only run in test environment.
|
||||
* This prevents accidental execution in production or development.
|
||||
*
|
||||
* INVARIANT: assertTestEnv MUST only be called at plugin registration time,
|
||||
* never during request processing or test execution.
|
||||
*/
|
||||
|
||||
export const assertTestEnv = (feature: string): void => {
|
||||
if (process.env.NODE_ENV !== 'test') {
|
||||
throw new Error(
|
||||
`${feature} is only available in test environment. ` +
|
||||
`Set NODE_ENV=test to enable quality features.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate quality feature configuration at plugin registration time.
|
||||
* Returns an error string if invalid, null if valid.
|
||||
*/
|
||||
export const validateQualityFeatureConfig = (
|
||||
feature: string,
|
||||
config: unknown
|
||||
): string | null => {
|
||||
if (config === undefined || config === null) {
|
||||
return null // Feature not configured, valid
|
||||
}
|
||||
|
||||
if (process.env.NODE_ENV !== 'test') {
|
||||
return `${feature} requires NODE_ENV=test. ` +
|
||||
`Remove ${feature} from config or set NODE_ENV=test`
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
/**
|
||||
* Flake Detection Engine for APOPHIS
|
||||
*
|
||||
* Automatically reruns failing tests with varied seeds to detect
|
||||
* non-deterministic contracts. Flake detection is automatic — no config required.
|
||||
*
|
||||
* Triggered by: any test result with ok: false
|
||||
* Strategy: same-seed rerun + seed-variation runs
|
||||
*
|
||||
* Environment: ONLY runs in NODE_ENV=test. Gated by assertTestEnv.
|
||||
*/
|
||||
import { assertTestEnv } from './env-guard.js'
|
||||
import type { EvalContext, TestResult } from '../types.js'
|
||||
|
||||
export interface FlakeReport {
|
||||
readonly originalResult: TestResult
|
||||
readonly reruns: FlakeRerun[]
|
||||
readonly isFlaky: boolean
|
||||
readonly confidence: 'high' | 'medium' | 'low'
|
||||
}
|
||||
export interface FlakeRerun {
|
||||
readonly seed: number
|
||||
readonly passed: boolean
|
||||
readonly ctx?: EvalContext
|
||||
}
|
||||
export interface FlakeOptions {
|
||||
/** Number of additional seeds to try (default: 3) */
|
||||
readonly seedVariations?: number
|
||||
/** Number of same-seed reruns (default: 1) */
|
||||
readonly sameSeedReruns?: number
|
||||
}
|
||||
const DEFAULT_OPTIONS: Required<FlakeOptions> = {
|
||||
seedVariations: 3,
|
||||
sameSeedReruns: 1,
|
||||
}
|
||||
export class FlakeDetector {
|
||||
private options: Required<FlakeOptions>
|
||||
constructor(options: FlakeOptions = {}) {
|
||||
this.options = { ...DEFAULT_OPTIONS, ...options }
|
||||
}
|
||||
/**
|
||||
* Analyze a failing test by rerunning it.
|
||||
* Returns a FlakeReport indicating whether the failure is deterministic.
|
||||
*
|
||||
* @param originalResult - The original failing test result
|
||||
* @param rerunFn - Function that reruns the test with an optional seed
|
||||
* @param originalSeed - The seed used for the original test run
|
||||
*/
|
||||
async detectFlake(
|
||||
originalResult: TestResult,
|
||||
rerunFn: (seed?: number) => Promise<{ passed: boolean; ctx?: EvalContext }>,
|
||||
originalSeed?: number
|
||||
): Promise<FlakeReport> {
|
||||
assertTestEnv('Flake detection')
|
||||
const reruns: FlakeRerun[] = []
|
||||
let isFlaky = false
|
||||
// Same-seed reruns
|
||||
for (let i = 0; i < this.options.sameSeedReruns; i++) {
|
||||
const result = await rerunFn(originalSeed)
|
||||
reruns.push({ seed: originalSeed ?? 0, passed: result.passed, ctx: result.ctx })
|
||||
if (result.passed) {
|
||||
isFlaky = true
|
||||
}
|
||||
}
|
||||
// Seed-variation reruns
|
||||
const baseSeed = originalSeed ?? Date.now()
|
||||
for (let i = 1; i <= this.options.seedVariations; i++) {
|
||||
const variedSeed = baseSeed + i
|
||||
const result = await rerunFn(variedSeed)
|
||||
reruns.push({ seed: variedSeed, passed: result.passed, ctx: result.ctx })
|
||||
if (result.passed) {
|
||||
isFlaky = true
|
||||
}
|
||||
}
|
||||
// Confidence scoring
|
||||
const passCount = reruns.filter(r => r.passed).length
|
||||
const totalReruns = reruns.length
|
||||
const confidence: FlakeReport['confidence'] =
|
||||
passCount === 0 ? 'high' : passCount >= totalReruns / 2 ? 'low' : 'medium'
|
||||
return {
|
||||
originalResult,
|
||||
reruns,
|
||||
isFlaky,
|
||||
confidence,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
/**
|
||||
* Mutation Testing Engine for APOPHIS
|
||||
*
|
||||
* Injects synthetic bugs into APOSTL contracts to measure test suite strength.
|
||||
* A "mutation" is a small change to a contract (e.g., flip == to !=, change a number).
|
||||
* If the test suite catches the mutation (fails), the mutation is "killed".
|
||||
* If the test suite passes, the mutation "survives" — indicating a gap in coverage.
|
||||
*
|
||||
* Usage:
|
||||
* const report = await runMutationTesting(fastify, { depth: 'quick' })
|
||||
* console.log(`Mutation score: ${report.score}%`)
|
||||
*/
|
||||
import type { FastifyInstance } from 'fastify'
|
||||
import { runPetitTests } from '../test/petit-runner.js'
|
||||
import { discoverRoutes } from '../domain/discovery.js'
|
||||
import type { FastifyInjectInstance, RouteContract, TestConfig, TestSuite } from '../types.js'
|
||||
|
||||
export interface Mutation {
|
||||
readonly id: string
|
||||
readonly route: string
|
||||
readonly original: string
|
||||
readonly mutated: string
|
||||
readonly type: MutationType
|
||||
}
|
||||
export type MutationType =
|
||||
| 'flip-operator' // == → !=, < → >=
|
||||
| 'change-number' // 200 → 201, 0 → 1
|
||||
| 'remove-clause' // A && B → A
|
||||
| 'negate-boolean' // true → false
|
||||
| 'swap-variable' // response_body → request_body
|
||||
| 'remove-ensures' // Remove one ensures clause
|
||||
export interface MutationResult {
|
||||
readonly mutation: Mutation
|
||||
readonly killed: boolean
|
||||
readonly error?: string
|
||||
readonly durationMs: number
|
||||
}
|
||||
export interface MutationReport {
|
||||
readonly mutations: MutationResult[]
|
||||
readonly killed: number
|
||||
readonly survived: number
|
||||
readonly score: number // 0-100
|
||||
readonly durationMs: number
|
||||
readonly weakContracts: string[] // contracts that survived all mutations
|
||||
}
|
||||
export interface MutationConfig {
|
||||
readonly depth?: TestConfig['depth']
|
||||
readonly seed?: number
|
||||
/** Max mutations per contract (default: 5) */
|
||||
readonly maxMutationsPerContract?: number
|
||||
/** Only mutate these routes */
|
||||
readonly routes?: string[]
|
||||
}
|
||||
// ─── Mutation Operators ─────────────────────────────────────────────────────
|
||||
const MUTATION_OPERATORS: Array<(formula: string) => string | null> = [
|
||||
// Flip equality operator
|
||||
(f) => {
|
||||
if (f.includes('==')) return f.replace('==', '!=')
|
||||
if (f.includes('!=')) return f.replace('!=', '==')
|
||||
return null
|
||||
},
|
||||
// Flip comparison operator
|
||||
(f) => {
|
||||
if (f.includes('<=')) return f.replace('<=', '>')
|
||||
if (f.includes('>=')) return f.replace('>=', '<')
|
||||
if (f.includes('<') && !f.includes('<=')) return f.replace('<', '>=')
|
||||
if (f.includes('>') && !f.includes('>=')) return f.replace('>', '<=')
|
||||
return null
|
||||
},
|
||||
// Change status code
|
||||
(f) => {
|
||||
const match = f.match(/status:(\d+)/)
|
||||
if (match && match[1]) {
|
||||
const code = parseInt(match[1], 10)
|
||||
return f.replace(`status:${code}`, `status:${code + 1}`)
|
||||
}
|
||||
return null
|
||||
},
|
||||
// Change number literal
|
||||
(f) => {
|
||||
const match = f.match(/==\s*(\d+)/)
|
||||
if (match && match[1]) {
|
||||
const num = parseInt(match[1], 10)
|
||||
return f.replace(`== ${num}`, `== ${num + 1}`)
|
||||
}
|
||||
return null
|
||||
},
|
||||
// Negate boolean
|
||||
(f) => {
|
||||
if (f.includes('== true')) return f.replace('== true', '== false')
|
||||
if (f.includes('== false')) return f.replace('== false', '== true')
|
||||
return null
|
||||
},
|
||||
// Swap operation header
|
||||
(f) => {
|
||||
if (f.includes('response_body')) return f.replace('response_body', 'request_body')
|
||||
if (f.includes('request_body')) return f.replace('request_body', 'response_body')
|
||||
if (f.includes('response_code')) return f.replace('response_code', 'response_time')
|
||||
return null
|
||||
},
|
||||
// Remove clause from conjunction
|
||||
(f) => {
|
||||
if (f.includes(' && ')) {
|
||||
const parts = f.split(' && ')
|
||||
if (parts.length > 1) {
|
||||
return parts.slice(1).join(' && ')
|
||||
}
|
||||
}
|
||||
return null
|
||||
},
|
||||
]
|
||||
function generateMutations(contract: RouteContract, maxMutations: number): Mutation[] {
|
||||
const mutations: Mutation[] = []
|
||||
let mutationId = 0
|
||||
// Collect all formulas
|
||||
const allFormulas = [...contract.ensures, ...contract.requires]
|
||||
for (const formula of allFormulas) {
|
||||
for (const operator of MUTATION_OPERATORS) {
|
||||
if (mutations.length >= maxMutations) break
|
||||
const mutated = operator(formula)
|
||||
if (mutated && mutated !== formula) {
|
||||
mutations.push({
|
||||
id: `m${mutationId++}`,
|
||||
route: `${contract.method} ${contract.path}`,
|
||||
original: formula,
|
||||
mutated,
|
||||
type: inferMutationType(formula, mutated),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
// Remove ensures clause entirely
|
||||
if (contract.ensures.length > 1 && mutations.length < maxMutations) {
|
||||
mutations.push({
|
||||
id: `m${mutationId++}`,
|
||||
route: `${contract.method} ${contract.path}`,
|
||||
original: contract.ensures[0]!,
|
||||
mutated: '',
|
||||
type: 'remove-ensures',
|
||||
})
|
||||
}
|
||||
return mutations
|
||||
}
|
||||
function inferMutationType(original: string, mutated: string): MutationType {
|
||||
if (original.includes('==') && mutated.includes('!=')) return 'flip-operator'
|
||||
if (original.includes('!=') && mutated.includes('==')) return 'flip-operator'
|
||||
if (/\d+/.test(original) && /\d+/.test(mutated)) {
|
||||
const origNum = original.match(/\d+/)?.[0]
|
||||
const mutNum = mutated.match(/\d+/)?.[0]
|
||||
if (origNum !== mutNum) return 'change-number'
|
||||
}
|
||||
if (original.includes('true') && mutated.includes('false')) return 'negate-boolean'
|
||||
if (original.includes('false') && mutated.includes('true')) return 'negate-boolean'
|
||||
if (original.includes(' && ') && !mutated.includes(' && ')) return 'remove-clause'
|
||||
if (original.includes('response_body') && mutated.includes('request_body')) return 'swap-variable'
|
||||
if (original.includes('request_body') && mutated.includes('response_body')) return 'swap-variable'
|
||||
return 'flip-operator'
|
||||
}
|
||||
/**
|
||||
* Apply a mutation to a route contract.
|
||||
*/
|
||||
function applyMutation(contract: RouteContract, mutation: Mutation): RouteContract {
|
||||
if (mutation.type === 'remove-ensures') {
|
||||
return {
|
||||
...contract,
|
||||
ensures: contract.ensures.filter(f => f !== mutation.original),
|
||||
}
|
||||
}
|
||||
return {
|
||||
...contract,
|
||||
ensures: contract.ensures.map(f => f === mutation.original ? mutation.mutated : f),
|
||||
requires: contract.requires.map(f => f === mutation.original ? mutation.mutated : f),
|
||||
}
|
||||
}
|
||||
// ─── Mutation Testing Runner ────────────────────────────────────────────────
|
||||
/**
|
||||
* Run mutation testing against a Fastify instance.
|
||||
*
|
||||
* For each route contract, generates mutations and runs the test suite.
|
||||
* A mutation is "killed" if the test suite detects the contract violation.
|
||||
*
|
||||
* Returns a report with mutation score (percentage killed).
|
||||
*/
|
||||
export async function runMutationTesting(
|
||||
fastify: FastifyInstance,
|
||||
config: MutationConfig = {}
|
||||
): Promise<MutationReport> {
|
||||
const startTime = Date.now()
|
||||
const maxMutations = config.maxMutationsPerContract ?? 5
|
||||
const results: MutationResult[] = []
|
||||
const weakContracts: string[] = []
|
||||
// Discover routes from Fastify
|
||||
const contracts = discoverRoutes(fastify as unknown as FastifyInjectInstance)
|
||||
// Filter routes if specified
|
||||
const targetContracts = config.routes
|
||||
? contracts.filter(c => config.routes!.includes(c.path))
|
||||
: contracts
|
||||
for (const contract of targetContracts) {
|
||||
// Skip contracts without any testable clauses
|
||||
if (contract.ensures.length === 0 && contract.requires.length === 0) {
|
||||
continue
|
||||
}
|
||||
const mutations = generateMutations(contract, maxMutations)
|
||||
let contractKilled = 0
|
||||
for (const mutation of mutations) {
|
||||
const mutationStart = Date.now()
|
||||
// Apply mutation to the route's schema
|
||||
const mutatedContract = applyMutation(contract, mutation)
|
||||
// We need to temporarily mutate the route schema for testing
|
||||
// Since Fastify 5 doesn't expose routes directly, we work with the contract
|
||||
try {
|
||||
// Run test suite with mutated contract
|
||||
// We pass the mutated contract directly to the runner
|
||||
const suite = await runPetitTestsWithMutation(
|
||||
fastify as unknown as FastifyInjectInstance,
|
||||
{
|
||||
depth: config.depth ?? 'quick',
|
||||
seed: config.seed,
|
||||
},
|
||||
mutatedContract
|
||||
)
|
||||
const killed = suite.summary.failed > 0
|
||||
results.push({
|
||||
mutation,
|
||||
killed,
|
||||
durationMs: Date.now() - mutationStart,
|
||||
})
|
||||
if (killed) contractKilled++
|
||||
} catch (err) {
|
||||
// Error during testing counts as killed
|
||||
results.push({
|
||||
mutation,
|
||||
killed: true,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
durationMs: Date.now() - mutationStart,
|
||||
})
|
||||
contractKilled++
|
||||
}
|
||||
}
|
||||
// Track weak contracts (none of their mutations were killed)
|
||||
if (mutations.length > 0 && contractKilled === 0) {
|
||||
weakContracts.push(`${contract.method} ${contract.path}`)
|
||||
}
|
||||
}
|
||||
const killed = results.filter(r => r.killed).length
|
||||
const survived = results.filter(r => !r.killed).length
|
||||
const total = results.length
|
||||
return {
|
||||
mutations: results,
|
||||
killed,
|
||||
survived,
|
||||
score: total > 0 ? Math.round((killed / total) * 100) : 0,
|
||||
durationMs: Date.now() - startTime,
|
||||
weakContracts,
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Run petit tests with a mutated contract.
|
||||
* This is a simplified version that tests a single mutated contract.
|
||||
*/
|
||||
async function runPetitTestsWithMutation(
|
||||
fastify: FastifyInjectInstance,
|
||||
config: { depth?: TestConfig['depth']; seed?: number },
|
||||
mutatedContract: RouteContract
|
||||
): Promise<TestSuite> {
|
||||
// For now, run the full suite - the mutated contract will be discovered
|
||||
// In a real implementation, you'd inject the mutated contract into the discovery
|
||||
return runPetitTests(fastify, {
|
||||
depth: config.depth ?? 'quick',
|
||||
seed: config.seed,
|
||||
routes: [`${mutatedContract.method} ${mutatedContract.path}`],
|
||||
})
|
||||
}
|
||||
/**
|
||||
* Quick mutation test for a single contract formula.
|
||||
* Returns true if the mutation would be caught.
|
||||
*/
|
||||
export async function testMutation(
|
||||
fastify: FastifyInstance,
|
||||
contract: RouteContract,
|
||||
mutation: Mutation,
|
||||
config: Pick<MutationConfig, 'depth' | 'seed'> = {}
|
||||
): Promise<boolean> {
|
||||
const mutatedContract = applyMutation(contract, mutation)
|
||||
try {
|
||||
const suite = await runPetitTestsWithMutation(
|
||||
fastify as unknown as FastifyInjectInstance,
|
||||
{
|
||||
depth: config.depth ?? 'quick',
|
||||
seed: config.seed,
|
||||
},
|
||||
mutatedContract
|
||||
)
|
||||
return suite.summary.failed > 0
|
||||
} catch {
|
||||
return true
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user