(mess) Stuffing commit.
This commit is contained in:
John Dvorak
@@ -0,0 +1,425 @@
|
||||
/**
|
||||
* Quality Engine Tests — Flake Detection, Mutation Testing, Chaos
|
||||
*/
|
||||
import { test } from 'node:test'
|
||||
import assert from 'node:assert'
|
||||
import Fastify from 'fastify'
|
||||
import swagger from '@fastify/swagger'
|
||||
import apophisPlugin from '../index.js'
|
||||
import { FlakeDetector } from '../quality/flake.js'
|
||||
import { runMutationTesting, testMutation, type Mutation } from '../quality/mutation.js'
|
||||
import * as fc from 'fast-check'
|
||||
import {
|
||||
applyChaosToExecution,
|
||||
applyChaosToAllResponses,
|
||||
applyChaosToDependencyResponse,
|
||||
createChaosEventArbitrary,
|
||||
extractDelays,
|
||||
sleep,
|
||||
hasAppliedChaos,
|
||||
formatChaosEvents,
|
||||
type ChaosEvent,
|
||||
} from '../quality/chaos-v3.js'
|
||||
import type { EvalContext, TestResult, RouteContract } from '../types.js'
|
||||
|
||||
// Quality engines require test environment
|
||||
process.env.NODE_ENV = 'test'
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function makeCtx(overrides?: Partial<EvalContext['response']>): EvalContext {
|
||||
return {
|
||||
request: { body: {}, headers: {}, query: {}, params: {} },
|
||||
response: { body: { id: '1' }, headers: {}, statusCode: 200, ...overrides },
|
||||
} as EvalContext
|
||||
}
|
||||
|
||||
function makeTestResult(ok: boolean): TestResult {
|
||||
return {
|
||||
ok,
|
||||
name: 'test',
|
||||
id: 1,
|
||||
diagnostics: ok ? {} : { error: 'failed' },
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Chaos Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('applyChaosToExecution: no chaos when events are empty', () => {
|
||||
const ctx = makeCtx()
|
||||
const result = applyChaosToExecution(ctx, [])
|
||||
assert.strictEqual(result.applied, false)
|
||||
assert.strictEqual(result.ctx.response.statusCode, 200)
|
||||
})
|
||||
|
||||
test('applyChaosToExecution: inbound error changes status code', () => {
|
||||
const ctx = makeCtx()
|
||||
const result = applyChaosToExecution(ctx, [
|
||||
{ type: 'inbound-error', target: 'inbound', statusCode: 503 },
|
||||
])
|
||||
assert.strictEqual(result.applied, true)
|
||||
assert.strictEqual(result.ctx.response.statusCode, 503)
|
||||
})
|
||||
|
||||
test('applyChaosToExecution: inbound dropout simulates gateway timeout', () => {
|
||||
const ctx = makeCtx()
|
||||
const result = applyChaosToExecution(ctx, [
|
||||
{ type: 'inbound-dropout', target: 'inbound' },
|
||||
])
|
||||
assert.strictEqual(result.ctx.response.statusCode, 504)
|
||||
})
|
||||
|
||||
test('applyChaosToExecution: inbound corruption truncates response body', () => {
|
||||
const ctx = makeCtx({ body: { id: '1', name: 'Alice', email: 'a@b.com' } })
|
||||
const result = applyChaosToExecution(ctx, [
|
||||
{ type: 'inbound-corruption', target: 'inbound', corruptionStrategy: 'truncate' },
|
||||
])
|
||||
const body = result.ctx.response.body as Record<string, unknown>
|
||||
assert.ok(Object.keys(body).length <= 2)
|
||||
})
|
||||
|
||||
test('applyChaosToExecution: inbound corruption with field-corrupt', () => {
|
||||
const ctx = makeCtx({ body: { id: '1', name: 'Alice' } })
|
||||
const result = applyChaosToExecution(ctx, [
|
||||
{ type: 'inbound-corruption', target: 'inbound', corruptionStrategy: 'field-corrupt', corruptionField: 'name' },
|
||||
])
|
||||
const body = result.ctx.response.body as Record<string, unknown>
|
||||
assert.strictEqual(body.name, null)
|
||||
})
|
||||
|
||||
test('applyChaosToExecution: inbound corruption malformed', () => {
|
||||
const ctx = makeCtx({ body: { id: '1' } })
|
||||
const result = applyChaosToExecution(ctx, [
|
||||
{ type: 'inbound-corruption', target: 'inbound', corruptionStrategy: 'malformed' },
|
||||
])
|
||||
assert.strictEqual(result.ctx.response.body, '{"broken":')
|
||||
})
|
||||
|
||||
test('applyChaosToExecution: ignores none events', () => {
|
||||
const ctx = makeCtx()
|
||||
const result = applyChaosToExecution(ctx, [
|
||||
{ type: 'none', target: 'inbound' },
|
||||
])
|
||||
assert.strictEqual(result.applied, false)
|
||||
})
|
||||
|
||||
test('applyChaosToExecution: only applies first non-delay event', () => {
|
||||
const ctx = makeCtx()
|
||||
const result = applyChaosToExecution(ctx, [
|
||||
{ type: 'inbound-error', target: 'inbound', statusCode: 503 },
|
||||
{ type: 'inbound-dropout', target: 'inbound' },
|
||||
])
|
||||
assert.strictEqual(result.ctx.response.statusCode, 503)
|
||||
})
|
||||
|
||||
test('applyChaosToDependencyResponse: outbound error changes status', () => {
|
||||
const response = { contractName: 'api', statusCode: 200, body: { ok: true } }
|
||||
const result = applyChaosToDependencyResponse(response, [
|
||||
{ type: 'outbound-error', target: 'outbound', contractName: 'api', statusCode: 503 },
|
||||
])
|
||||
assert.strictEqual(result.statusCode, 503)
|
||||
})
|
||||
|
||||
test('applyChaosToDependencyResponse: ignores events for other contracts', () => {
|
||||
const response = { contractName: 'api', statusCode: 200, body: {} }
|
||||
const result = applyChaosToDependencyResponse(response, [
|
||||
{ type: 'outbound-error', target: 'outbound', contractName: 'other', statusCode: 503 },
|
||||
])
|
||||
assert.strictEqual(result.statusCode, 200)
|
||||
})
|
||||
|
||||
test('applyChaosToAllResponses: applies chaos to multiple responses', () => {
|
||||
const responses = [
|
||||
{ contractName: 'a', statusCode: 200, body: {} },
|
||||
{ contractName: 'b', statusCode: 200, body: {} },
|
||||
]
|
||||
const events = [
|
||||
{ type: 'outbound-error', target: 'outbound', contractName: 'a', statusCode: 503 },
|
||||
{ type: 'outbound-error', target: 'outbound', contractName: 'b', statusCode: 504 },
|
||||
] as ChaosEvent[]
|
||||
const result = applyChaosToAllResponses(responses, events)
|
||||
assert.strictEqual(result[0]!.statusCode, 503)
|
||||
assert.strictEqual(result[1]!.statusCode, 504)
|
||||
})
|
||||
|
||||
test('extractDelays: computes total delay', () => {
|
||||
const events = [
|
||||
{ type: 'inbound-delay', target: 'inbound', delayMs: 100 },
|
||||
{ type: 'outbound-delay', target: 'outbound', delayMs: 200 },
|
||||
] as ChaosEvent[]
|
||||
const result = extractDelays(events)
|
||||
assert.strictEqual(result.totalMs, 300)
|
||||
assert.strictEqual(result.events.length, 2)
|
||||
})
|
||||
|
||||
test('sleep: resolves after specified ms', async () => {
|
||||
const start = Date.now()
|
||||
await sleep(10)
|
||||
const elapsed = Date.now() - start
|
||||
assert.ok(elapsed >= 8 && elapsed < 100)
|
||||
})
|
||||
|
||||
test('hasAppliedChaos: detects applied chaos', () => {
|
||||
assert.strictEqual(hasAppliedChaos([{ type: 'none', target: 'inbound' }]), false)
|
||||
assert.strictEqual(hasAppliedChaos([{ type: 'inbound-error', target: 'inbound' }]), true)
|
||||
})
|
||||
|
||||
test('formatChaosEvents: formats events for diagnostics', () => {
|
||||
const events = [
|
||||
{ type: 'inbound-error', target: 'inbound', statusCode: 503 },
|
||||
] as ChaosEvent[]
|
||||
const formatted = formatChaosEvents(events)
|
||||
assert.ok(formatted.includes('inbound-error'))
|
||||
assert.ok(formatted.includes('503'))
|
||||
})
|
||||
|
||||
test('formatChaosEvents: returns "No chaos applied" for empty events', () => {
|
||||
assert.strictEqual(formatChaosEvents([]), 'No chaos applied')
|
||||
assert.strictEqual(formatChaosEvents([{ type: 'none', target: 'inbound' }]), 'No chaos applied')
|
||||
})
|
||||
|
||||
test('createChaosEventArbitrary: generates deterministic events with seed', () => {
|
||||
const arb = createChaosEventArbitrary(
|
||||
{ probability: 1, delay: { probability: 1, minMs: 100, maxMs: 200 } },
|
||||
[]
|
||||
)
|
||||
const sample1 = fc.sample(arb, { seed: 42, numRuns: 5 })
|
||||
const sample2 = fc.sample(arb, { seed: 42, numRuns: 5 })
|
||||
assert.deepStrictEqual(sample1, sample2)
|
||||
})
|
||||
|
||||
test('createChaosEventArbitrary: returns empty array when no config', () => {
|
||||
const arb = createChaosEventArbitrary(undefined, [])
|
||||
const sample = fc.sample(arb, { numRuns: 10 })
|
||||
assert.ok(sample.every((events) => events.length === 0 || events.every((e) => e.type === 'none')))
|
||||
})
|
||||
|
||||
test('createChaosEventArbitrary: generates outbound events for contracts', () => {
|
||||
const arb = createChaosEventArbitrary(
|
||||
{
|
||||
probability: 1,
|
||||
error: { probability: 1, statusCode: 503 },
|
||||
outbound: [
|
||||
{
|
||||
target: 'api',
|
||||
error: {
|
||||
probability: 1,
|
||||
responses: [{ statusCode: 500 }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
['api']
|
||||
)
|
||||
const sample = fc.sample(arb, { numRuns: 20 })
|
||||
const hasOutbound = sample.some((events) =>
|
||||
events.some((e) => e.target === 'outbound')
|
||||
)
|
||||
assert.ok(hasOutbound, 'expected some outbound events')
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Flake Detection Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('FlakeDetector: deterministic failure returns high confidence', async () => {
|
||||
const detector = new FlakeDetector({ sameSeedReruns: 1, seedVariations: 2 })
|
||||
const report = await detector.detectFlake(
|
||||
makeTestResult(false),
|
||||
async () => ({ passed: false }),
|
||||
42
|
||||
)
|
||||
assert.strictEqual(report.isFlaky, false)
|
||||
assert.strictEqual(report.confidence, 'high')
|
||||
assert.strictEqual(report.reruns.length, 3)
|
||||
})
|
||||
|
||||
test('FlakeDetector: passing same-seed rerun flags flaky', async () => {
|
||||
const detector = new FlakeDetector({ sameSeedReruns: 1, seedVariations: 0 })
|
||||
let callCount = 0
|
||||
const report = await detector.detectFlake(
|
||||
makeTestResult(false),
|
||||
async () => {
|
||||
callCount++
|
||||
return { passed: callCount === 1 }
|
||||
},
|
||||
42
|
||||
)
|
||||
assert.strictEqual(report.isFlaky, true)
|
||||
assert.strictEqual(report.confidence, 'low')
|
||||
})
|
||||
|
||||
test('FlakeDetector: medium confidence when some pass', async () => {
|
||||
const detector = new FlakeDetector({ sameSeedReruns: 1, seedVariations: 2 })
|
||||
let callCount = 0
|
||||
const report = await detector.detectFlake(
|
||||
makeTestResult(false),
|
||||
async () => {
|
||||
callCount++
|
||||
return { passed: callCount % 2 === 0 }
|
||||
},
|
||||
42
|
||||
)
|
||||
assert.strictEqual(report.isFlaky, true)
|
||||
assert.strictEqual(report.confidence, 'medium')
|
||||
})
|
||||
|
||||
test('FlakeDetector: uses Date.now() when no seed provided', async () => {
|
||||
const detector = new FlakeDetector({ sameSeedReruns: 0, seedVariations: 1 })
|
||||
const report = await detector.detectFlake(
|
||||
makeTestResult(false),
|
||||
async () => ({ passed: false })
|
||||
)
|
||||
assert.strictEqual(report.reruns.length, 1)
|
||||
assert.ok(report.reruns[0]!.seed > 0)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Mutation Testing Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function makeContract(ensures: string[], requires: string[] = []): RouteContract {
|
||||
return {
|
||||
path: '/items',
|
||||
method: 'POST',
|
||||
category: 'constructor',
|
||||
requires,
|
||||
ensures,
|
||||
invariants: [],
|
||||
regexPatterns: {},
|
||||
validateRuntime: false,
|
||||
schema: {
|
||||
body: {
|
||||
type: 'object',
|
||||
properties: { name: { type: 'string' } },
|
||||
},
|
||||
response: {
|
||||
201: {
|
||||
type: 'object',
|
||||
properties: { id: { type: 'string' } },
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
test('testMutation: detects flipped operator mutation', async () => {
|
||||
const fastify = Fastify()
|
||||
await fastify.register(swagger)
|
||||
await fastify.register(apophisPlugin)
|
||||
fastify.post('/items', {
|
||||
schema: {
|
||||
body: { type: 'object', properties: { name: { type: 'string' } } },
|
||||
response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
|
||||
'x-ensures': ['response_code(this) == 201'],
|
||||
},
|
||||
}, async (request, reply) => {
|
||||
reply.status(201)
|
||||
return { id: '1' }
|
||||
})
|
||||
await fastify.ready()
|
||||
|
||||
const mutation: Mutation = {
|
||||
id: 'm1',
|
||||
route: 'POST /items',
|
||||
original: 'response_code(this) == 201',
|
||||
mutated: 'response_code(this) != 201',
|
||||
type: 'flip-operator',
|
||||
}
|
||||
|
||||
const killed = await testMutation(fastify, makeContract(['response_code(this) == 201']), mutation)
|
||||
assert.strictEqual(killed, true)
|
||||
})
|
||||
|
||||
test('runMutationTesting: produces report with score', async () => {
|
||||
const fastify = Fastify()
|
||||
await fastify.register(swagger)
|
||||
await fastify.register(apophisPlugin)
|
||||
fastify.post('/items', {
|
||||
schema: {
|
||||
body: { type: 'object', properties: { name: { type: 'string' } } },
|
||||
response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
|
||||
'x-ensures': ['response_code(this) == 201'],
|
||||
},
|
||||
}, async (request, reply) => {
|
||||
reply.status(201)
|
||||
return { id: '1' }
|
||||
})
|
||||
await fastify.ready()
|
||||
|
||||
const report = await runMutationTesting(fastify, { runs: 5, seed: 42, maxMutationsPerContract: 3 })
|
||||
assert.ok(typeof report.score === 'number')
|
||||
assert.ok(report.score >= 0 && report.score <= 100)
|
||||
assert.ok(report.mutations.length > 0)
|
||||
assert.ok(report.durationMs > 0)
|
||||
})
|
||||
|
||||
test('runMutationTesting: skips routes without contracts', async () => {
|
||||
const fastify = Fastify()
|
||||
await fastify.register(swagger)
|
||||
await fastify.register(apophisPlugin)
|
||||
fastify.get('/health', {}, async () => 'ok')
|
||||
await fastify.ready()
|
||||
|
||||
const report = await runMutationTesting(fastify, { runs: 5, seed: 42 })
|
||||
assert.strictEqual(report.mutations.length, 0)
|
||||
assert.strictEqual(report.score, 0)
|
||||
})
|
||||
|
||||
test('runMutationTesting: filters by route', async () => {
|
||||
const fastify = Fastify()
|
||||
await fastify.register(swagger)
|
||||
await fastify.register(apophisPlugin)
|
||||
fastify.post('/items', {
|
||||
schema: {
|
||||
body: { type: 'object', properties: { name: { type: 'string' } } },
|
||||
response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
|
||||
'x-ensures': ['response_code(this) == 201'],
|
||||
},
|
||||
}, async (request, reply) => {
|
||||
reply.status(201)
|
||||
return { id: '1' }
|
||||
})
|
||||
fastify.post('/other', {
|
||||
schema: {
|
||||
body: { type: 'object', properties: { name: { type: 'string' } } },
|
||||
response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
|
||||
'x-ensures': ['response_code(this) == 201'],
|
||||
},
|
||||
}, async (request, reply) => {
|
||||
reply.status(201)
|
||||
return { id: '2' }
|
||||
})
|
||||
await fastify.ready()
|
||||
|
||||
const report = await runMutationTesting(fastify, { runs: 5, seed: 42, routes: ['/items'] })
|
||||
const itemsMutations = report.mutations.filter((m) => m.mutation.route.includes('/items'))
|
||||
const otherMutations = report.mutations.filter((m) => m.mutation.route.includes('/other'))
|
||||
assert.ok(itemsMutations.length > 0)
|
||||
assert.strictEqual(otherMutations.length, 0)
|
||||
})
|
||||
|
||||
test('runMutationTesting: identifies weak contracts', async () => {
|
||||
const fastify = Fastify()
|
||||
await fastify.register(swagger)
|
||||
await fastify.register(apophisPlugin)
|
||||
fastify.post('/items', {
|
||||
schema: {
|
||||
body: { type: 'object', properties: { name: { type: 'string' } } },
|
||||
response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
|
||||
'x-ensures': ['response_code(this) == 201', 'response_body(this).id != null'],
|
||||
},
|
||||
}, async (request, reply) => {
|
||||
reply.status(201)
|
||||
return { id: '1' }
|
||||
})
|
||||
await fastify.ready()
|
||||
|
||||
const report = await runMutationTesting(fastify, { runs: 5, seed: 42 })
|
||||
// All mutations should be killed since the handler always returns 201 with id
|
||||
assert.strictEqual(report.weakContracts.length, 0)
|
||||
})
|
||||
Reference in New Issue
Block a user