John Dvorak
115d3465b1
- Remove unused generationProfile parameter from verify runner - Integrate PluginContractRegistry into petit-runner and stateful-runner - Add deterministic hashStringToSeed to doctor (replaces Math.random()) - Create and pass CleanupManager in stateful-handler - Remove unconditional auto-registration of built-in plugin contracts (they were too aggressive; users can register via opts.pluginContracts) - Build: clean | Tests: 849 pass, 0 fail
490 lines
13 KiB
TypeScript
490 lines
13 KiB
TypeScript
/**
|
|
* S4: Verify thread - Runner for deterministic contract verification
|
|
*
|
|
* Responsibilities:
|
|
* - Route discovery from Fastify app
|
|
* - Route filtering by patterns and git changes
|
|
* - Contract execution using existing plugin/evaluator code
|
|
* - Deterministic execution with seed
|
|
* - Result aggregation
|
|
*
|
|
* Architecture:
|
|
* - Pure execution functions that accept injected dependencies
|
|
* - Reuses existing APOPHIS plugin and formula code
|
|
* - No reimplementation of parser/evaluator
|
|
*/
|
|
|
|
import { discoverRoutes } from '../../../domain/discovery.js'
|
|
import { extractContract } from '../../../domain/contract.js'
|
|
import { executeHttp } from '../../../infrastructure/http-executor.js'
|
|
import { parse } from '../../../formula/parser.js'
|
|
import { evaluateAsync } from '../../../formula/evaluator.js'
|
|
import { createOperationResolver } from '../../../formula/runtime.js'
|
|
import type { EvalContext, RouteContract, FastifyInjectInstance } from '../../../types.js'
|
|
import type { RouteResult } from '../../core/types.js'
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Types
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface VerifyFailure {
|
|
route: string
|
|
contract: string
|
|
expected: string
|
|
observed: string
|
|
artifactPath?: string
|
|
}
|
|
|
|
export interface VerifyRunResult {
|
|
passed: boolean
|
|
total: number
|
|
passedCount: number
|
|
failed: number
|
|
failures: VerifyFailure[]
|
|
durationMs: number
|
|
noRoutesMatched: boolean
|
|
noContractsFound: boolean
|
|
notGitRepo?: boolean
|
|
noRelevantChanges?: boolean
|
|
availableRoutes?: string[]
|
|
artifactPaths: string[]
|
|
}
|
|
|
|
export interface VerifyRunnerDeps {
|
|
fastify: FastifyInjectInstance
|
|
seed: number
|
|
timeout?: number
|
|
routeFilters?: string[]
|
|
changed?: boolean
|
|
profileRoutes?: string[]
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Route discovery
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Discover routes from a Fastify instance.
|
|
* Uses the existing discovery module.
|
|
*/
|
|
export async function discoverAppRoutes(fastify: FastifyInjectInstance): Promise<RouteContract[]> {
|
|
return discoverRoutes(fastify)
|
|
}
|
|
|
|
/**
|
|
* Check if specific routes exist in a Fastify instance using hasRoute.
|
|
* Used when the APOPHIS plugin wasn't registered before routes.
|
|
*/
|
|
export async function discoverSpecificRoutes(
|
|
fastify: FastifyInjectInstance,
|
|
routePatterns: string[],
|
|
): Promise<RouteContract[]> {
|
|
if (typeof fastify.hasRoute !== 'function') {
|
|
return []
|
|
}
|
|
|
|
const routes: RouteContract[] = []
|
|
const seen = new Set<string>()
|
|
|
|
for (const pattern of routePatterns) {
|
|
// Parse pattern like "GET /users" or "POST /api/*"
|
|
const parts = pattern.split(' ')
|
|
const method = parts[0] || 'GET'
|
|
const path = parts.slice(1).join(' ')
|
|
|
|
// For exact routes (no wildcards), check if route exists
|
|
if (!pattern.includes('*') && !pattern.includes('?')) {
|
|
try {
|
|
if (fastify.hasRoute({ url: path, method })) {
|
|
const key = `${method} ${path}`
|
|
if (!seen.has(key)) {
|
|
seen.add(key)
|
|
routes.push({
|
|
method: method as RouteContract['method'],
|
|
path,
|
|
category: 'observer',
|
|
schema: {},
|
|
requires: [],
|
|
ensures: [],
|
|
invariants: [],
|
|
regexPatterns: {},
|
|
validateRuntime: false,
|
|
})
|
|
}
|
|
}
|
|
} catch {
|
|
// Route doesn't exist
|
|
}
|
|
}
|
|
}
|
|
|
|
return routes
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Route filtering
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Check if a route matches a filter pattern.
|
|
* Supports wildcards: * matches any characters.
|
|
*/
|
|
function matchRoutePattern(route: string, pattern: string): boolean {
|
|
// Convert pattern to regex
|
|
const regexPattern = pattern
|
|
.replace(/\*/g, '.*')
|
|
.replace(/\?/g, '.')
|
|
|
|
const regex = new RegExp(`^${regexPattern}$`, 'i')
|
|
return regex.test(route)
|
|
}
|
|
|
|
/**
|
|
* Filter routes by patterns.
|
|
*/
|
|
function filterRoutesByPatterns(routes: RouteContract[], patterns: string[]): RouteContract[] {
|
|
return routes.filter(route => {
|
|
const routeStr = `${route.method} ${route.path}`
|
|
return patterns.some(pattern => matchRoutePattern(routeStr, pattern))
|
|
})
|
|
}
|
|
|
|
/**
|
|
* Check if cwd is inside a git repository.
|
|
*/
|
|
async function isGitRepo(cwd: string): Promise<boolean> {
|
|
try {
|
|
const { execSync } = await import('node:child_process')
|
|
execSync('git rev-parse --git-dir', { cwd, encoding: 'utf-8', stdio: 'pipe' })
|
|
return true
|
|
} catch {
|
|
return false
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get git-modified files for --changed filtering.
|
|
*/
|
|
async function getGitChangedFiles(cwd: string): Promise<string[]> {
|
|
try {
|
|
const { execSync } = await import('node:child_process')
|
|
const output = execSync('git diff --name-only HEAD', { cwd, encoding: 'utf-8' })
|
|
return output.split('\n').filter(Boolean)
|
|
} catch {
|
|
return []
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Filter routes to only those modified in git.
|
|
*/
|
|
async function filterChangedRoutes(
|
|
routes: RouteContract[],
|
|
cwd: string,
|
|
): Promise<RouteContract[]> {
|
|
const changedFiles = await getGitChangedFiles(cwd)
|
|
|
|
// Map route paths to potential file paths (heuristic)
|
|
return routes.filter(route => {
|
|
const routePath = route.path
|
|
// Check if any changed file might contain this route
|
|
return changedFiles.some(file => {
|
|
// Simple heuristic: check if route path segments appear in file path
|
|
const segments = routePath.split('/').filter(Boolean)
|
|
return segments.some(segment => file.includes(segment))
|
|
})
|
|
})
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Contract execution
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Build a request for a route.
|
|
*/
|
|
function buildRouteRequest(route: RouteContract): {
|
|
method: string
|
|
url: string
|
|
body?: unknown
|
|
headers: Record<string, string>
|
|
} {
|
|
const headers: Record<string, string> = {
|
|
'content-type': 'application/json',
|
|
}
|
|
|
|
// Build body from schema if available
|
|
let body: unknown = undefined
|
|
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
|
|
if (bodySchema && route.method === 'POST') {
|
|
body = buildExampleBody(bodySchema)
|
|
}
|
|
|
|
return {
|
|
method: route.method,
|
|
url: route.path,
|
|
body,
|
|
headers,
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Build an example body from JSON Schema.
|
|
*/
|
|
function buildExampleBody(schema: Record<string, unknown>): unknown {
|
|
if (schema.type === 'object' && schema.properties) {
|
|
const obj: Record<string, unknown> = {}
|
|
const properties = schema.properties as Record<string, Record<string, unknown>>
|
|
for (const [key, propSchema] of Object.entries(properties)) {
|
|
obj[key] = buildExampleValue(propSchema)
|
|
}
|
|
return obj
|
|
}
|
|
return undefined
|
|
}
|
|
|
|
/**
|
|
* Build an example value from a property schema.
|
|
*/
|
|
function buildExampleValue(schema: Record<string, unknown>): unknown {
|
|
if (schema.type === 'string') {
|
|
if (schema.enum && Array.isArray(schema.enum) && schema.enum.length > 0) {
|
|
return schema.enum[0]
|
|
}
|
|
return 'test'
|
|
}
|
|
if (schema.type === 'number' || schema.type === 'integer') {
|
|
return 1
|
|
}
|
|
if (schema.type === 'boolean') {
|
|
return true
|
|
}
|
|
if (schema.type === 'array') {
|
|
return []
|
|
}
|
|
if (schema.type === 'object' && schema.properties) {
|
|
return buildExampleBody(schema)
|
|
}
|
|
return undefined
|
|
}
|
|
|
|
/**
|
|
* Execute a single contract for a route.
|
|
* Returns the evaluation context and any failure.
|
|
*/
|
|
async function executeContract(
|
|
fastify: FastifyInjectInstance,
|
|
route: RouteContract,
|
|
contract: string,
|
|
timeout?: number,
|
|
variant?: { name: string; headers?: Record<string, string> },
|
|
): Promise<{ ctx: EvalContext; failure?: VerifyFailure }> {
|
|
const request = buildRouteRequest(route)
|
|
|
|
// Merge variant headers if provided
|
|
const headers = variant?.headers
|
|
? { ...request.headers, ...variant.headers }
|
|
: request.headers
|
|
|
|
// Execute the primary request
|
|
const ctx = await executeHttp(fastify, route, {
|
|
method: request.method,
|
|
url: request.url,
|
|
body: request.body,
|
|
headers,
|
|
query: {},
|
|
}, undefined, timeout)
|
|
|
|
// Build eval context with operation resolver for cross-operation calls
|
|
const evalCtx: EvalContext = {
|
|
...ctx,
|
|
operationResolver: createOperationResolver(fastify, headers, ctx),
|
|
}
|
|
|
|
// Parse and evaluate the contract
|
|
try {
|
|
const parsed = parse(contract)
|
|
const result = await evaluateAsync(parsed.ast, evalCtx)
|
|
|
|
if (!result.success || !result.value) {
|
|
return {
|
|
ctx: evalCtx,
|
|
failure: {
|
|
route: variant && variant.name !== 'default'
|
|
? `[variant:${variant.name}] ${route.method} ${route.path}`
|
|
: `${route.method} ${route.path}`,
|
|
contract,
|
|
expected: 'true',
|
|
observed: result.success ? String(result.value) : result.error,
|
|
},
|
|
}
|
|
}
|
|
|
|
return { ctx: evalCtx }
|
|
} catch (error) {
|
|
return {
|
|
ctx: evalCtx,
|
|
failure: {
|
|
route: variant && variant.name !== 'default'
|
|
? `[variant:${variant.name}] ${route.method} ${route.path}`
|
|
: `${route.method} ${route.path}`,
|
|
contract,
|
|
expected: 'true',
|
|
observed: error instanceof Error ? error.message : String(error),
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Main verify runner
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Run deterministic contract verification.
|
|
*
|
|
* Flow:
|
|
* 1. Discover routes from Fastify app
|
|
* 2. Apply route filters (patterns, changed, profile routes)
|
|
* 3. Check for behavioral contracts
|
|
* 4. Execute each contract deterministically
|
|
* 5. Aggregate results
|
|
*/
|
|
export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult> {
|
|
const started = Date.now()
|
|
const { fastify, routeFilters, changed, profileRoutes } = deps
|
|
|
|
// 1. Discover routes
|
|
let allRoutes = await discoverAppRoutes(fastify)
|
|
|
|
// If no routes discovered (plugin not registered before routes),
|
|
// try to discover specific routes from filters
|
|
if (allRoutes.length === 0 && (routeFilters?.length || profileRoutes?.length)) {
|
|
const patternsToCheck = [
|
|
...(routeFilters || []),
|
|
...(profileRoutes || []),
|
|
]
|
|
allRoutes = await discoverSpecificRoutes(fastify, patternsToCheck)
|
|
}
|
|
|
|
const availableRoutes = allRoutes.map(r => `${r.method} ${r.path}`)
|
|
|
|
// 2. Apply filters
|
|
let routes = allRoutes
|
|
|
|
// Apply profile routes filter first
|
|
if (profileRoutes && profileRoutes.length > 0) {
|
|
routes = filterRoutesByPatterns(routes, profileRoutes)
|
|
}
|
|
|
|
// Apply --routes flag filter
|
|
if (routeFilters && routeFilters.length > 0) {
|
|
routes = filterRoutesByPatterns(routes, routeFilters)
|
|
}
|
|
|
|
// Apply --changed filter
|
|
if (changed) {
|
|
const cwd = process.cwd()
|
|
const inGit = await isGitRepo(cwd)
|
|
if (!inGit) {
|
|
return {
|
|
passed: false,
|
|
total: 0,
|
|
passedCount: 0,
|
|
failed: 0,
|
|
failures: [],
|
|
durationMs: Date.now() - started,
|
|
noRoutesMatched: false,
|
|
noContractsFound: false,
|
|
availableRoutes,
|
|
artifactPaths: [],
|
|
notGitRepo: true,
|
|
}
|
|
}
|
|
routes = await filterChangedRoutes(routes, cwd)
|
|
}
|
|
|
|
// Check if any routes matched
|
|
if (routes.length === 0) {
|
|
return {
|
|
passed: false,
|
|
total: 0,
|
|
passedCount: 0,
|
|
failed: 0,
|
|
failures: [],
|
|
durationMs: Date.now() - started,
|
|
noRoutesMatched: true,
|
|
noContractsFound: false,
|
|
availableRoutes,
|
|
artifactPaths: [],
|
|
}
|
|
}
|
|
|
|
// 3. Check for behavioral contracts
|
|
const routesWithContracts = routes.filter(route =>
|
|
route.ensures.length > 0 || route.requires.length > 0
|
|
)
|
|
|
|
if (routesWithContracts.length === 0) {
|
|
return {
|
|
passed: false,
|
|
total: 0,
|
|
passedCount: 0,
|
|
failed: 0,
|
|
failures: [],
|
|
durationMs: Date.now() - started,
|
|
noRoutesMatched: false,
|
|
noContractsFound: true,
|
|
availableRoutes,
|
|
artifactPaths: [],
|
|
}
|
|
}
|
|
|
|
// 4. Execute contracts (with variant expansion)
|
|
const failures: VerifyFailure[] = []
|
|
let total = 0
|
|
let passedCount = 0
|
|
|
|
for (const route of routesWithContracts) {
|
|
const contracts = [...route.requires, ...route.ensures]
|
|
const variants = route.variants && route.variants.length > 0
|
|
? route.variants
|
|
: [{ name: 'default' }]
|
|
|
|
for (const variant of variants) {
|
|
for (const contract of contracts) {
|
|
total++
|
|
const result = await executeContract(fastify, route, contract, deps.timeout, variant)
|
|
|
|
if (result.failure) {
|
|
failures.push(result.failure)
|
|
} else {
|
|
passedCount++
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
const durationMs = Date.now() - started
|
|
|
|
// Sort failures deterministically by route then contract for stable output
|
|
const sortedFailures = failures.sort((a, b) => {
|
|
const routeCmp = a.route.localeCompare(b.route)
|
|
if (routeCmp !== 0) return routeCmp
|
|
return a.contract.localeCompare(b.contract)
|
|
})
|
|
|
|
return {
|
|
passed: failures.length === 0,
|
|
total,
|
|
passedCount,
|
|
failed: failures.length,
|
|
failures: sortedFailures,
|
|
durationMs,
|
|
noRoutesMatched: false,
|
|
noContractsFound: false,
|
|
availableRoutes,
|
|
artifactPaths: [],
|
|
}
|
|
}
|