chore: crush git history - reborn from consolidation on 2026-03-10

This commit is contained in:
John Dvorak
2026-03-10 00:00:00 -07:00
commit d278c4b105
313 changed files with 87549 additions and 0 deletions
+367
View File
@@ -0,0 +1,367 @@
/**
* S8: Doctor thread - Config validation checks
*
* Checks:
* - Config file exists and is loadable
* - Unknown keys rejection with exact path
* - Legacy config detection (deprecated field names)
* - Mixed legacy/new config style detection
*/
import {
loadConfig,
loadConfigFile,
discoverConfig,
ConfigValidationError,
type Config,
type LoadConfigResult,
} from '../../../core/config-loader.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface ConfigCheckResult {
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
remediation?: string;
mode: 'all' | 'verify' | 'observe' | 'qualify';
}
export interface ConfigCheckOptions {
cwd: string;
configPath?: string;
}
// ---------------------------------------------------------------------------
// Legacy field detection
// ---------------------------------------------------------------------------
/**
* Map of deprecated field names to their modern equivalents.
*/
const LEGACY_FIELDS: Record<string, string> = {
testMode: 'mode',
testProfiles: 'profiles',
testPresets: 'presets',
envPolicies: 'environments',
usesPreset: 'preset',
routeFilter: 'routes',
testDepth: 'depth',
maxDuration: 'timeout',
canVerify: 'allowVerify',
};
/**
* Recursively scan an object for legacy field names.
* Returns array of { path, legacyKey, modernKey } tuples.
*/
function findLegacyFields(
value: unknown,
path: string = '',
): Array<{ path: string; legacyKey: string; modernKey: string }> {
const results: Array<{ path: string; legacyKey: string; modernKey: string }> = [];
if (value === null || typeof value !== 'object') {
return results;
}
const obj = value as Record<string, unknown>;
for (const key of Object.keys(obj)) {
const currentPath = path ? `${path}.${key}` : key;
// Check if this key is legacy
if (LEGACY_FIELDS[key]) {
results.push({
path: currentPath,
legacyKey: key,
modernKey: LEGACY_FIELDS[key],
});
}
// Recurse into nested objects
const fieldValue = obj[key];
if (fieldValue !== null && typeof fieldValue === 'object' && !Array.isArray(fieldValue)) {
results.push(...findLegacyFields(fieldValue, currentPath));
}
}
return results;
}
/**
* Check if config contains legacy field names.
*/
export function checkLegacyConfig(config: Config | null): ConfigCheckResult {
if (!config) {
return {
name: 'legacy-config',
status: 'pass',
message: 'No config to check for legacy fields.',
mode: 'all',
};
}
const legacyFields = findLegacyFields(config);
if (legacyFields.length > 0) {
const details = legacyFields
.map(f => ` ${f.path}: "${f.legacyKey}" → "${f.modernKey}"`)
.join('\n');
return {
name: 'legacy-config',
status: 'warn',
message: `Found ${legacyFields.length} legacy field(s) in config.`,
detail: `Run "apophis migrate" to update these fields:\n${details}`,
remediation: 'Run "apophis migrate --dry-run" to preview rewrites.',
mode: 'all',
};
}
return {
name: 'legacy-config',
status: 'pass',
message: 'No legacy config fields detected.',
mode: 'all',
};
}
/**
* Check for mixed legacy and new config styles.
* This happens when some fields use old names and others use new names.
*/
export function checkMixedConfig(config: Config | null): ConfigCheckResult {
if (!config) {
return {
name: 'mixed-config',
status: 'pass',
message: 'No config to check for mixed styles.',
mode: 'all',
};
}
const legacyFields = findLegacyFields(config);
const hasLegacy = legacyFields.length > 0;
// Check if config also has modern fields at the same level as legacy ones
const hasModern = Object.keys(config).some(key => !LEGACY_FIELDS[key] && key !== 'name');
if (hasLegacy && hasModern) {
const legacyTopLevel = Object.keys(config).filter(key => LEGACY_FIELDS[key]);
const modernTopLevel = Object.keys(config).filter(key => !LEGACY_FIELDS[key] && key !== 'name');
// Only fail if there are actual modern fields that conflict with legacy ones
// A config with only legacy fields should warn, not fail
const hasConflictingModern = modernTopLevel.length > 0 &&
legacyTopLevel.some(lf => LEGACY_FIELDS[lf] !== undefined && modernTopLevel.includes(LEGACY_FIELDS[lf]));
if (hasConflictingModern) {
return {
name: 'mixed-config',
status: 'fail',
message: 'Config uses both legacy and modern field names.',
detail:
`Legacy fields: ${legacyTopLevel.join(', ')}\n` +
`Modern fields: ${modernTopLevel.join(', ')}\n` +
`Run "apophis migrate" to unify your config to the modern schema.`,
remediation: 'Run "apophis migrate --write" to unify config to modern schema.',
mode: 'all',
};
}
// Has both legacy and other modern fields - still warn but don't fail
return {
name: 'mixed-config',
status: 'warn',
message: 'Config contains legacy field names alongside modern fields.',
detail:
`Legacy fields: ${legacyTopLevel.join(', ')}\n` +
`Run "apophis migrate" to update to the modern schema.`,
remediation: 'Run "apophis migrate --dry-run" to preview rewrites.',
mode: 'all',
};
}
if (hasLegacy) {
return {
name: 'mixed-config',
status: 'warn',
message: 'Config uses legacy field names only.',
detail: 'Run "apophis migrate" to update to the modern schema.',
remediation: 'Run "apophis migrate --write" to update to modern schema.',
mode: 'all',
};
}
return {
name: 'mixed-config',
status: 'pass',
message: 'Config uses consistent modern field names.',
mode: 'all',
};
}
// ---------------------------------------------------------------------------
// Unknown key check
// ---------------------------------------------------------------------------
/**
* Check config for unknown keys by loading with strict validation.
*/
export async function checkUnknownKeys(options: ConfigCheckOptions): Promise<ConfigCheckResult> {
const { cwd, configPath } = options;
try {
const loadResult = await loadConfig({
cwd,
configPath,
});
if (!loadResult.configPath) {
return {
name: 'unknown-keys',
status: 'warn',
message: 'No config file found. Skipping unknown key check.',
detail: 'Run "apophis init" to create a config file.',
remediation: 'Run "apophis init --preset safe-ci" to scaffold a config.',
mode: 'all',
};
}
return {
name: 'unknown-keys',
status: 'pass',
message: 'Config keys are valid.',
mode: 'all',
};
} catch (error) {
if (error instanceof ConfigValidationError) {
return {
name: 'unknown-keys',
status: 'fail',
message: `Unknown config key at ${error.path}`,
detail: `Key "${error.key}" is not recognized by the APOPHIS config schema.`,
remediation: `Remove "${error.key}" from your config or check the docs for valid keys.`,
mode: 'all',
};
}
const message = error instanceof Error ? error.message : String(error);
return {
name: 'unknown-keys',
status: 'fail',
message: `Config validation failed: ${message}`,
remediation: 'Check your config file syntax and ensure it exports a valid object.',
mode: 'all',
};
}
}
// ---------------------------------------------------------------------------
// Config load check
// ---------------------------------------------------------------------------
/**
* Check if config can be loaded successfully.
*/
export async function checkConfigLoad(options: ConfigCheckOptions): Promise<ConfigCheckResult> {
const { cwd, configPath } = options;
try {
const loadResult = await loadConfig({
cwd,
configPath,
});
if (!loadResult.configPath) {
return {
name: 'config-load',
status: 'warn',
message: 'No config file found.',
detail: 'APOPHIS will use defaults. Run "apophis init" to create a config.',
remediation: 'Run "apophis init --preset safe-ci" to scaffold a config.',
mode: 'all',
};
}
return {
name: 'config-load',
status: 'pass',
message: `Config loaded from ${loadResult.configPath}`,
mode: 'all',
};
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return {
name: 'config-load',
status: 'fail',
message: `Failed to load config: ${message}`,
remediation: 'Check your config file syntax and ensure it exports a valid object.',
mode: 'all',
};
}
}
// ---------------------------------------------------------------------------
// Raw config loader (without validation)
// ---------------------------------------------------------------------------
/**
* Load raw config without schema validation.
* Used for legacy detection when validation would fail on legacy keys.
*/
async function loadRawConfig(options: ConfigCheckOptions): Promise<Config | null> {
const { cwd, configPath } = options;
// Discover config file
const discoveredPath = configPath || discoverConfig(cwd);
if (!discoveredPath) {
return null;
}
return await loadConfigFile(discoveredPath);
}
// ---------------------------------------------------------------------------
// Main config check runner
// ---------------------------------------------------------------------------
/**
* Run all config checks.
*/
export async function runConfigChecks(options: ConfigCheckOptions): Promise<ConfigCheckResult[]> {
const results: ConfigCheckResult[] = [];
// 1. Check config can be loaded
results.push(await checkConfigLoad(options));
// 2. Check for unknown keys
results.push(await checkUnknownKeys(options));
// 3. Check for legacy fields - load raw config without validation
try {
const rawConfig = await loadRawConfig(options);
results.push(checkLegacyConfig(rawConfig));
results.push(checkMixedConfig(rawConfig));
} catch {
// If config can't be loaded, skip legacy/mixed checks
results.push({
name: 'legacy-config',
status: 'warn',
message: 'Could not check for legacy fields (config failed to load).',
mode: 'all',
});
results.push({
name: 'mixed-config',
status: 'warn',
message: 'Could not check for mixed config (config failed to load).',
mode: 'all',
});
}
return results;
}
@@ -0,0 +1,242 @@
/**
* S8: Doctor thread - Dependency checks
*
* Checks:
* - Node.js version compatibility
* - Fastify installation and version
* - @fastify/swagger installation and version
* - Peer dependency completeness
*/
import { readFileSync, existsSync } from 'node:fs';
import { resolve } from 'node:path';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface DependencyCheckResult {
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
remediation?: string;
mode: 'all' | 'verify' | 'observe' | 'qualify';
}
export interface DependencyCheckOptions {
cwd: string;
nodeVersion: string;
}
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
const MIN_NODE_VERSION = 18;
const REQUIRED_PEER_DEPS = ['fastify', '@fastify/swagger'];
// ---------------------------------------------------------------------------
// Node.js version check
// ---------------------------------------------------------------------------
/**
* Parse major version from Node.js version string.
*/
function parseNodeMajor(version: string): number {
const match = version.match(/v?(\d+)/);
return match && match[1] ? parseInt(match[1], 10) : 0;
}
/**
* Check if Node.js version meets minimum requirement.
*/
export function checkNodeVersion(nodeVersion: string): DependencyCheckResult {
const major = parseNodeMajor(nodeVersion);
if (major < MIN_NODE_VERSION) {
return {
name: 'node-version',
status: 'fail',
message: `Node.js ${nodeVersion} is not supported. Minimum required: ${MIN_NODE_VERSION}.x`,
detail: `APOPHIS requires Node.js ${MIN_NODE_VERSION} or higher for ESM and modern features.`,
remediation: `Upgrade Node.js to ${MIN_NODE_VERSION}.x or higher (use nvm, fnm, or your package manager).`,
mode: 'all',
};
}
return {
name: 'node-version',
status: 'pass',
message: `Node.js ${nodeVersion} meets minimum requirement (${MIN_NODE_VERSION}+)`,
mode: 'all',
};
}
// ---------------------------------------------------------------------------
// Package.json dependency checks
// ---------------------------------------------------------------------------
/**
* Load and parse package.json from cwd.
*/
function loadPackageJson(cwd: string): Record<string, unknown> | null {
const pkgPath = resolve(cwd, 'package.json');
if (!existsSync(pkgPath)) {
return null;
}
try {
return JSON.parse(readFileSync(pkgPath, 'utf-8'));
} catch {
return null;
}
}
/**
* Check if a dependency is installed (declared in package.json).
*/
function hasDependency(pkg: Record<string, unknown>, name: string): boolean {
const deps = pkg.dependencies as Record<string, string> | undefined;
const devDeps = pkg.devDependencies as Record<string, string> | undefined;
return !!(deps?.[name] || devDeps?.[name]);
}
/**
* Get installed version range for a dependency.
*/
function getDependencyVersion(pkg: Record<string, unknown>, name: string): string | undefined {
const deps = pkg.dependencies as Record<string, string> | undefined;
const devDeps = pkg.devDependencies as Record<string, string> | undefined;
return deps?.[name] || devDeps?.[name];
}
/**
* Check Fastify installation and version.
*/
export function checkFastify(pkg: Record<string, unknown> | null): DependencyCheckResult {
if (!pkg) {
return {
name: 'fastify',
status: 'fail',
message: 'No package.json found. Cannot check Fastify installation.',
detail: 'Ensure you are running from a project root with a package.json file.',
remediation: 'Run npm init -y in your project root, then install dependencies.',
mode: 'all',
};
}
if (!hasDependency(pkg, 'fastify')) {
return {
name: 'fastify',
status: 'fail',
message: 'Fastify is not installed.',
detail: 'Install it with: npm install fastify@^5.0.0',
remediation: 'npm install fastify@^5.0.0',
mode: 'all',
};
}
const version = getDependencyVersion(pkg, 'fastify');
// Check if version is 5.x (recommended)
if (version != null && !version.includes('5')) {
return {
name: 'fastify',
status: 'warn',
message: `Fastify ${version} is installed. APOPHIS is tested with Fastify 5.x.`,
detail: 'Consider upgrading to fastify@^5.0.0 for best compatibility.',
mode: 'all',
};
}
return {
name: 'fastify',
status: 'pass',
message: `Fastify ${version || 'installed'} is present.`,
mode: 'all',
};
}
/**
* Check @fastify/swagger installation.
*/
export function checkSwagger(pkg: Record<string, unknown> | null): DependencyCheckResult {
if (!pkg) {
return {
name: '@fastify/swagger',
status: 'fail',
message: 'No package.json found. Cannot check @fastify/swagger installation.',
detail: 'Ensure you are running from a project root with a package.json file.',
remediation: 'Run npm init -y in your project root, then install dependencies.',
mode: 'all',
};
}
if (!hasDependency(pkg, '@fastify/swagger')) {
return {
name: '@fastify/swagger',
status: 'fail',
message: '@fastify/swagger is not installed.',
detail: 'APOPHIS requires @fastify/swagger for route discovery. Install with: npm install @fastify/swagger@^9.0.0',
remediation: 'npm install @fastify/swagger@^9.0.0',
mode: 'all',
};
}
const version = getDependencyVersion(pkg, '@fastify/swagger');
return {
name: '@fastify/swagger',
status: 'pass',
message: `@fastify/swagger ${version || 'installed'} is present.`,
mode: 'all',
};
}
// ---------------------------------------------------------------------------
// Main dependency check runner
// ---------------------------------------------------------------------------
/**
* Run all dependency checks.
*/
export function runDependencyChecks(options: DependencyCheckOptions): DependencyCheckResult[] {
const { cwd, nodeVersion } = options;
const pkg = loadPackageJson(cwd);
const results: DependencyCheckResult[] = [];
// Node version
results.push(checkNodeVersion(nodeVersion));
// Fastify
results.push(checkFastify(pkg));
// Swagger
results.push(checkSwagger(pkg));
// Check for other missing peer deps
if (pkg) {
const missing = REQUIRED_PEER_DEPS.filter(dep => !hasDependency(pkg, dep));
if (missing.length > 0) {
results.push({
name: 'peer-dependencies',
status: 'fail',
message: `Missing peer dependencies: ${missing.join(', ')}`,
detail: 'Install missing packages to ensure full APOPHIS functionality.',
remediation: `npm install ${missing.join(' ')}`,
mode: 'all',
});
} else {
results.push({
name: 'peer-dependencies',
status: 'pass',
message: 'All required peer dependencies are installed.',
mode: 'all',
});
}
}
return results;
}
+265
View File
@@ -0,0 +1,265 @@
/**
* S8: Doctor thread - Docs and example smoke checks
*
* Checks:
* - Docs examples match current config schema
* - README/APOPHIS.md exists and is readable
* - In CI mode: fail if docs drift from reality
*/
import { readFileSync, existsSync, readdirSync } from 'node:fs';
import { resolve } from 'node:path';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface DocsCheckResult {
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
remediation?: string;
mode?: 'all' | 'verify' | 'observe' | 'qualify';
}
export interface DocsCheckOptions {
cwd: string;
isCI: boolean;
}
// ---------------------------------------------------------------------------
// README / APOPHIS.md check
// ---------------------------------------------------------------------------
/**
* Check if project has documentation files.
*/
export function checkDocsExist(options: DocsCheckOptions): DocsCheckResult {
const { cwd } = options;
const readmePath = resolve(cwd, 'README.md');
const apophisPath = resolve(cwd, 'APOPHIS.md');
const hasReadme = existsSync(readmePath);
const hasApophis = existsSync(apophisPath);
if (hasApophis) {
return {
name: 'docs-exist',
status: 'pass',
message: 'APOPHIS.md documentation found.',
mode: 'all',
};
}
if (hasReadme) {
return {
name: 'docs-exist',
status: 'pass',
message: 'README.md found (no APOPHIS.md).',
detail: 'Consider creating APOPHIS.md for APOPHIS-specific documentation.',
mode: 'all',
};
}
return {
name: 'docs-exist',
status: 'warn',
message: 'No README.md or APOPHIS.md found.',
detail: 'Documentation helps team members understand your APOPHIS setup.',
remediation: 'Create APOPHIS.md with setup instructions for your team.',
mode: 'all',
};
}
// ---------------------------------------------------------------------------
// Config schema drift check
// ---------------------------------------------------------------------------
/**
* Known legacy field names that should not appear in docs.
*/
const LEGACY_FIELD_NAMES = [
'testMode',
'testProfiles',
'testPresets',
'envPolicies',
'usesPreset',
'routeFilter',
'testDepth',
'maxDuration',
'canVerify',
];
/**
* Check if docs contain legacy field names (indicating stale docs).
*/
export function checkDocsSchemaDrift(options: DocsCheckOptions): DocsCheckResult {
const { cwd, isCI } = options;
const docsFiles = findDocsFiles(cwd);
if (docsFiles.length === 0) {
return {
name: 'docs-schema-drift',
status: 'warn',
message: 'No documentation files found to check for schema drift.',
mode: 'all',
};
}
const drift: Array<{ file: string; legacyFields: string[] }> = [];
for (const file of docsFiles) {
try {
const content = readFileSync(file, 'utf-8');
const foundLegacy = LEGACY_FIELD_NAMES.filter(field => content.includes(field));
if (foundLegacy.length > 0) {
drift.push({ file, legacyFields: foundLegacy });
}
} catch {
// Skip unreadable files
}
}
if (drift.length > 0) {
const details = drift
.map(d => ` ${d.file}: ${d.legacyFields.join(', ')}`)
.join('\n');
return {
name: 'docs-schema-drift',
status: isCI ? 'fail' : 'warn',
message: `Found ${drift.length} documentation file(s) with legacy field names.`,
detail: `Update docs to use current config schema:\n${details}\n\nRun "apophis migrate --dry-run" to see rewrites.`,
remediation: 'Update docs to use current field names, or run "apophis migrate --dry-run" to see rewrites.',
mode: 'all',
};
}
return {
name: 'docs-schema-drift',
status: 'pass',
message: 'No schema drift detected in documentation.',
mode: 'all',
};
}
/**
* Find documentation files in the project.
*/
function findDocsFiles(cwd: string): string[] {
const files: string[] = [];
const candidates = [
'README.md',
'APOPHIS.md',
'docs',
];
for (const candidate of candidates) {
const fullPath = resolve(cwd, candidate);
if (existsSync(fullPath)) {
if (candidate.endsWith('.md')) {
files.push(fullPath);
} else {
// It's a directory, scan for .md files
try {
const entries = readdirSync(fullPath, { withFileTypes: true });
for (const entry of entries) {
if (entry.isFile() && entry.name.endsWith('.md')) {
files.push(resolve(fullPath, entry.name));
}
}
} catch {
// Skip unreadable directories
}
}
}
}
return files;
}
// ---------------------------------------------------------------------------
// Example code check
// ---------------------------------------------------------------------------
/**
* Check if docs contain runnable examples that match current API.
*/
export function checkExamplesValid(options: DocsCheckOptions): DocsCheckResult {
const { cwd } = options;
const apophisPath = resolve(cwd, 'APOPHIS.md');
if (!existsSync(apophisPath)) {
return {
name: 'examples-valid',
status: 'pass',
message: 'No APOPHIS.md to check for examples.',
mode: 'all',
};
}
try {
const content = readFileSync(apophisPath, 'utf-8');
// Check for common example patterns
const hasVerifyExample = content.includes('apophis verify');
const hasObserveExample = content.includes('apophis observe');
const hasQualifyExample = content.includes('apophis qualify');
const issues: string[] = [];
if (!hasVerifyExample) {
issues.push('No verify example found.');
}
if (!hasObserveExample) {
issues.push('No observe example found.');
}
if (issues.length > 0) {
return {
name: 'examples-valid',
status: 'warn',
message: 'APOPHIS.md is missing some command examples.',
detail: issues.join('\n'),
remediation: 'Add examples for verify, observe, and qualify commands to APOPHIS.md.',
mode: 'all',
};
}
return {
name: 'examples-valid',
status: 'pass',
message: 'APOPHIS.md contains examples for core commands.',
mode: 'all',
};
} catch {
return {
name: 'examples-valid',
status: 'warn',
message: 'Could not read APOPHIS.md to check examples.',
mode: 'all',
};
}
}
// ---------------------------------------------------------------------------
// Main docs check runner
// ---------------------------------------------------------------------------
/**
* Run all docs checks.
*/
export function runDocsChecks(options: DocsCheckOptions): DocsCheckResult[] {
const results: DocsCheckResult[] = [];
results.push(checkDocsExist(options));
results.push(checkDocsSchemaDrift(options));
results.push(checkExamplesValid(options));
return results;
}
+282
View File
@@ -0,0 +1,282 @@
/**
* S8: Doctor thread - Route discovery checks
*
* Checks:
* - Can we discover routes from the Fastify app?
* - Are routes properly registered with swagger?
* - Is the app file loadable?
*/
import { existsSync } from 'node:fs';
import { resolve } from 'node:path';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface RouteCheckResult {
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
remediation?: string;
mode: 'all' | 'verify' | 'observe' | 'qualify';
}
export interface RouteCheckOptions {
cwd: string;
configPath?: string;
}
// ---------------------------------------------------------------------------
// App file detection
// ---------------------------------------------------------------------------
const APP_CANDIDATES = [
'app.js',
'app.ts',
'server.js',
'server.ts',
'index.js',
'index.ts',
'src/app.js',
'src/app.ts',
'src/server.js',
'src/server.ts',
'src/index.js',
'src/index.ts',
];
/**
* Find the Fastify app entrypoint file.
*/
function findAppFile(cwd: string): string | null {
for (const candidate of APP_CANDIDATES) {
const fullPath = resolve(cwd, candidate);
if (existsSync(fullPath)) {
return candidate;
}
}
return null;
}
/**
* Check if app file exists and is readable.
*/
export function checkAppFile(options: RouteCheckOptions): RouteCheckResult {
const appFile = findAppFile(options.cwd);
if (!appFile) {
return {
name: 'app-file',
status: 'warn',
message: 'No Fastify app file found.',
detail: `Searched for: ${APP_CANDIDATES.join(', ')}. ` +
'APOPHIS needs an app.js or similar to discover routes.',
remediation: 'Create an app.js or server.js that exports a Fastify instance.',
mode: 'all',
};
}
return {
name: 'app-file',
status: 'pass',
message: `Found Fastify app: ${appFile}`,
mode: 'all',
};
}
// ---------------------------------------------------------------------------
// Route discovery check
// ---------------------------------------------------------------------------
/**
* Attempt to load the app and discover routes.
*/
export async function checkRouteDiscovery(options: RouteCheckOptions): Promise<RouteCheckResult> {
const appFile = findAppFile(options.cwd);
if (!appFile) {
return {
name: 'route-discovery',
status: 'warn',
message: 'Skipping route discovery (no app file found).',
mode: 'all',
};
}
try {
const appPath = resolve(options.cwd, appFile);
const appModule = await import(appPath);
const app = appModule.default || appModule;
// Check if it looks like a Fastify instance
if (!app || typeof app !== 'object') {
return {
name: 'route-discovery',
status: 'fail',
message: `App file ${appFile} does not export a valid object.`,
detail: 'Ensure the app file exports a Fastify instance as default.',
remediation: 'Export your Fastify instance as default: export default app;',
mode: 'all',
};
}
// Try to register APOPHIS plugin for route capture
// Skip if already registered to avoid "decorator already added" errors
const isAlreadyRegistered = app.hasDecorator && typeof app.hasDecorator === 'function' && app.hasDecorator('apophis');
if (!isAlreadyRegistered) {
try {
const apophisPlugin = (await import('../../../../index.js')).default;
if (typeof apophisPlugin === 'function' && typeof app.register === 'function') {
await app.register(apophisPlugin, { runtime: 'off' });
}
} catch (err) {
const errMessage = err instanceof Error ? err.message : String(err);
// If decorator already added, the plugin is pre-registered — that's fine
if (errMessage.includes("decorator 'apophis' has already been added")) {
// Plugin is already registered, proceed with discovery
}
// Otherwise, plugin registration is optional for discovery
}
}
// Try to ready the app so routes are registered
if (typeof app.ready === 'function') {
await app.ready();
}
// Check for routes
let routeCount = 0;
// Fastify 5+ routes access
if (app.routes && typeof app.routes === 'function') {
const routes = app.routes();
routeCount = Array.isArray(routes) ? routes.length : 0;
}
// Fallback: check if we can get routes via inject or other methods
if (routeCount === 0 && app.hasRoute) {
// We can't enumerate, but we can at least verify the app is functional
routeCount = -1; // Unknown but app seems functional
}
if (routeCount === 0) {
return {
name: 'route-discovery',
status: 'warn',
message: `App loaded from ${appFile} but no routes were discovered.`,
detail: 'Ensure routes are registered before exporting the app. ' +
'APOPHIS discovers routes via the onRoute hook.',
remediation: 'Register routes before exporting the app, or ensure the APOPHIS plugin is registered.',
mode: 'all',
};
}
if (routeCount < 0) {
return {
name: 'route-discovery',
status: 'pass',
message: `App loaded from ${appFile}. Route enumeration not available (app is functional).`,
detail: 'Route count could not be determined, but the app appears to be a valid Fastify instance.',
mode: 'all',
};
}
return {
name: 'route-discovery',
status: 'pass',
message: `Discovered ${routeCount} route(s) from ${appFile}.`,
mode: 'all',
};
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
// If the error is a module not found, treat as warn (dependencies may not be installed in test env)
if (message.includes('Cannot find module') || message.includes('Cannot resolve')) {
return {
name: 'route-discovery',
status: 'warn',
message: `Could not load app from ${appFile}: ${message}`,
detail: 'Dependencies may not be installed. Run npm install to resolve.',
remediation: 'Run npm install to install missing dependencies.',
mode: 'all',
};
}
return {
name: 'route-discovery',
status: 'fail',
message: `Failed to load app from ${appFile}: ${message}`,
detail: 'Check that the app file exports a valid Fastify instance and all imports resolve.',
remediation: 'Verify all imports in your app file are correct and the file exports a Fastify instance.',
mode: 'all',
};
}
}
// ---------------------------------------------------------------------------
// Swagger registration check
// ---------------------------------------------------------------------------
/**
* Check if @fastify/swagger is registered in the app.
*/
export async function checkSwaggerRegistration(options: RouteCheckOptions): Promise<RouteCheckResult> {
const appFile = findAppFile(options.cwd);
if (!appFile) {
return {
name: 'swagger-registration',
status: 'warn',
message: 'Skipping swagger check (no app file found).',
mode: 'all',
};
}
try {
const appPath = resolve(options.cwd, appFile);
const content = (await import('node:fs')).readFileSync(appPath, 'utf-8');
if (content.includes('@fastify/swagger') || content.includes('fastify-swagger')) {
return {
name: 'swagger-registration',
status: 'pass',
message: `@fastify/swagger appears to be imported in ${appFile}.`,
mode: 'all',
};
}
return {
name: 'swagger-registration',
status: 'warn',
message: `@fastify/swagger not found in ${appFile}.`,
detail: 'APOPHIS requires @fastify/swagger for route discovery. ' +
'Register it with: await app.register(import("@fastify/swagger"), { openapi: { info: { title: "API", version: "1.0.0" } } });',
remediation: 'npm install @fastify/swagger@^9.0.0 and register it in your app file.',
mode: 'all',
};
} catch {
return {
name: 'swagger-registration',
status: 'warn',
message: `Could not read ${appFile} to check swagger registration.`,
mode: 'all',
};
}
}
// ---------------------------------------------------------------------------
// Main route check runner
// ---------------------------------------------------------------------------
/**
* Run all route discovery checks.
*/
export async function runRouteChecks(options: RouteCheckOptions): Promise<RouteCheckResult[]> {
const results: RouteCheckResult[] = [];
results.push(checkAppFile(options));
results.push(await checkRouteDiscovery(options));
results.push(await checkSwaggerRegistration(options));
return results;
}
+230
View File
@@ -0,0 +1,230 @@
/**
* S8: Doctor thread - Safety checks
*
* Checks:
* - Qualify mode in unsafe environment
* - Environment policy validation
* - Mixed config style safety
*/
import { PolicyEngine, detectEnvironment } from '../../../core/policy-engine.js';
import type { Config } from '../../../core/config-loader.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface SafetyCheckResult {
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
remediation?: string;
mode: 'all' | 'verify' | 'observe' | 'qualify';
}
export interface SafetyCheckOptions {
cwd: string;
config: Config;
env?: string;
modeFilter?: 'verify' | 'observe' | 'qualify' | undefined;
}
// ---------------------------------------------------------------------------
// Qualify in unsafe environment
// ---------------------------------------------------------------------------
/**
* Check if qualify mode would be allowed in the current environment.
*/
export function checkQualifySafety(options: SafetyCheckOptions): SafetyCheckResult {
const { config, env: explicitEnv } = options;
const env = explicitEnv || detectEnvironment();
// Use the policy engine's qualify-specific safety check directly
// The PolicyEngine.check() runs ALL checks including mode-allowed, profile features, etc.
// We only care about whether qualify is blocked in this environment.
const engine = new PolicyEngine({
config,
env,
mode: 'qualify',
});
const result = engine.check();
if (!result.allowed) {
// Find the specific error about qualify being blocked
const qualifyBlockError = result.errors.find(e =>
e.includes('blocked') || e.includes('not allowed') || e.includes('Qualify')
);
if (qualifyBlockError) {
return {
name: 'qualify-safety',
status: 'fail',
message: `Qualify mode is blocked in environment "${env}".`,
detail: qualifyBlockError,
remediation: 'Run in a local or test environment, or update environment policy to allow qualify.',
mode: 'qualify',
};
}
// Other errors (profile features, etc.) are warnings in doctor context
return {
name: 'qualify-safety',
status: 'warn',
message: `Qualify mode has warnings in environment "${env}".`,
detail: result.errors.join('\n') + '\n' + result.warnings.join('\n'),
mode: 'qualify',
};
}
// Even if allowed, there may be warnings
if (result.warnings.length > 0) {
return {
name: 'qualify-safety',
status: 'warn',
message: `Qualify mode is allowed in environment "${env}" with warnings.`,
detail: result.warnings.join('\n'),
mode: 'qualify',
};
}
return {
name: 'qualify-safety',
status: 'pass',
message: `Qualify mode is allowed in environment "${env}".`,
mode: 'qualify',
};
}
// ---------------------------------------------------------------------------
// Environment policy validation
// ---------------------------------------------------------------------------
/**
* Check if environment policies are well-formed.
*/
export function checkEnvironmentPolicies(options: SafetyCheckOptions): SafetyCheckResult {
const { config } = options;
if (!config.environments || Object.keys(config.environments).length === 0) {
return {
name: 'environment-policies',
status: 'pass',
message: 'No environment policies configured. Using defaults.',
detail: 'Default policies: local/test allow all, production blocks qualify/chaos.',
mode: 'all',
};
}
const issues: string[] = [];
for (const [envName, policy] of Object.entries(config.environments)) {
if (!policy.name) {
issues.push(`Environment "${envName}" is missing a name field.`);
}
// Check for inconsistent policy settings
if (policy.allowQualify && policy.blockQualify) {
issues.push(`Environment "${envName}" has both allowQualify and blockQualify set.`);
}
}
if (issues.length > 0) {
return {
name: 'environment-policies',
status: 'fail',
message: `Found ${issues.length} issue(s) in environment policies.`,
detail: issues.join('\n'),
remediation: 'Fix the listed issues in your config environments section.',
mode: 'all',
};
}
return {
name: 'environment-policies',
status: 'pass',
message: `Environment policies are well-formed (${Object.keys(config.environments).length} defined).`,
mode: 'all',
};
}
// ---------------------------------------------------------------------------
// Production safety
// ---------------------------------------------------------------------------
/**
* Check production-specific safety concerns.
*/
export function checkProductionSafety(options: SafetyCheckOptions): SafetyCheckResult {
const { config, env: explicitEnv } = options;
const env = explicitEnv || detectEnvironment();
const isProd = env === 'production' || env === 'prod';
if (!isProd) {
return {
name: 'production-safety',
status: 'pass',
message: `Not in production environment (current: ${env}).`,
mode: 'all',
};
}
const warnings: string[] = [];
// Check if chaos is somehow enabled in prod
const prodPolicy = config.environments?.production || config.environments?.prod;
if (prodPolicy?.allowChaos) {
warnings.push('Chaos is explicitly allowed in production. Ensure this is intentional.');
}
// Check if blocking is enabled in prod
if (prodPolicy?.allowBlocking) {
warnings.push('Blocking behavior is explicitly allowed in production. Ensure this is intentional.');
}
if (warnings.length > 0) {
return {
name: 'production-safety',
status: 'warn',
message: 'Production environment has potentially unsafe settings.',
detail: warnings.join('\n'),
remediation: 'Review your production environment policy and disable chaos/blocking unless intentional.',
mode: 'all',
};
}
return {
name: 'production-safety',
status: 'pass',
message: 'Production environment safety checks passed.',
mode: 'all',
};
}
// ---------------------------------------------------------------------------
// Main safety check runner
// ---------------------------------------------------------------------------
/**
* Run all safety checks, filtering by mode if requested.
* When modeFilter is 'observe', skip qualify-specific checks to avoid noisy failures.
*/
export function runSafetyChecks(options: SafetyCheckOptions): SafetyCheckResult[] {
const results: SafetyCheckResult[] = [];
const { modeFilter } = options;
// Qualify-safety check: run when no filter, or when filtering for qualify
// Skip when filtering for observe (observe users don't care about qualify safety)
// Also skip when filtering for verify (verify users don't care about qualify safety)
if (modeFilter !== 'observe' && modeFilter !== 'verify') {
results.push(checkQualifySafety(options));
}
results.push(checkEnvironmentPolicies(options));
results.push(checkProductionSafety(options));
return results;
}
+491
View File
@@ -0,0 +1,491 @@
/**
* S8: Doctor thread - Main command handler
*
* Responsibilities:
* - Run all diagnostic checks (dependencies, config, routes, safety, docs)
* - Aggregate results with clear pass/fail output
* - Monorepo per-package reporting
* - Exit 0 if all pass, 2 if any fail
* - Mode-scoped checks: --mode verify|observe|qualify filters checks
* - Explicit --config honored uniformly
* - Warnings do not fail unless --strict is passed
*/
import type { CliContext } from '../../core/context.js';
import { loadConfig, detectMonorepo, findWorkspacePackages } from '../../core/config-loader.js';
import { detectEnvironment } from '../../core/policy-engine.js';
import { SUCCESS, USAGE_ERROR } from '../../core/exit-codes.js';
import type { WorkspaceResult, WorkspaceRun } from '../../core/types.js';
import { runWorkspace, formatWorkspaceHuman, formatWorkspaceJson, formatWorkspaceNdjson } from '../../core/workspace-runner.js';
import { runDependencyChecks } from './checks/dependencies.js';
import { runConfigChecks } from './checks/config.js';
import { runRouteChecks } from './checks/routes.js';
import { runSafetyChecks } from './checks/safety.js';
import { runDocsChecks } from './checks/docs.js';
import { renderJson } from '../../renderers/json.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export type DoctorMode = 'verify' | 'observe' | 'qualify' | undefined;
export interface DoctorOptions {
config?: string;
cwd?: string;
format?: 'human' | 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary';
quiet?: boolean;
verbose?: boolean;
mode?: DoctorMode;
strict?: boolean;
}
export interface DoctorCheck {
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
remediation?: string;
mode?: 'all' | 'verify' | 'observe' | 'qualify';
package?: string;
}
export interface DoctorResult {
exitCode: number;
message?: string;
checks: DoctorCheck[];
summary: {
total: number;
passed: number;
failed: number;
warnings: number;
};
}
// ---------------------------------------------------------------------------
// Check filtering
// ---------------------------------------------------------------------------
function shouldRunCheck(checkMode: string | undefined, modeFilter: DoctorMode): boolean {
if (!modeFilter) return true;
if (!checkMode || checkMode === 'all') return true;
return checkMode === modeFilter;
}
// ---------------------------------------------------------------------------
// Monorepo detection
// ---------------------------------------------------------------------------
/**
* Find all packages in a monorepo.
*/
import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
import { resolve } from 'node:path';
function findMonorepoPackages(cwd: string): string[] {
const pkgPath = resolve(cwd, 'package.json');
if (!existsSync(pkgPath)) {
return [];
}
try {
const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
const workspaces = pkg.workspaces;
if (!workspaces || !Array.isArray(workspaces)) {
return [];
}
const packages: string[] = [];
for (const pattern of workspaces) {
if (pattern.endsWith('/*')) {
const dir = pattern.slice(0, -2);
const dirPath = resolve(cwd, dir);
if (existsSync(dirPath)) {
const entries = readdirSync(dirPath, { withFileTypes: true });
for (const entry of entries) {
if (entry.isDirectory()) {
packages.push(resolve(dirPath, entry.name));
}
}
}
} else {
// Handle exact paths like "packages/api"
const exactPath = resolve(cwd, pattern);
if (existsSync(exactPath)) {
const stat = statSync(exactPath);
if (stat.isDirectory()) {
packages.push(exactPath);
}
}
}
}
return packages;
} catch {
return [];
}
}
// ---------------------------------------------------------------------------
// Check runners per package
// ---------------------------------------------------------------------------
/**
* Run all checks for a single package directory.
* Honors explicit configPath and mode filter.
*/
async function runPackageChecks(
cwd: string,
ctx: CliContext,
configPath: string | undefined,
modeFilter: DoctorMode,
packageName?: string,
): Promise<DoctorCheck[]> {
const checks: DoctorCheck[] = [];
// 1. Dependency checks (all modes)
const depResults = runDependencyChecks({
cwd,
nodeVersion: process.version,
});
for (const result of depResults) {
checks.push({ ...result, package: packageName });
}
// 2. Config checks (all modes) — honor explicit configPath
const configResults = await runConfigChecks({ cwd, configPath });
for (const result of configResults) {
checks.push({ ...result, package: packageName });
}
// 3. Route checks (all modes)
const routeResults = await runRouteChecks({ cwd, configPath });
for (const result of routeResults) {
checks.push({ ...result, package: packageName });
}
// 4. Safety checks (mode-scoped) — need loaded config, honor explicit configPath
try {
const loadResult = await loadConfig({ cwd, configPath });
const env = detectEnvironment();
const safetyResults = runSafetyChecks({
cwd,
config: loadResult.config,
env,
modeFilter,
});
for (const result of safetyResults) {
checks.push({ ...result, package: packageName });
}
} catch {
// If config can't be loaded, add a safety check note only if not filtering for observe
if (!modeFilter || modeFilter !== 'observe') {
checks.push({
name: 'safety-checks',
status: 'warn',
message: 'Could not run safety checks (config failed to load).',
mode: 'all',
package: packageName,
});
}
}
// 5. Docs checks (all modes)
const docsResults = runDocsChecks({
cwd,
isCI: ctx.isCI,
});
for (const result of docsResults) {
checks.push({ ...result, package: packageName });
}
// 6. Determinism trust signal
const testSeed = Math.floor(Math.random() * 0x7fffffff);
checks.push({
name: 'determinism',
status: 'pass',
message: `Environment supports deterministic replay (test seed: ${testSeed})`,
detail: `Run with --seed ${testSeed} to reproduce the exact same test sequence`,
mode: 'all',
package: packageName,
});
return checks;
}
// ---------------------------------------------------------------------------
// Output formatting
// ---------------------------------------------------------------------------
/**
* Format check results for human-readable output.
* Each check shows: name, status, message, mode relevance, remediation.
*/
function formatHumanOutput(result: DoctorResult, isMonorepo: boolean, modeFilter?: DoctorMode): string {
const lines: string[] = [];
lines.push('APOPHIS Doctor');
if (modeFilter) {
lines.push(`Mode: ${modeFilter}`);
}
lines.push('');
if (isMonorepo && result.checks.some(c => c.package)) {
// Group by package
const packages = new Map<string | undefined, DoctorCheck[]>();
for (const check of result.checks) {
const pkg = check.package || 'root';
if (!packages.has(pkg)) {
packages.set(pkg, []);
}
packages.get(pkg)!.push(check);
}
for (const [pkg, checks] of packages) {
lines.push(`📦 ${pkg}`);
lines.push('');
for (const check of checks) {
const icon = check.status === 'pass' ? '✓' : check.status === 'warn' ? '⚠' : '✗';
const modeLabel = check.mode === 'all' ? '' : ` [${check.mode}]`;
lines.push(` ${icon} ${check.name}${modeLabel}: ${check.message}`);
if (check.detail) {
lines.push(` ${check.detail}`);
}
if (check.remediation) {
lines.push(`${check.remediation}`);
}
}
lines.push('');
}
} else {
// Flat list
for (const check of result.checks) {
const icon = check.status === 'pass' ? '✓' : check.status === 'warn' ? '⚠' : '✗';
const modeLabel = check.mode === 'all' ? '' : ` [${check.mode}]`;
lines.push(` ${icon} ${check.name}${modeLabel}: ${check.message}`);
if (check.detail) {
lines.push(` ${check.detail}`);
}
if (check.remediation) {
lines.push(`${check.remediation}`);
}
}
lines.push('');
}
// Summary
const { summary } = result;
lines.push(`Summary: ${summary.passed} passed, ${summary.failed} failed, ${summary.warnings} warnings`);
if (summary.failed > 0) {
lines.push('');
lines.push('Run "apophis migrate" to fix legacy config issues.');
lines.push('Run "apophis init" to scaffold missing configuration.');
}
return lines.join('\n');
}
// ---------------------------------------------------------------------------
// Main command handler
// ---------------------------------------------------------------------------
/**
* Main doctor command handler.
*
* Flow:
* 1. Parse mode and strict flags
* 2. Detect if monorepo
* 3. Run checks for root package (honoring explicit configPath)
* 4. If monorepo, run checks for each workspace package
* 5. Aggregate results
* 6. Format output
* 7. Return exit code (warnings fail only if --strict)
*/
export async function doctorCommand(
options: DoctorOptions,
ctx: CliContext,
): Promise<DoctorResult> {
const { config: configPath, cwd, mode: modeFilter, strict } = options;
const workingDir = cwd || ctx.cwd;
try {
// Detect monorepo
const isMonorepo = detectMonorepo(workingDir);
const allChecks: DoctorCheck[] = [];
// Run checks for root — pass explicit configPath so every check uses it
const rootChecks = await runPackageChecks(
workingDir,
ctx,
configPath,
modeFilter,
isMonorepo ? 'root' : undefined,
);
allChecks.push(...rootChecks);
// If monorepo, run checks for each package
if (isMonorepo) {
const packages = findMonorepoPackages(workingDir);
for (const pkgPath of packages) {
const pkgName = pkgPath.split('/').pop() || 'unknown';
const pkgChecks = await runPackageChecks(pkgPath, ctx, configPath, modeFilter, pkgName);
allChecks.push(...pkgChecks);
}
}
// Calculate summary
const passed = allChecks.filter(c => c.status === 'pass').length;
const failed = allChecks.filter(c => c.status === 'fail').length;
const warnings = allChecks.filter(c => c.status === 'warn').length;
// Warnings fail the run only when --strict is passed
const effectiveFailed = failed + (strict ? warnings : 0);
const result: DoctorResult = {
exitCode: effectiveFailed > 0 ? USAGE_ERROR : SUCCESS,
checks: allChecks,
summary: {
total: allChecks.length,
passed,
failed,
warnings,
},
};
// Format message
result.message = formatHumanOutput(result, isMonorepo, modeFilter);
return result;
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return {
exitCode: USAGE_ERROR,
message: `Doctor command failed: ${message}`,
checks: [],
summary: { total: 0, passed: 0, failed: 0, warnings: 0 },
};
}
}
// ---------------------------------------------------------------------------
// CLI adapter
// ---------------------------------------------------------------------------
/**
* Adapter that bridges the CLI framework (cac) to the doctor command handler.
* Parses --mode and --strict from CLI args.
*/
export async function handleDoctor(
args: string[],
ctx: CliContext,
): Promise<number> {
// Parse --mode and --strict from raw args (cac doesn't expose unknown flags nicely)
const modeFlag = args.find(a => a.startsWith('--mode='));
const modeArg = args.find(a => a === '--mode');
const modeIndex = modeArg ? args.indexOf(modeArg) : -1;
let mode: DoctorMode = undefined;
if (modeFlag) {
const value = modeFlag.split('=')[1];
if (value === 'verify' || value === 'observe' || value === 'qualify') {
mode = value;
}
} else if (modeIndex >= 0 && args[modeIndex + 1]) {
const value = args[modeIndex + 1];
if (value === 'verify' || value === 'observe' || value === 'qualify') {
mode = value;
}
}
const strict = args.includes('--strict');
const options: DoctorOptions = {
config: ctx.options.config || undefined,
cwd: ctx.cwd,
format: ctx.options.format as Exclude<DoctorOptions['format'], undefined>,
quiet: ctx.options.quiet,
verbose: ctx.options.verbose,
mode,
strict,
};
const workspaceMode = args.includes('--workspace');
if (workspaceMode) {
const workspaceResult = await runWorkspace(
{
runCommand: async (pkgCtx) => {
const pkgOptions = { ...options, cwd: pkgCtx.cwd };
const pkgResult = await doctorCommand(pkgOptions, pkgCtx);
return {
exitCode: pkgResult.exitCode,
artifact: {
version: 'apophis-artifact/1',
command: 'doctor',
cwd: pkgCtx.cwd,
startedAt: new Date().toISOString(),
durationMs: 0,
summary: {
total: pkgResult.summary.total,
passed: pkgResult.summary.passed,
failed: pkgResult.summary.failed,
},
failures: [],
artifacts: [],
warnings: pkgResult.checks
.filter(c => c.status === 'warn' || c.status === 'fail')
.map(c => `${c.name}: ${c.message}`),
exitReason: pkgResult.exitCode === SUCCESS ? 'success' : 'behavioral_failure',
},
warnings: pkgResult.checks
.filter(c => c.status === 'warn')
.map(c => c.message),
};
},
},
ctx,
);
if (!ctx.options.quiet) {
const format = options.format || ctx.options.format || 'human';
if (format === 'json') {
console.log(formatWorkspaceJson(workspaceResult));
} else if (format === 'ndjson') {
console.log(formatWorkspaceNdjson(workspaceResult));
} else {
console.log(formatWorkspaceHuman(workspaceResult));
}
}
return workspaceResult.exitCode;
}
const result = await doctorCommand(options, ctx);
// Output result based on format
if (!ctx.options.quiet && result.message) {
const format = options.format || ctx.options.format || 'human';
if (format === 'json') {
console.log(renderJson({
exitCode: result.exitCode,
summary: result.summary,
checks: result.checks,
}));
} else if (format === 'ndjson') {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'doctor',
exitCode: result.exitCode,
summary: result.summary,
checks: result.checks,
}) + '\n');
} else {
console.log(result.message);
}
}
return result.exitCode;
}
+644
View File
@@ -0,0 +1,644 @@
/**
* S3: Init command for APOPHIS CLI
* Scaffold config, scripts, and example usage in one pass.
*/
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
import { resolve } from 'node:path';
import type { CliContext } from '../../core/types.js';
import { USAGE_ERROR, SUCCESS } from '../../core/exit-codes.js';
import { getScaffoldForPreset, getPresetNames, type ScaffoldResult } from './scaffolds/index.js';
// ─────────────────────────────────────────────────────────────────────────────
// Types
// ─────────────────────────────────────────────────────────────────────────────
export interface InitOptions {
preset?: string;
force?: boolean;
noninteractive?: boolean;
cwd?: string;
}
export interface InitResult {
exitCode: number;
message: string;
filesWritten: string[];
nextCommand: string;
}
const DEFAULT_INSTALL_PM: Exclude<CliContext['packageManager'], 'unknown'> = 'npm';
function normalizePackageManager(packageManager: CliContext['packageManager'] | undefined): Exclude<CliContext['packageManager'], 'unknown'> {
if (!packageManager || packageManager === 'unknown') {
return DEFAULT_INSTALL_PM;
}
return packageManager;
}
function renderInstallCommand(
packageManager: CliContext['packageManager'] | undefined,
packages: string[],
): string {
const normalized = normalizePackageManager(packageManager);
if (normalized === 'yarn') {
return `yarn add ${packages.join(' ')}`;
}
if (normalized === 'pnpm') {
return `pnpm add ${packages.join(' ')}`;
}
if (normalized === 'bun') {
return `bun add ${packages.join(' ')}`;
}
return `npm install ${packages.join(' ')}`;
}
// ─────────────────────────────────────────────────────────────────────────────
// Fastify detection
// ─────────────────────────────────────────────────────────────────────────────
/**
* Detect if the project is a Fastify app by looking for:
* - fastify imports in JS/TS files
* - Common server file names (server.js, app.js, index.js, etc.)
*/
export async function detectFastifyEntrypoint(cwd: string): Promise<string | null> {
const candidates = [
'app.js',
'app.ts',
'server.js',
'server.ts',
'index.js',
'index.ts',
'src/app.js',
'src/app.ts',
'src/server.js',
'src/server.ts',
'src/index.js',
'src/index.ts',
];
for (const candidate of candidates) {
const fullPath = resolve(cwd, candidate);
if (!existsSync(fullPath)) continue;
const content = readFileSync(fullPath, 'utf-8');
// Look for fastify import patterns
if (
content.includes('fastify') ||
content.includes('Fastify') ||
content.includes('@fastify') ||
content.includes('fastify-plugin')
) {
return candidate;
}
}
return null;
}
/**
* Check if @fastify/swagger is registered in the project.
* We check package.json dependencies and the entrypoint file.
*/
export async function checkSwaggerRegistration(cwd: string, entrypoint: string | null): Promise<{
hasSwaggerDep: boolean;
hasSwaggerImport: boolean;
}> {
const pkgPath = resolve(cwd, 'package.json');
let hasSwaggerDep = false;
if (existsSync(pkgPath)) {
try {
const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
const deps = {
...pkg.dependencies,
...pkg.devDependencies,
};
hasSwaggerDep = '@fastify/swagger' in deps;
} catch {
// Ignore parse errors
}
}
let hasSwaggerImport = false;
if (entrypoint) {
const entryPath = resolve(cwd, entrypoint);
if (existsSync(entryPath)) {
const content = readFileSync(entryPath, 'utf-8');
hasSwaggerImport =
content.includes('@fastify/swagger') ||
content.includes('fastify-swagger');
}
}
return { hasSwaggerDep, hasSwaggerImport };
}
/**
* Detect if the project uses TypeScript.
*/
export function detectTypeScript(cwd: string): boolean {
return (
existsSync(resolve(cwd, 'tsconfig.json')) ||
existsSync(resolve(cwd, 'src/app.ts')) ||
existsSync(resolve(cwd, 'src/server.ts')) ||
existsSync(resolve(cwd, 'src/index.ts'))
);
}
// ─────────────────────────────────────────────────────────────────────────────
// Package.json script merging
// ─────────────────────────────────────────────────────────────────────────────
/**
* Merge apophis scripts into package.json without clobbering existing scripts.
*/
export function mergePackageScripts(pkg: Record<string, unknown>): Record<string, unknown> {
const scripts = (pkg.scripts as Record<string, string>) || {};
const apophisScripts: Record<string, string> = {
'apophis:verify': 'apophis verify --profile quick',
'apophis:doctor': 'apophis doctor',
};
const mergedScripts = { ...scripts };
for (const [key, value] of Object.entries(apophisScripts)) {
// Only add if not already present
if (!(key in mergedScripts)) {
mergedScripts[key] = value;
}
}
return {
...pkg,
scripts: mergedScripts,
};
}
// ─────────────────────────────────────────────────────────────────────────────
// File writing
// ─────────────────────────────────────────────────────────────────────────────
/**
* Write the config file (apophis.config.js or .ts).
*/
export function writeConfigFile(
cwd: string,
scaffold: ScaffoldResult,
isTypeScript: boolean,
force: boolean,
): { path: string; existed: boolean } {
const ext = isTypeScript ? 'ts' : 'js';
const configPath = resolve(cwd, `apophis.config.${ext}`);
const existed = existsSync(configPath);
if (existed && !force) {
return { path: configPath, existed: true };
}
const configContent = generateConfigContent(scaffold.config, isTypeScript);
writeFileSync(configPath, configContent, 'utf-8');
return { path: configPath, existed: false };
}
/**
* Generate config file content as a formatted string.
*/
function generateConfigContent(config: ScaffoldResult['config'], isTypeScript: boolean): string {
const lines: string[] = [];
lines.push('/**');
lines.push(' * APOPHIS configuration');
lines.push(' * Generated by `apophis init`');
lines.push(' */');
lines.push('');
if (isTypeScript) {
lines.push('import type { ApophisConfig } from "apophis-fastify/cli";');
lines.push('');
lines.push('const config: ApophisConfig = ' + stringifyConfig(config) + ';');
lines.push('');
lines.push('export default config;');
} else {
lines.push('export default ' + stringifyConfig(config) + ';');
}
lines.push('');
return lines.join('\n');
}
/**
* Stringify a config object with proper indentation.
*/
function stringifyConfig(obj: unknown, indent = 2): string {
if (obj === null) return 'null';
if (typeof obj === 'string') return JSON.stringify(obj);
if (typeof obj === 'number') return String(obj);
if (typeof obj === 'boolean') return String(obj);
if (Array.isArray(obj)) {
if (obj.length === 0) return '[]';
const items = obj.map(item => stringifyConfig(item, indent + 2)).join(',\n' + ' '.repeat(indent));
return '[\n' + ' '.repeat(indent) + items + '\n' + ' '.repeat(indent - 2) + ']';
}
if (typeof obj === 'object') {
const entries = Object.entries(obj as Record<string, unknown>);
if (entries.length === 0) return '{}';
const items = entries
.map(([key, value]) => {
const keyStr = /^[a-zA-Z_$][a-zA-Z0-9_$]*$/.test(key) ? key : JSON.stringify(key);
return `${keyStr}: ${stringifyConfig(value, indent + 2)}`;
})
.join(',\n' + ' '.repeat(indent));
return '{\n' + ' '.repeat(indent) + items + '\n' + ' '.repeat(indent - 2) + '}';
}
return String(obj);
}
/**
* Write the README guidance file.
*/
export function writeReadmeFile(
cwd: string,
scaffold: ScaffoldResult,
force: boolean,
): { path: string; existed: boolean } {
const readmePath = resolve(cwd, 'APOPHIS.md');
const existed = existsSync(readmePath);
if (existed && !force) {
return { path: readmePath, existed: true };
}
writeFileSync(readmePath, scaffold.readmeContent.trim() + '\n', 'utf-8');
return { path: readmePath, existed: false };
}
/**
* Update package.json with merged scripts.
*/
export function updatePackageJson(cwd: string): { path: string; modified: boolean; error?: string } {
const pkgPath = resolve(cwd, 'package.json');
if (!existsSync(pkgPath)) {
const bootstrapPackage = {
name: 'apophis-app',
version: '0.1.0',
private: true,
type: 'module',
scripts: {
'apophis:doctor': 'apophis doctor',
'apophis:verify': 'apophis verify --profile quick',
},
dependencies: {
fastify: '^5.0.0',
'@fastify/swagger': '^9.0.0',
},
};
try {
writeFileSync(pkgPath, JSON.stringify(bootstrapPackage, null, 2) + '\n', 'utf-8');
return { path: pkgPath, modified: true };
} catch (err) {
return { path: pkgPath, modified: false, error: String(err) };
}
}
try {
const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
const merged = mergePackageScripts(pkg);
// Check if anything changed
const originalScripts = JSON.stringify(pkg.scripts || {});
const mergedScripts = JSON.stringify(merged.scripts || {});
if (originalScripts === mergedScripts) {
return { path: pkgPath, modified: false };
}
writeFileSync(pkgPath, JSON.stringify(merged, null, 2) + '\n', 'utf-8');
return { path: pkgPath, modified: true };
} catch (err) {
return { path: pkgPath, modified: false, error: String(err) };
}
}
export function writeBootstrapAppFile(
cwd: string,
existingEntrypoint: string | null,
): { path: string; created: boolean } {
const appPath = resolve(cwd, 'app.js');
if (existingEntrypoint || existsSync(appPath)) {
return { path: appPath, created: false };
}
const appContent = `/**
* Generated by \`apophis init\`.
* This is a minimal Fastify-like app that is runnable with \`apophis verify\`.
*/
const routes = [
{
method: 'POST',
url: '/users',
schema: {
body: {
type: 'object',
required: ['name'],
properties: {
name: { type: 'string' },
},
},
response: {
201: {
type: 'object',
properties: {
id: { type: 'string' },
name: { type: 'string' },
},
},
},
'x-ensures': [
'response_code(this) == 201',
],
},
},
];
const app = {
routes,
async ready() {},
hasRoute({ method, url }) {
const normalizedMethod = String(method || '').toUpperCase();
return routes.some(route => route.method === normalizedMethod && route.url === url);
},
async inject({ method, url, payload }) {
const normalizedMethod = String(method || '').toUpperCase();
if (normalizedMethod === 'POST' && url === '/users') {
const body = {
id: 'usr-1',
name: payload && typeof payload === 'object' && 'name' in payload
? String(payload.name)
: 'test',
};
return {
statusCode: 201,
headers: { 'content-type': 'application/json' },
body,
json() {
return body;
},
};
}
const body = { message: 'not found' };
return {
statusCode: 404,
headers: { 'content-type': 'application/json' },
body,
json() {
return body;
},
};
},
};
export default app;
`;
writeFileSync(appPath, appContent, 'utf-8');
return { path: appPath, created: true };
}
// ─────────────────────────────────────────────────────────────────────────────
// Interactive prompts (lazy-loaded)
// ─────────────────────────────────────────────────────────────────────────────
interface PromptsModule {
select: (opts: { message: string; options: { value: string; label: string }[] }) => Promise<string>;
confirm: (opts: { message: string }) => Promise<boolean>;
text: (opts: { message: string; placeholder?: string }) => Promise<string>;
}
async function loadPrompts(): Promise<PromptsModule> {
// Lazy-load @clack/prompts only when interactive
const mod = await import('@clack/prompts');
return mod as unknown as PromptsModule;
}
// ─────────────────────────────────────────────────────────────────────────────
// Main init handler
// ─────────────────────────────────────────────────────────────────────────────
export async function initHandler(args: string[], ctx: CliContext): Promise<InitResult> {
const options = parseInitOptions(args, ctx);
const cwd = options.cwd || ctx.cwd;
// Detect project structure
const isTypeScript = detectTypeScript(cwd);
const fastifyEntry = await detectFastifyEntrypoint(cwd);
const swaggerCheck = await checkSwaggerRegistration(cwd, fastifyEntry);
// Determine preset
let preset = options.preset;
if (!preset) {
if (options.noninteractive) {
return {
exitCode: USAGE_ERROR,
message: 'Missing required --preset flag in non-interactive mode. Use one of: ' + getPresetNames().join(', '),
filesWritten: [],
nextCommand: '',
};
}
// Interactive mode: prompt for preset
if (ctx.isTTY && !ctx.isCI) {
try {
const prompts = await loadPrompts();
const presetNames = getPresetNames();
const choice = await prompts.select({
message: 'Choose a preset:',
options: presetNames.map(name => ({
value: name,
label: name,
})),
});
preset = choice;
} catch {
// Fallback if prompts fail
return {
exitCode: USAGE_ERROR,
message: 'Failed to prompt for preset. Use --preset <name> in non-interactive mode.',
filesWritten: [],
nextCommand: '',
};
}
} else {
// Non-TTY, non-CI: default to safe-ci
preset = 'safe-ci';
}
}
// Validate preset
const scaffold = getScaffoldForPreset(preset);
if (!scaffold) {
return {
exitCode: USAGE_ERROR,
message: `Unknown preset "${preset}". Available presets: ${getPresetNames().join(', ')}`,
filesWritten: [],
nextCommand: '',
};
}
// Check for existing config
const configExt = isTypeScript ? 'ts' : 'js';
const configPath = resolve(cwd, `apophis.config.${configExt}`);
const configExisted = existsSync(configPath);
if (configExisted && !options.force) {
return {
exitCode: USAGE_ERROR,
message: `Config file already exists: apophis.config.${configExt}. Use --force to overwrite.`,
filesWritten: [],
nextCommand: '',
};
}
// Write files
const filesWritten: string[] = [];
const forceWrite = options.force ?? false;
const configResult = writeConfigFile(cwd, scaffold, isTypeScript, forceWrite);
if (configResult.existed && !forceWrite) {
return {
exitCode: USAGE_ERROR,
message: `Config file already exists: ${configResult.path}. Use --force to overwrite.`,
filesWritten: [],
nextCommand: '',
};
}
filesWritten.push(configResult.path);
const readmeResult = writeReadmeFile(cwd, scaffold, forceWrite);
if (!readmeResult.existed || forceWrite) {
filesWritten.push(readmeResult.path);
}
const pkgResult = updatePackageJson(cwd);
if (pkgResult.modified) {
filesWritten.push(pkgResult.path);
}
const bootstrapAppResult = writeBootstrapAppFile(cwd, fastifyEntry);
if (bootstrapAppResult.created) {
filesWritten.push(bootstrapAppResult.path);
}
// Build next command
const profileName = scaffold.config.profile || 'quick';
const routeHint = scaffold.config.routes?.[0] || '';
const nextCommand = routeHint
? `apophis verify --profile ${profileName} --routes "${routeHint}"`
: `apophis verify --profile ${profileName}`;
// Build message
const lines: string[] = [];
lines.push(`Initialized APOPHIS with preset "${preset}"`);
lines.push('');
lines.push('Files written:');
for (const file of filesWritten) {
lines.push(` ${file}`);
}
const installPeerDepsCommand = renderInstallCommand(ctx.packageManager, ['fastify', '@fastify/swagger']);
const installSwaggerCommand = renderInstallCommand(ctx.packageManager, ['@fastify/swagger']);
lines.push('');
lines.push('First success path:');
lines.push(` 1. ${installPeerDepsCommand}`);
lines.push(' 2. apophis doctor');
lines.push(` 3. ${nextCommand}`);
lines.push('');
lines.push('If verify says "No behavioral contracts found", add x-ensures to your route schema:');
lines.push(' "x-ensures": [');
lines.push(' "response_code(GET /users/{response_body(this).id}) == 200"');
lines.push(' ]');
lines.push('');
lines.push('See APOPHIS.md and docs/getting-started.md for full examples.');
if (!swaggerCheck.hasSwaggerDep && !bootstrapAppResult.created) {
lines.push('');
lines.push('Warning: @fastify/swagger not found in dependencies.');
lines.push('APOPHIS requires @fastify/swagger to discover routes.');
lines.push('Install it with:');
lines.push(` ${installSwaggerCommand}`);
} else if (!bootstrapAppResult.created && !swaggerCheck.hasSwaggerImport) {
lines.push('');
lines.push('Warning: @fastify/swagger is installed but not imported in your entrypoint.');
lines.push('Register it in your Fastify app:');
lines.push(` await app.register(import("@fastify/swagger"), { openapi: { info: { title: "API", version: "1.0.0" } } });`);
}
if (fastifyEntry) {
lines.push('');
lines.push(`Detected Fastify entrypoint: ${fastifyEntry}`);
}
lines.push('');
lines.push('Next command:');
lines.push(` ${nextCommand}`);
return {
exitCode: SUCCESS,
message: lines.join('\n'),
filesWritten,
nextCommand,
};
}
// ─────────────────────────────────────────────────────────────────────────────
// Option parsing
// ─────────────────────────────────────────────────────────────────────────────
function parseInitOptions(args: string[], ctx: CliContext): InitOptions {
const options: InitOptions = {};
for (let i = 0; i < args.length; i++) {
const arg = args[i];
if (arg === '--preset' || arg === '-p') {
options.preset = args[++i];
} else if (arg === '--force' || arg === '-f') {
options.force = true;
} else if (arg === '--noninteractive') {
options.noninteractive = true;
} else if (arg === '--cwd') {
options.cwd = args[++i];
}
}
// Non-interactive if CI or not TTY
if (ctx.isCI || !ctx.isTTY) {
options.noninteractive = true;
}
return options;
}
// ─────────────────────────────────────────────────────────────────────────────
// CLI adapter
// ─────────────────────────────────────────────────────────────────────────────
export async function handleInit(args: string[], ctx: CliContext): Promise<number> {
const result = await initHandler(args, ctx);
if (result.message) {
console.log(result.message);
}
return result.exitCode;
}
+374
View File
@@ -0,0 +1,374 @@
/**
* S3: Init command scaffold templates
* Each preset returns a config object and file contents for the init command.
*/
import type { ApophisConfig, PresetDefinition, ProfileDefinition, EnvironmentPolicy } from '../../../core/types.js';
export interface ScaffoldResult {
config: ApophisConfig;
readmeContent: string;
}
// ─────────────────────────────────────────────────────────────────────────────
// safe-ci: Minimal CI-safe preset (default)
// ─────────────────────────────────────────────────────────────────────────────
export function safeCiScaffold(): ScaffoldResult {
const preset: PresetDefinition = {
name: 'safe-ci',
depth: 'quick',
timeout: 5000,
parallel: false,
chaos: false,
observe: false,
};
const profile: ProfileDefinition = {
name: 'quick',
mode: 'verify',
preset: 'safe-ci',
routes: ['POST /users'],
};
const envLocal: EnvironmentPolicy = {
name: 'local',
allowVerify: true,
allowObserve: true,
allowQualify: false,
allowChaos: false,
allowBlocking: true,
requireSink: false,
};
const config: ApophisConfig = {
mode: 'verify',
profiles: { quick: profile },
presets: { 'safe-ci': preset },
environments: { local: envLocal },
};
const readmeContent = `
# APOPHIS Setup — safe-ci preset
This project was scaffolded with \`apophis init --preset safe-ci\`.
## Quick Start
1. Ensure you have a Fastify app with @fastify/swagger registered.
2. Add behavioral contracts to your route schemas using \`x-ensures\`.
3. Run: apophis verify --profile quick
## What This Preset Does
- Runs only behavioral contracts (not schema-only routes).
- No chaos, no observe, no stateful testing.
- Safe for CI pipelines.
- Timeout: 5s per route.
## Example Behavioral Contract
Add this inside your route schema to check that a created resource is retrievable:
\`\`\`javascript
"x-ensures": [
"response_code(GET /users/{response_body(this).id}) == 200"
]
\`\`\`
If \`apophis verify\` says "No behavioral contracts found", it means your routes have schemas but no \`x-ensures\` or \`x-requires\` clauses. Add at least one clause per route you want to verify.
## Next Steps
- Add more routes to the \`routes\` array in your profile.
- Try \`apophis init --preset platform-observe\` for production readiness.
- Try \`apophis init --preset protocol-lab\` for multi-step flows.
`;
return { config, readmeContent };
}
// ─────────────────────────────────────────────────────────────────────────────
// platform-observe: Production-ready with observe mode
// ─────────────────────────────────────────────────────────────────────────────
export function platformObserveScaffold(): ScaffoldResult {
const preset: PresetDefinition = {
name: 'platform-observe',
depth: 'standard',
timeout: 10000,
parallel: true,
chaos: false,
observe: true,
};
const profile: ProfileDefinition = {
name: 'staging-observe',
mode: 'observe',
preset: 'platform-observe',
routes: [],
};
const envStaging: EnvironmentPolicy = {
name: 'staging',
allowVerify: true,
allowObserve: true,
allowQualify: true,
allowChaos: false,
allowBlocking: false,
requireSink: true,
};
const envProduction: EnvironmentPolicy = {
name: 'production',
allowVerify: true,
allowObserve: true,
allowQualify: false,
allowChaos: false,
allowBlocking: false,
requireSink: true,
};
const config: ApophisConfig = {
mode: 'observe',
profile: 'staging-observe',
profiles: { 'staging-observe': profile },
presets: { 'platform-observe': preset },
environments: {
staging: envStaging,
production: envProduction,
},
};
const readmeContent = `
# APOPHIS Setup — platform-observe preset
This project was scaffolded with \`apophis init --preset platform-observe\`.
## Quick Start
1. Ensure you have a Fastify app with @fastify/swagger registered.
2. Configure your reporting sink (see environments.staging.requireSink).
3. Run: apophis observe --profile staging-observe
## What This Preset Does
- Enables observe mode for production readiness checks.
- Validates non-blocking semantics and sink configuration.
- Parallel execution for faster feedback.
- Requires sink config in staging/production.
## Safety
- Observe mode is non-blocking by default.
- Production requires explicit policy to enable blocking.
- Chaos is disabled in this preset.
## Next Steps
- Add a sink configuration to your environment policy.
- Run \`apophis doctor\` to validate the full setup.
`;
return { config, readmeContent };
}
// ─────────────────────────────────────────────────────────────────────────────
// llm-safe: Minimal preset for LLM-generated codebases
// ─────────────────────────────────────────────────────────────────────────────
export function llmSafeScaffold(): ScaffoldResult {
const preset: PresetDefinition = {
name: 'llm-safe',
depth: 'quick',
timeout: 3000,
parallel: false,
chaos: false,
observe: false,
};
const profile: ProfileDefinition = {
name: 'llm-check',
mode: 'verify',
preset: 'llm-safe',
routes: [],
};
const envLocal: EnvironmentPolicy = {
name: 'local',
allowVerify: true,
allowObserve: false,
allowQualify: false,
allowChaos: false,
allowBlocking: false,
requireSink: false,
};
const config: ApophisConfig = {
mode: 'verify',
profile: 'llm-check',
profiles: { 'llm-check': profile },
presets: { 'llm-safe': preset },
environments: { local: envLocal },
};
const readmeContent = `
# APOPHIS Setup — llm-safe preset
This project was scaffolded with \`apophis init --preset llm-safe\`.
## Quick Start
1. Ensure you have a Fastify app with @fastify/swagger registered.
2. Add behavioral contracts to your route schemas using \`x-ensures\`.
3. Run: apophis verify --profile llm-check
## What This Preset Does
- Ultra-minimal preset for LLM-generated codebases.
- 3s timeout per route (fast feedback).
- No observe, no qualify, no chaos — verify only.
- Conservative defaults to avoid surprising failures.
## Example Behavioral Contract
Add this inside your route schema to check that a created resource is retrievable:
\`\`\`javascript
"x-ensures": [
"response_code(GET /users/{response_body(this).id}) == 200"
]
\`\`\`
If \`apophis verify\` says "No behavioral contracts found", it means your routes have schemas but no \`x-ensures\` or \`x-requires\` clauses. Add at least one clause per route you want to verify.
## Next Steps
- Add routes to the \`routes\` array once you have behavioral contracts.
- Run \`apophis doctor\` to check for missing dependencies.
`;
return { config, readmeContent };
}
// ─────────────────────────────────────────────────────────────────────────────
// protocol-lab: Multi-step flow and stateful testing
// ─────────────────────────────────────────────────────────────────────────────
export function protocolLabScaffold(): ScaffoldResult {
const preset: PresetDefinition = {
name: 'protocol-lab',
depth: 'deep',
timeout: 15000,
parallel: false,
chaos: true,
observe: false,
};
const profile: ProfileDefinition = {
name: 'oauth-nightly',
mode: 'qualify',
preset: 'protocol-lab',
routes: [],
seed: 42,
};
const envLocal: EnvironmentPolicy = {
name: 'local',
allowVerify: true,
allowObserve: true,
allowQualify: true,
allowChaos: true,
allowBlocking: true,
requireSink: false,
};
const envTest: EnvironmentPolicy = {
name: 'test',
allowVerify: true,
allowObserve: true,
allowQualify: true,
allowChaos: true,
allowBlocking: true,
requireSink: false,
};
const config: ApophisConfig = {
mode: 'qualify',
profile: 'oauth-nightly',
profiles: { 'oauth-nightly': profile },
presets: { 'protocol-lab': preset },
environments: {
local: envLocal,
test: envTest,
},
};
const readmeContent = `
# APOPHIS Setup — protocol-lab preset
This project was scaffolded with \`apophis init --preset protocol-lab\`.
## Quick Start
1. Ensure you have a Fastify app with @fastify/swagger registered.
2. Define multi-step flows in your route schemas.
3. Run: apophis qualify --profile oauth-nightly --seed 42
## What This Preset Does
- Enables qualify mode for stateful and scenario testing.
- Chaos engineering enabled (local/test only).
- Deep depth for thorough exploration.
- 15s timeout per route.
## Safety
- Chaos is blocked in production by default.
- Use \`apophis doctor\` to validate environment safety before qualifying.
## Machine Output in CI
Qualify can produce large output. In CI, use machine-readable formats and filter events:
- \`--format json\` emits a single stable JSON artifact (good for small-to-medium runs).
- \`--format ndjson\` emits one event per line (good for streaming parsers).
- Use \`--quiet\` to suppress human progress text.
- Pipe ndjson to \`jq\` or a custom filter to extract only failures:
\`\`\`bash
apophis qualify --profile oauth-nightly --format ndjson | jq 'select(.type == "route.failed")'
\`\`\`
- For very large runs, consider writing artifacts to a directory and parsing the JSON file instead of stdout:
\`\`\`bash
apophis qualify --profile oauth-nightly --format json --artifact-dir reports/apophis
\`\`\`
## Next Steps
- Define scenario sequences in your config.
- Add route allowlists for chaos if needed.
- Run \`apophis replay --artifact <path>\` to debug failures.
`;
return { config, readmeContent };
}
// ─────────────────────────────────────────────────────────────────────────────
// Preset registry
// ─────────────────────────────────────────────────────────────────────────────
export const PRESETS: Record<string, () => ScaffoldResult> = {
'safe-ci': safeCiScaffold,
'platform-observe': platformObserveScaffold,
'llm-safe': llmSafeScaffold,
'protocol-lab': protocolLabScaffold,
};
export function getPresetNames(): string[] {
return Object.keys(PRESETS);
}
export function getScaffoldForPreset(preset: string): ScaffoldResult | null {
const fn = PRESETS[preset];
return fn ? fn() : null;
}
+610
View File
@@ -0,0 +1,610 @@
/**
* S9: Migrate thread - Config migration command
*
* Responsibilities:
* - Detect legacy config patterns and deprecated API usage
* - Support --check (detect only, don't write)
* - Support --dry-run (show rewrites without writing) - DEFAULT
* - Support --write (perform rewrites)
* - Map legacy fields to new fields with exact replacements
* - Preserve comments/formatting where feasible
* - Handle ambiguous rewrites (stop, require manual choice)
* - Report completed and remaining items separately
* - Exit 0 if nothing to migrate, 2 if issues found, 1 if --write performed
* - Mixed legacy/modern config detection with clear reporting
* - Exact dry-run output with file path, line number, legacy text, replacement text
* - Ambiguous rewrite handling with surrounding context and possible resolutions
* - Safe by default: dry-run is the default mode
*
* Architecture:
* - Dependency injection: all dependencies passed explicitly
* - No optional imports — everything is required or injected
* - Inline comments for documentation
*/
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
import { resolve } from 'node:path';
import type { CliContext } from '../../core/context.js';
import { loadConfig, discoverConfig } from '../../core/config-loader.js';
import { SUCCESS, USAGE_ERROR, BEHAVIORAL_FAILURE } from '../../core/exit-codes.js';
import type { CommandResult } from '../../core/types.js';
import {
rewriteConfigFile,
detectLegacyConfigFields,
detectLegacyFieldsNoEquivalent,
detectMixedLegacyModernFields,
} from './rewriters/config-rewriter.js';
import {
rewriteRouteAnnotations,
detectLegacyRouteAnnotations,
detectAmbiguousRoutePatterns,
} from './rewriters/route-rewriter.js';
import {
rewriteCodePatterns,
detectLegacyCodePatterns,
detectAmbiguousCodePatterns,
} from './rewriters/code-rewriter.js';
import { renderJson } from '../../renderers/json.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface MigrateOptions {
check?: boolean;
dryRun?: boolean;
write?: boolean;
config?: string;
cwd?: string;
format?: 'human' | 'json' | 'ndjson';
quiet?: boolean;
verbose?: boolean;
}
export interface MigrationItem {
type: 'config-field' | 'route-annotation' | 'code-pattern';
file: string;
line?: number;
legacy: string;
replacement: string;
guidance?: string;
ambiguous?: boolean;
}
export interface MigrationResult {
exitCode: number;
items: MigrationItem[];
completed: MigrationItem[];
remaining: MigrationItem[];
message?: string;
filesModified?: string[];
filesWouldBeModified?: string[];
totalRewrites?: number;
manualChoicesRequired?: number;
}
// ---------------------------------------------------------------------------
// File discovery
// ---------------------------------------------------------------------------
/**
* Discover files that may contain legacy patterns.
* Scans the working directory for config files, route files, and code files.
*/
export async function discoverMigrationFiles(
cwd: string,
configPath?: string,
): Promise<{ configFile: string | null; appFiles: string[] }> {
const configFile = configPath
? resolve(cwd, configPath)
: discoverConfig(cwd);
const appFiles: string[] = [];
const candidates = [
'app.js',
'app.ts',
'src/app.js',
'src/app.ts',
'routes.js',
'routes.ts',
'src/routes.js',
'src/routes.ts',
];
for (const candidate of candidates) {
const fullPath = resolve(cwd, candidate);
if (existsSync(fullPath)) {
appFiles.push(fullPath);
}
}
return { configFile, appFiles };
}
// ---------------------------------------------------------------------------
// Pattern detection
// ---------------------------------------------------------------------------
/**
* Detect all legacy patterns in a set of files.
* Includes: legacy fields, route annotations, code patterns,
* ambiguous patterns, fields with no equivalent, and mixed legacy/modern fields.
*/
export async function detectAllLegacyPatterns(
configFile: string | null,
appFiles: string[],
): Promise<MigrationItem[]> {
const items: MigrationItem[] = [];
// Detect config fields
if (configFile && existsSync(configFile)) {
const configContent = readFileSync(configFile, 'utf-8');
items.push(...detectLegacyConfigFields(configContent, configFile));
items.push(...detectLegacyFieldsNoEquivalent(configContent, configFile));
items.push(...detectLegacyRouteAnnotations(configContent, configFile));
items.push(...detectAmbiguousRoutePatterns(configContent, configFile));
items.push(...detectLegacyCodePatterns(configContent, configFile));
items.push(...detectAmbiguousCodePatterns(configContent, configFile));
}
// Detect patterns in app files
for (const appFile of appFiles) {
const content = readFileSync(appFile, 'utf-8');
items.push(...detectLegacyRouteAnnotations(content, appFile));
items.push(...detectAmbiguousRoutePatterns(content, appFile));
items.push(...detectLegacyCodePatterns(content, appFile));
items.push(...detectAmbiguousCodePatterns(content, appFile));
}
return items;
}
// ---------------------------------------------------------------------------
// Migration execution
// ---------------------------------------------------------------------------
/**
* Run the migration process.
*
* Flow:
* 1. Discover config files in the working directory
* 2. Detect legacy patterns in all relevant files
* 3. If --check, report findings and exit
* 4. If --dry-run (default), show exact rewrites without writing
* 5. If --write, perform rewrites
* 6. Report completed and remaining items separately
* 7. Return appropriate exit code
*
* Safety: dry-run is the default mode. No files are modified unless --write is explicitly passed.
*/
export async function migrateCommand(
options: MigrateOptions,
ctx: CliContext,
): Promise<MigrationResult> {
const { check, dryRun, write, config: configPath, cwd } = options;
const workingDir = cwd || ctx.cwd;
// Determine mode: check < dry-run < write
// Default is dry-run (safe by default)
const mode = write ? 'write' : check ? 'check' : 'dry-run';
try {
// 1. Discover files
const { configFile, appFiles } = await discoverMigrationFiles(
workingDir,
configPath,
);
if (!configFile && appFiles.length === 0) {
return {
exitCode: USAGE_ERROR,
items: [],
completed: [],
remaining: [],
message: 'No config or app files found. Run "apophis init" to create one.',
};
}
// 2. Detect legacy patterns
const allItems = await detectAllLegacyPatterns(configFile, appFiles);
// 3. If no legacy patterns found, report success
if (allItems.length === 0) {
return {
exitCode: SUCCESS,
items: [],
completed: [],
remaining: [],
message: 'No legacy patterns detected. Config is up to date.',
};
}
// Separate ambiguous items
const ambiguousItems = allItems.filter((item) => item.ambiguous);
const unambiguousItems = allItems.filter((item) => !item.ambiguous);
// Calculate files that would be modified
const filesWouldBeModified = new Set<string>();
for (const item of allItems) {
filesWouldBeModified.add(item.file);
}
// If ambiguous items exist and we're writing, stop and require manual choice
if (ambiguousItems.length > 0 && mode === 'write') {
return {
exitCode: USAGE_ERROR,
items: allItems,
completed: [],
remaining: ambiguousItems,
message: formatAmbiguousOutput(ambiguousItems),
filesWouldBeModified: Array.from(filesWouldBeModified),
totalRewrites: allItems.length,
manualChoicesRequired: ambiguousItems.length,
};
}
// 4. Check mode: detect only
if (mode === 'check') {
return {
exitCode: BEHAVIORAL_FAILURE,
items: allItems,
completed: [],
remaining: allItems,
message: formatCheckOutput(allItems),
filesWouldBeModified: Array.from(filesWouldBeModified),
totalRewrites: allItems.length,
manualChoicesRequired: ambiguousItems.length,
};
}
// 5. Dry-run mode: show exact rewrites without writing
if (mode === 'dry-run') {
return {
exitCode: BEHAVIORAL_FAILURE,
items: allItems,
completed: [],
remaining: allItems,
message: formatDryRunOutput(allItems),
filesWouldBeModified: Array.from(filesWouldBeModified),
totalRewrites: allItems.length,
manualChoicesRequired: ambiguousItems.length,
};
}
// 6. Write mode: perform rewrites
const filesModified: string[] = [];
const completed: MigrationItem[] = [];
const remaining: MigrationItem[] = [];
// Rewrite config file
if (configFile && existsSync(configFile)) {
const configItems = unambiguousItems.filter(
(item) => item.file === configFile && item.type === 'config-field',
);
if (configItems.length > 0) {
const result = rewriteConfigFile(configFile, configItems);
if (result.modified) {
writeFileSync(configFile, result.content, 'utf-8');
filesModified.push(configFile);
completed.push(...result.itemsRewritten);
remaining.push(...result.itemsRemaining);
} else {
remaining.push(...configItems);
}
}
// Route annotations in config file
const routeItems = unambiguousItems.filter(
(item) => item.file === configFile && item.type === 'route-annotation',
);
if (routeItems.length > 0) {
const result = rewriteRouteAnnotations(configFile, routeItems);
if (result.modified) {
writeFileSync(configFile, result.content, 'utf-8');
if (!filesModified.includes(configFile)) {
filesModified.push(configFile);
}
completed.push(...result.itemsRewritten);
remaining.push(...result.itemsRemaining);
} else {
remaining.push(...routeItems);
}
}
// Code patterns in config file
const codeItems = unambiguousItems.filter(
(item) => item.file === configFile && item.type === 'code-pattern',
);
if (codeItems.length > 0) {
const result = rewriteCodePatterns(configFile, codeItems);
if (result.modified) {
writeFileSync(configFile, result.content, 'utf-8');
if (!filesModified.includes(configFile)) {
filesModified.push(configFile);
}
completed.push(...result.itemsRewritten);
remaining.push(...result.itemsRemaining);
} else {
remaining.push(...codeItems);
}
}
}
// Rewrite app files
for (const appFile of appFiles) {
const fileItems = unambiguousItems.filter((item) => item.file === appFile);
const routeItems = fileItems.filter(
(item) => item.type === 'route-annotation',
);
const codeItems = fileItems.filter(
(item) => item.type === 'code-pattern',
);
let fileModified = false;
let currentContent = readFileSync(appFile, 'utf-8');
if (routeItems.length > 0) {
const result = rewriteRouteAnnotations(appFile, routeItems);
if (result.modified) {
currentContent = result.content;
fileModified = true;
completed.push(...result.itemsRewritten);
remaining.push(...result.itemsRemaining);
} else {
remaining.push(...routeItems);
}
}
if (codeItems.length > 0) {
const result = rewriteCodePatterns(appFile, codeItems);
if (result.modified) {
currentContent = result.content;
fileModified = true;
completed.push(...result.itemsRewritten);
remaining.push(...result.itemsRemaining);
} else {
remaining.push(...codeItems);
}
}
if (fileModified) {
writeFileSync(appFile, currentContent, 'utf-8');
filesModified.push(appFile);
}
}
// Ambiguous items always remain
remaining.push(...ambiguousItems);
return {
exitCode: completed.length > 0 ? BEHAVIORAL_FAILURE : SUCCESS,
items: allItems,
completed,
remaining,
message: formatWriteOutput(completed, remaining),
filesModified,
filesWouldBeModified: Array.from(filesWouldBeModified),
totalRewrites: allItems.length,
manualChoicesRequired: ambiguousItems.length,
};
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return {
exitCode: USAGE_ERROR,
items: [],
completed: [],
remaining: [],
message: `Migration failed: ${message}`,
};
}
}
// ---------------------------------------------------------------------------
// Output formatting
// ---------------------------------------------------------------------------
function formatCheckOutput(items: MigrationItem[]): string {
const lines: string[] = [];
lines.push('Legacy config patterns detected:');
lines.push('');
for (const item of items) {
const location = item.line ? `${item.file}:${item.line}` : item.file;
lines.push(` ${location}`);
lines.push(` Legacy: ${item.legacy}`);
lines.push(` Replace: ${item.replacement}`);
if (item.guidance) {
lines.push(` Guidance: ${item.guidance}`);
}
if (item.ambiguous) {
lines.push(` ⚠ Ambiguous — requires manual choice`);
}
lines.push('');
}
lines.push(`Found ${items.length} item(s) to migrate.`);
lines.push('');
lines.push('Run "apophis migrate --dry-run" to preview rewrites.');
lines.push('Run "apophis migrate --write" to apply rewrites.');
return lines.join('\n');
}
function formatDryRunOutput(items: MigrationItem[]): string {
const lines: string[] = [];
const files = new Set<string>();
const ambiguousCount = items.filter((item) => item.ambiguous).length;
lines.push('Dry run — the following rewrites would be applied:');
lines.push('');
for (const item of items) {
const location = item.line ? `${item.file}:${item.line}` : item.file;
files.add(item.file);
lines.push(` ${location}`);
lines.push(` - ${item.legacy}`);
lines.push(` + ${item.replacement}`);
if (item.guidance) {
lines.push(` # ${item.guidance}`);
}
if (item.ambiguous) {
lines.push(` ⚠ Skipped (ambiguous — requires manual choice)`);
}
lines.push('');
}
lines.push(`Total: ${items.length} item(s) to migrate.`);
lines.push(`Files that would be modified: ${files.size}`);
if (ambiguousCount > 0) {
lines.push(`Items requiring manual choice: ${ambiguousCount}`);
}
lines.push('');
lines.push('Run "apophis migrate --write" to apply these rewrites.');
return lines.join('\n');
}
function formatWriteOutput(
completed: MigrationItem[],
remaining: MigrationItem[],
): string {
const lines: string[] = [];
lines.push('Migration complete:');
lines.push('');
if (completed.length > 0) {
lines.push(` Completed (${completed.length}):`);
for (const item of completed) {
const location = item.line
? `${item.file}:${item.line}`
: item.file;
lines.push(
`${location}${item.legacy}${item.replacement}`,
);
}
lines.push('');
}
if (remaining.length > 0) {
lines.push(` Remaining (${remaining.length}):`);
for (const item of remaining) {
const location = item.line
? `${item.file}:${item.line}`
: item.file;
lines.push(` - ${location}${item.legacy}`);
if (item.ambiguous) {
lines.push(` ⚠ Ambiguous — requires manual choice`);
} else if (item.guidance) {
lines.push(` # ${item.guidance}`);
}
}
lines.push('');
}
if (remaining.length === 0) {
lines.push('All items migrated successfully.');
} else {
lines.push(`Run "apophis migrate --check" to review remaining items.`);
}
return lines.join('\n');
}
function formatAmbiguousOutput(items: MigrationItem[]): string {
const lines: string[] = [];
lines.push('Ambiguous rewrites detected — migration stopped:');
lines.push('');
for (const item of items) {
const location = item.line ? `${item.file}:${item.line}` : item.file;
lines.push(` ${location}`);
lines.push(` ${item.legacy}`);
lines.push(` ⚠ This pattern is ambiguous and requires manual choice.`);
if (item.guidance) {
lines.push(` Consider: ${item.guidance}`);
}
lines.push('');
}
lines.push('Please resolve these items manually, then re-run migrate.');
return lines.join('\n');
}
// ---------------------------------------------------------------------------
// CLI adapter
// ---------------------------------------------------------------------------
/**
* Adapter that bridges the CLI framework (cac) to the migrate command handler.
* This function signature matches what the CLI core expects.
*
* Safety: dry-run is the default mode. No files are modified unless --write is explicitly passed.
*/
export async function handleMigrate(
_args: string[],
ctx: CliContext,
): Promise<number> {
const options: MigrateOptions = {
config: ctx.options.config || undefined,
cwd: ctx.cwd,
format: ctx.options.format as MigrateOptions['format'],
quiet: ctx.options.quiet,
verbose: ctx.options.verbose,
};
// Parse command-specific flags from process.argv
const argv = process.argv.slice(2);
if (argv.includes('--check')) {
options.check = true;
}
if (argv.includes('--dry-run')) {
options.dryRun = true;
}
if (argv.includes('--write')) {
options.write = true;
}
const result = await migrateCommand(options, ctx);
// Output result based on format
if (!ctx.options.quiet && result.message) {
const format = options.format || ctx.options.format || 'human';
if (format === 'json') {
console.log(renderJson({
exitCode: result.exitCode,
items: result.items,
completed: result.completed,
remaining: result.remaining,
filesModified: result.filesModified,
filesWouldBeModified: result.filesWouldBeModified,
totalRewrites: result.totalRewrites,
manualChoicesRequired: result.manualChoicesRequired,
}));
} else if (format === 'ndjson') {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'migrate',
exitCode: result.exitCode,
items: result.items,
completed: result.completed,
remaining: result.remaining,
filesModified: result.filesModified,
filesWouldBeModified: result.filesWouldBeModified,
totalRewrites: result.totalRewrites,
manualChoicesRequired: result.manualChoicesRequired,
}) + '\n');
} else {
console.log(result.message);
}
}
return result.exitCode;
}
@@ -0,0 +1,257 @@
/**
* Code rewriter for APOPHIS migrate command.
*
* Responsibilities:
* - Rewrite legacy JS/TS plugin code patterns
* - contract() → verify({ kind: 'contract' })
* - stateful() → qualify({ kind: 'stateful' })
* - scenario() → qualify({ kind: 'scenario' })
* - Handle ambiguous patterns (stop, require manual choice)
* - Preserve code formatting and comments
* - Show surrounding context for ambiguous patterns
*
* Architecture:
* - Dependency injection: all dependencies passed explicitly
* - No optional imports
* - Inline comments for documentation
*/
import { readFileSync, writeFileSync } from 'node:fs';
import type { MigrationItem } from '../index.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface CodeRewriteResult {
content: string;
modified: boolean;
itemsRewritten: MigrationItem[];
itemsRemaining: MigrationItem[];
}
export interface AmbiguousCodePattern {
pattern: string;
line: number;
context: string[];
possibleResolutions: string[];
}
// ---------------------------------------------------------------------------
// Legacy code pattern mappings
// ---------------------------------------------------------------------------
/**
* Mapping of deprecated code patterns to their modern equivalents.
*
* Some patterns are marked as ambiguous because the semantic intent
* may not be clear from syntax alone (e.g., contract() could mean
* different things in different contexts).
*/
export const LEGACY_CODE_PATTERNS: Record<
string,
{ replacement: string; ambiguous?: boolean }
> = {
'contract()': { replacement: "verify({ kind: 'contract' })", ambiguous: false },
'stateful()': { replacement: "qualify({ kind: 'stateful' })", ambiguous: false },
'scenario()': { replacement: "qualify({ kind: 'scenario' })", ambiguous: false },
};
/**
* Ambiguous code patterns that require manual choice.
* These patterns could mean different things depending on context.
*/
export const AMBIGUOUS_CODE_PATTERNS: Record<
string,
{ possibleResolutions: string[]; guidance: string }
> = {
'oldApi()': {
possibleResolutions: [
"verify({ kind: 'contract' }) — if this is a contract test",
"qualify({ kind: 'stateful' }) — if this is a stateful test",
"Remove the call — if this is dead code",
],
guidance:
'The oldApi() pattern is ambiguous. It could be a contract test, stateful test, or dead code. Review the surrounding context to determine the correct replacement.',
},
'legacyPlugin()': {
possibleResolutions: [
"app.register(newPlugin()) — if migrating to a new plugin",
"Remove the call — if the plugin is no longer needed",
"// TODO: migrate plugin — if manual migration is required",
],
guidance:
'The legacyPlugin() pattern is ambiguous. Determine if the plugin has a modern equivalent or should be removed.',
},
};
// ---------------------------------------------------------------------------
// Core rewriting logic
// ---------------------------------------------------------------------------
/**
* Rewrite legacy code patterns in a JS/TS file.
*
* Strategy:
* 1. Read the raw file content
* 2. For each legacy pattern, replace occurrences
* 3. Skip ambiguous patterns unless explicitly allowed
* 4. Preserve formatting by only replacing the pattern text
* 5. Track which items were rewritten and which remain
*/
export function rewriteCodePatterns(
filePath: string,
items: MigrationItem[],
allowAmbiguous: boolean = false,
): CodeRewriteResult {
const content = readFileSync(filePath, 'utf-8');
let modifiedContent = content;
let modified = false;
const itemsRewritten: MigrationItem[] = [];
const itemsRemaining: MigrationItem[] = [];
for (const item of items) {
if (item.type !== 'code-pattern') {
itemsRemaining.push(item);
continue;
}
// Skip ambiguous items unless explicitly allowed
if (item.ambiguous && !allowAmbiguous) {
itemsRemaining.push(item);
continue;
}
const legacy = item.legacy;
const replacement = item.replacement;
// Match the exact pattern (e.g., contract())
// Need to escape the parentheses in the pattern
// Note: word boundary \b doesn't work after (), so we use a different approach
const escapedLegacy = escapeRegex(legacy);
const regex = new RegExp(
`(^|[^a-zA-Z0-9_])${escapedLegacy}($|[^a-zA-Z0-9_])`,
'g',
);
const newContent = modifiedContent.replace(
regex,
(match, prefix, suffix) => {
return prefix + replacement + suffix;
},
);
if (newContent !== modifiedContent) {
modifiedContent = newContent;
modified = true;
itemsRewritten.push(item);
} else {
itemsRemaining.push(item);
}
}
return {
content: modifiedContent,
modified,
itemsRewritten,
itemsRemaining,
};
}
/**
* Write the rewritten code file to disk.
*/
export function writeRewrittenCode(filePath: string, content: string): void {
writeFileSync(filePath, content, 'utf-8');
}
/**
* Detect legacy code patterns in raw text content.
* Returns migration items for each occurrence.
*/
export function detectLegacyCodePatterns(
content: string,
filePath: string,
): MigrationItem[] {
const items: MigrationItem[] = [];
const lines = content.split('\n');
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line === undefined) continue;
for (const [legacy, mapping] of Object.entries(LEGACY_CODE_PATTERNS)) {
// Match the pattern as a standalone call
// Escape parentheses in the legacy pattern
// Note: word boundary \b doesn't work after (), so we use a different approach
const escapedLegacy = escapeRegex(legacy);
const regex = new RegExp(
`(^|[^a-zA-Z0-9_])${escapedLegacy}($|[^a-zA-Z0-9_])`,
);
if (regex.test(line)) {
items.push({
type: 'code-pattern',
file: filePath,
line: i + 1,
legacy,
replacement: mapping.replacement,
guidance: `Replace '${legacy}' with '${mapping.replacement}'`,
ambiguous: mapping.ambiguous,
});
}
}
}
return items;
}
/**
* Detect ambiguous code patterns that require manual choice.
* Returns ambiguous patterns with surrounding context for human review.
*/
export function detectAmbiguousCodePatterns(
content: string,
filePath: string,
): MigrationItem[] {
const items: MigrationItem[] = [];
const lines = content.split('\n');
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line === undefined) continue;
for (const [pattern, info] of Object.entries(AMBIGUOUS_CODE_PATTERNS)) {
const escapedPattern = escapeRegex(pattern);
const regex = new RegExp(
`(^|[^a-zA-Z0-9_])${escapedPattern}($|[^a-zA-Z0-9_])`,
);
if (regex.test(line)) {
// Capture surrounding context (2 lines before and after)
const contextStart = Math.max(0, i - 2);
const contextEnd = Math.min(lines.length, i + 3);
const context = lines.slice(contextStart, contextEnd);
items.push({
type: 'code-pattern',
file: filePath,
line: i + 1,
legacy: pattern,
replacement: '(ambiguous — see guidance)',
guidance: `${info.guidance}\nPossible resolutions:\n${info.possibleResolutions.map((r) => ` - ${r}`).join('\n')}\n\nContext:\n${context.map((l, idx) => ` ${contextStart + idx + 1}: ${l}`).join('\n')}`,
ambiguous: true,
});
}
}
}
return items;
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function escapeRegex(str: string): string {
return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
@@ -0,0 +1,268 @@
/**
* Config rewriter for APOPHIS migrate command.
*
* Responsibilities:
* - Rewrite config files, replacing legacy fields with modern equivalents
* - Preserve comments and formatting where feasible
* - Handle nested object rewrites
* - Report what was changed and what remains
* - Detect mixed legacy/modern configs and report clearly
* - Emit human guidance for legacy fields with no direct equivalent
*
* Architecture:
* - Dependency injection: all dependencies passed explicitly
* - No optional imports
* - Inline comments for documentation
*/
import { readFileSync, writeFileSync } from 'node:fs';
import type { MigrationItem } from '../index.js';
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function escapeRegex(str: string): string {
return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface ConfigRewriteResult {
content: string;
modified: boolean;
itemsRewritten: MigrationItem[];
itemsRemaining: MigrationItem[];
}
export interface MixedFieldReport {
legacy: string;
modern: string;
line: number;
guidance: string;
}
// ---------------------------------------------------------------------------
// Legacy field mappings
// ---------------------------------------------------------------------------
/**
* Mapping of deprecated config fields to their modern equivalents.
*/
export const LEGACY_CONFIG_MAPPINGS: Record<string, string> = {
// Top-level fields
testMode: 'mode',
// Profile container
testProfiles: 'profiles',
// Profile fields
usesPreset: 'preset',
routeFilter: 'routes',
// Preset container
testPresets: 'presets',
// Preset fields
testDepth: 'depth',
maxDuration: 'timeout',
// Environment container
envPolicies: 'environments',
// Environment fields
canVerify: 'allowVerify',
};
/**
* Legacy fields with no direct equivalent — emit human guidance instead of auto-rewrite.
*/
export const LEGACY_FIELDS_NO_EQUIVALENT: Record<string, { guidance: string; severity: 'warning' | 'error' }> = {
legacyField: {
guidance: 'This field has no modern equivalent. Remove it and review your config manually.',
severity: 'warning',
},
oldApiVersion: {
guidance: 'API versioning is now handled via profiles. Remove this field and set version in each profile.',
severity: 'warning',
},
deprecatedPlugin: {
guidance: 'This plugin is no longer supported. Remove the field and migrate to the new plugin system.',
severity: 'error',
},
};
// ---------------------------------------------------------------------------
// Core rewriting logic
// ---------------------------------------------------------------------------
/**
* Rewrite a config file, replacing legacy field names with modern equivalents.
*
* Strategy:
* 1. Read the raw file content
* 2. For each legacy field mapping, replace occurrences as property keys
* 3. Preserve formatting by only replacing the key name, not surrounding whitespace
* 4. Track which items were rewritten and which remain
*/
export function rewriteConfigFile(
filePath: string,
items: MigrationItem[],
): ConfigRewriteResult {
const content = readFileSync(filePath, 'utf-8');
let modifiedContent = content;
let modified = false;
const itemsRewritten: MigrationItem[] = [];
const itemsRemaining: MigrationItem[] = [];
for (const item of items) {
if (item.type !== 'config-field') {
itemsRemaining.push(item);
continue;
}
// The legacy field name (might be a nested path like "testProfiles.quick")
const legacyKey = item.legacy.split('.').pop() || item.legacy;
const replacement = item.replacement;
// Build a regex that matches the field as a property key
// This handles: key:, "key":, 'key':, key :, etc.
const regex = new RegExp(
`([\\s{,\\[])(['"]?)(${escapeRegex(legacyKey)})\\2\\s*:(?!\\/)`,
'g',
);
const newContent = modifiedContent.replace(regex, (match, prefix, quote, _key) => {
return `${prefix}${quote}${replacement}${quote}:`;
});
if (newContent !== modifiedContent) {
modifiedContent = newContent;
modified = true;
itemsRewritten.push(item);
} else {
itemsRemaining.push(item);
}
}
return {
content: modifiedContent,
modified,
itemsRewritten,
itemsRemaining,
};
}
/**
* Write the rewritten config to disk.
*/
export function writeRewrittenConfig(filePath: string, content: string): void {
writeFileSync(filePath, content, 'utf-8');
}
/**
* Detect legacy config fields in raw text content.
* Returns migration items for each occurrence.
*/
export function detectLegacyConfigFields(
content: string,
filePath: string,
): MigrationItem[] {
const items: MigrationItem[] = [];
const lines = content.split('\n');
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line === undefined) continue;
for (const [legacy, replacement] of Object.entries(LEGACY_CONFIG_MAPPINGS)) {
// Match the field as a property key, avoiding matches inside strings/comments
const regex = new RegExp(`\\b${escapeRegex(legacy)}\\s*:`);
if (regex.test(line)) {
items.push({
type: 'config-field',
file: filePath,
line: i + 1,
legacy,
replacement,
guidance: `Replace '${legacy}' with '${replacement}'`,
});
}
}
}
return items;
}
/**
* Detect legacy fields that have no direct modern equivalent.
* These emit human guidance instead of being auto-rewritten.
*/
export function detectLegacyFieldsNoEquivalent(
content: string,
filePath: string,
): MigrationItem[] {
const items: MigrationItem[] = [];
const lines = content.split('\n');
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line === undefined) continue;
for (const [legacy, info] of Object.entries(LEGACY_FIELDS_NO_EQUIVALENT)) {
const regex = new RegExp(`\\b${escapeRegex(legacy)}\\s*:`);
if (regex.test(line)) {
items.push({
type: 'config-field',
file: filePath,
line: i + 1,
legacy,
replacement: '(removed — see guidance)',
guidance: info.guidance,
});
}
}
}
return items;
}
/**
* Detect mixed legacy and modern config fields.
* When both legacy and modern versions of the same field exist, report each clearly.
*/
export function detectMixedLegacyModernFields(
content: string,
filePath: string,
): MixedFieldReport[] {
const reports: MixedFieldReport[] = [];
const lines = content.split('\n');
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line === undefined) continue;
for (const [legacy, modern] of Object.entries(LEGACY_CONFIG_MAPPINGS)) {
// Check if this line contains the legacy field
const legacyRegex = new RegExp(`\\b${escapeRegex(legacy)}\\s*:`);
if (legacyRegex.test(line)) {
// Check if the modern equivalent also exists somewhere in the file
const modernRegex = new RegExp(`\\b${escapeRegex(modern)}\\s*:`);
if (modernRegex.test(content)) {
reports.push({
legacy,
modern,
line: i + 1,
guidance: `Both '${legacy}' (legacy) and '${modern}' (modern) found. Remove '${legacy}' to avoid conflicts.`,
});
}
}
}
}
return reports;
}
@@ -0,0 +1,216 @@
/**
* Route rewriter for APOPHIS migrate command.
*
* Responsibilities:
* - Rewrite route schema annotations (e.g., x-validate-runtime → runtime)
* - Preserve schema structure and formatting
* - Handle annotations in Fastify route definitions
* - Detect ambiguous annotations and require manual choice
*
* Architecture:
* - Dependency injection: all dependencies passed explicitly
* - No optional imports
* - Inline comments for documentation
*/
import { readFileSync, writeFileSync } from 'node:fs';
import type { MigrationItem } from '../index.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface RouteRewriteResult {
content: string;
modified: boolean;
itemsRewritten: MigrationItem[];
itemsRemaining: MigrationItem[];
}
export interface AmbiguousRoutePattern {
pattern: string;
line: number;
context: string[];
possibleResolutions: string[];
}
// ---------------------------------------------------------------------------
// Legacy annotation mappings
// ---------------------------------------------------------------------------
/**
* Mapping of deprecated route schema annotations to their modern equivalents.
*/
export const LEGACY_ROUTE_ANNOTATIONS: Record<string, string> = {
'x-validate-runtime': 'runtime',
};
/**
* Ambiguous route patterns that require manual choice.
* These patterns could mean different things depending on context.
*/
export const AMBIGUOUS_ROUTE_PATTERNS: Record<string, { possibleResolutions: string[]; guidance: string }> = {
'x-validate': {
possibleResolutions: [
"'runtime' — validate at runtime",
"'build' — validate at build time",
"'both' — validate at both times",
],
guidance: 'The x-validate annotation is ambiguous. Choose the validation timing explicitly.',
},
'x-check': {
possibleResolutions: [
"'runtime' — runtime check",
"'contract' — contract check",
"'schema' — schema-only check",
],
guidance: 'The x-check annotation is ambiguous. Choose the check type explicitly.',
},
};
// ---------------------------------------------------------------------------
// Core rewriting logic
// ---------------------------------------------------------------------------
/**
* Rewrite route annotations in a file.
*
* Strategy:
* 1. Read the raw file content
* 2. For each legacy annotation, replace occurrences in string literals
* 3. Preserve formatting by only replacing the annotation name
* 4. Track which items were rewritten and which remain
*/
export function rewriteRouteAnnotations(
filePath: string,
items: MigrationItem[],
): RouteRewriteResult {
const content = readFileSync(filePath, 'utf-8');
let modifiedContent = content;
let modified = false;
const itemsRewritten: MigrationItem[] = [];
const itemsRemaining: MigrationItem[] = [];
for (const item of items) {
if (item.type !== 'route-annotation') {
itemsRemaining.push(item);
continue;
}
const legacy = item.legacy;
const replacement = item.replacement;
// Match the annotation in string literals (single or double quotes)
// The legacy string might have hyphens, so we need to be careful with word boundaries
const regex = new RegExp(
`(['"])${escapeRegex(legacy)}(['"])`,
'g',
);
const newContent = modifiedContent.replace(regex, `$1${replacement}$2`);
if (newContent !== modifiedContent) {
modifiedContent = newContent;
modified = true;
itemsRewritten.push(item);
} else {
itemsRemaining.push(item);
}
}
return {
content: modifiedContent,
modified,
itemsRewritten,
itemsRemaining,
};
}
/**
* Write the rewritten route file to disk.
*/
export function writeRewrittenRoutes(filePath: string, content: string): void {
writeFileSync(filePath, content, 'utf-8');
}
/**
* Detect legacy route annotations in raw text content.
* Returns migration items for each occurrence.
*/
export function detectLegacyRouteAnnotations(
content: string,
filePath: string,
): MigrationItem[] {
const items: MigrationItem[] = [];
const lines = content.split('\n');
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line === undefined) continue;
for (const [legacy, replacement] of Object.entries(LEGACY_ROUTE_ANNOTATIONS)) {
// Match the annotation in string literals
const regex = new RegExp(`['"]${escapeRegex(legacy)}['"]`);
if (regex.test(line)) {
items.push({
type: 'route-annotation',
file: filePath,
line: i + 1,
legacy,
replacement,
guidance: `Replace '${legacy}' with '${replacement}' in route schema`,
});
}
}
}
return items;
}
/**
* Detect ambiguous route patterns that require manual choice.
* Returns ambiguous patterns with surrounding context for human review.
*/
export function detectAmbiguousRoutePatterns(
content: string,
filePath: string,
): MigrationItem[] {
const items: MigrationItem[] = [];
const lines = content.split('\n');
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line === undefined) continue;
for (const [pattern, info] of Object.entries(AMBIGUOUS_ROUTE_PATTERNS)) {
const regex = new RegExp(`['"]${escapeRegex(pattern)}['"]`);
if (regex.test(line)) {
// Capture surrounding context (2 lines before and after)
const contextStart = Math.max(0, i - 2);
const contextEnd = Math.min(lines.length, i + 3);
const context = lines.slice(contextStart, contextEnd);
items.push({
type: 'route-annotation',
file: filePath,
line: i + 1,
legacy: pattern,
replacement: '(ambiguous — see guidance)',
guidance: `${info.guidance}\nPossible resolutions:\n${info.possibleResolutions.map(r => ` - ${r}`).join('\n')}`,
ambiguous: true,
});
}
}
}
return items;
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function escapeRegex(str: string): string {
return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$\u0026');
}
+328
View File
@@ -0,0 +1,328 @@
/**
* S5: Observe thread - Observe command handler
*
* Responsibilities:
* - Load config and resolve profile
* - Validate observe configuration
* - Check reporting sink setup (logs, metrics, traces)
* - Validate non-blocking semantics
* - Environment safety checks (block blocking behavior in prod by default)
* - Support --check-config (validate only, don't activate)
* - Explain what would be checked and why it is safe
* - Clear output about safety boundaries
* - Exit 0 on valid config, 2 on safety violation
*/
import type { CliContext } from '../../core/context.js';
import { loadConfig } from '../../core/config-loader.js';
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js';
import { SUCCESS, USAGE_ERROR } from '../../core/exit-codes.js';
import { validateObserveConfig } from './validator.js';
import { renderDoctorChecks } from '../../renderers/human.js';
import { renderJson } from '../../renderers/json.js';
import type { OutputContext } from '../../renderers/shared.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface ObserveOptions {
profile?: string;
checkConfig?: boolean;
config?: string;
cwd?: string;
format?: 'human' | 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary';
quiet?: boolean;
verbose?: boolean;
}
export interface ObserveResult {
exitCode: number;
message?: string;
checks?: Array<{
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
}>;
}
// ---------------------------------------------------------------------------
// Command handler
// ---------------------------------------------------------------------------
/**
* Main observe command handler.
*
* Flow:
* 1. Load and resolve config
* 2. Run policy engine checks
* 3. Validate observe-specific configuration
* 4. If --check-config, stop after validation
* 5. Otherwise, report what would be activated and why it is safe
* 6. Return appropriate exit code
*/
export async function observeCommand(
options: ObserveOptions,
ctx: CliContext,
): Promise<ObserveResult> {
const { profile, checkConfig, config: configPath, cwd } = options;
const workingDir = cwd || ctx.cwd;
// Detect environment from context
const env = detectEnvironment();
try {
// 1. Load config
const loadResult = await loadConfig({
cwd: workingDir,
configPath,
profileName: profile,
env,
});
if (!loadResult.configPath) {
return {
exitCode: USAGE_ERROR,
message: 'No config found. Run "apophis init" to create one.',
};
}
const config = loadResult.config;
// 2. Run policy engine checks
const policyEngine = new PolicyEngine({
config,
env,
mode: 'observe',
profileName: profile || undefined,
presetName: loadResult.presetName || undefined,
});
const policyResult = policyEngine.check();
if (!policyResult.allowed) {
const message = [
'Policy check failed:',
...policyResult.errors.map(e => `${e}`),
].join('\n');
return {
exitCode: USAGE_ERROR,
message,
};
}
// 3. Validate observe-specific configuration
const validationResult = validateObserveConfig(config, profile || undefined, env);
if (!validationResult.valid) {
const message = formatValidationOutput(validationResult, { checkConfig, env, profile });
return {
exitCode: USAGE_ERROR,
message,
checks: validationResult.checks,
};
}
// 4. If --check-config, stop after validation with success
if (checkConfig) {
const message = formatValidationOutput(validationResult, {
checkConfig: true,
env,
profile,
});
return {
exitCode: SUCCESS,
message,
checks: validationResult.checks,
};
}
// 5. Report what would be activated and why it is safe
const activationMessage = formatActivationOutput(validationResult, {
env,
profile,
configPath: loadResult.configPath,
});
return {
exitCode: SUCCESS,
message: activationMessage,
checks: validationResult.checks,
};
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return {
exitCode: USAGE_ERROR,
message: `Failed to run observe command: ${message}`,
checks: [],
};
}
}
// ---------------------------------------------------------------------------
// Output formatting
// ---------------------------------------------------------------------------
interface FormatOptions {
checkConfig?: boolean;
env: string;
profile?: string;
configPath?: string;
}
/**
* Format validation results for human-readable output.
*/
function formatValidationOutput(
result: import('./validator.js').ObserveValidationResult,
options: FormatOptions,
): string {
const lines: string[] = [];
const mode = options.checkConfig ? 'Config validation' : 'Observe validation';
lines.push(`${mode} for environment "${options.env}"`);
if (options.profile) {
lines.push(`Profile: ${options.profile}`);
}
lines.push('');
// Print each check
for (const check of result.checks) {
const icon = check.status === 'pass' ? '✓' : check.status === 'warn' ? '⚠' : '✗';
lines.push(` ${icon} ${check.name}: ${check.message}`);
if (check.detail) {
lines.push(` ${check.detail}`);
}
}
lines.push('');
// Summary
if (result.errors.length > 0) {
lines.push(`Failed with ${result.errors.length} error(s).`);
lines.push('');
lines.push('Safety boundaries:');
lines.push(' - Observe mode is non-blocking by default');
lines.push(' - Blocking behavior is prohibited in production');
lines.push(' - Qualify-only features (chaos, stateful, etc.) are not allowed');
lines.push(' - Sampling rate must be between 0.0 and 1.0');
lines.push(' - Sinks must be configured when required by environment policy');
} else if (result.warnings.length > 0) {
lines.push(`Passed with ${result.warnings.length} warning(s).`);
} else {
lines.push('All checks passed.');
}
return lines.join('\n');
}
/**
* Format activation output explaining what would be checked and why it is safe.
*/
function formatActivationOutput(
result: import('./validator.js').ObserveValidationResult,
options: FormatOptions,
): string {
const lines: string[] = [];
lines.push(`Observe mode ready for environment "${options.env}"`);
if (options.profile) {
lines.push(`Profile: ${options.profile}`);
}
if (options.configPath) {
lines.push(`Config: ${options.configPath}`);
}
lines.push('');
// Print checks
for (const check of result.checks) {
const icon = check.status === 'pass' ? '✓' : check.status === 'warn' ? '⚠' : '✗';
lines.push(` ${icon} ${check.name}: ${check.message}`);
}
lines.push('');
lines.push('What would be checked:');
lines.push(' - Request/response contracts are evaluated asynchronously');
lines.push(' - Violations are logged to configured sinks without blocking');
lines.push(' - Sampling controls the fraction of requests observed');
lines.push(' - Metrics and traces provide runtime visibility into contract health');
lines.push('');
lines.push('Why this is safe:');
lines.push(' - Non-blocking semantics guarantee observation does not affect latency');
lines.push(' - No chaos injection or stateful sequences are activated in observe mode');
lines.push(' - Production environments require explicit non-blocking configuration');
lines.push(' - All qualify-only features are blocked by validation');
if (result.warnings.length > 0) {
lines.push('');
lines.push('Warnings:');
for (const warning of result.warnings) {
lines.push(`${warning}`);
}
}
lines.push('');
lines.push('To activate observation, run without --check-config.');
return lines.join('\n');
}
// ---------------------------------------------------------------------------
// CLI adapter
// ---------------------------------------------------------------------------
/**
* Adapter that bridges the CLI framework (cac) to the observe command handler.
* This function signature matches what the CLI core expects.
*/
export async function handleObserve(
_args: string[],
ctx: CliContext,
): Promise<number> {
const options: ObserveOptions = {
profile: ctx.options.profile || undefined,
checkConfig: false,
config: ctx.options.config || undefined,
cwd: ctx.cwd,
format: ctx.options.format as ObserveOptions['format'],
quiet: ctx.options.quiet,
verbose: ctx.options.verbose,
};
// Parse command-specific flags from process.argv
// cac passes these as parsed options, but we need to extract --check-config
// Since cac doesn't expose parsed command-specific flags in the options object,
// we scan process.argv directly for observe-specific flags
const argv = process.argv.slice(2);
if (argv.includes('--check-config')) {
options.checkConfig = true;
}
const result = await observeCommand(options, ctx);
// Output result based on format
if (!ctx.options.quiet && result.message) {
const format = options.format || ctx.options.format || 'human';
if (format === 'json') {
console.log(renderJson({
exitCode: result.exitCode,
checks: result.checks,
message: result.message,
}));
} else if (format === 'ndjson') {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'observe',
exitCode: result.exitCode,
checks: result.checks,
message: result.message,
}) + '\n');
} else {
console.log(result.message);
}
}
return result.exitCode;
}
+539
View File
@@ -0,0 +1,539 @@
/**
* S5: Observe thread - Observe config validation logic
*
* Validates observe-specific configuration including:
* - Sink configuration checks (logs, metrics, traces)
* - Sampling rate validation
* - Feature restriction checks (no qualify-only features in observe)
* - Non-blocking semantics validation
*/
import type { Config, ProfileDefinition, PresetDefinition } from '../../core/config-loader.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface ObserveValidationResult {
valid: boolean;
errors: string[];
warnings: string[];
checks: ObserveCheck[];
}
export interface ObserveCheck {
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
}
export interface SinkConfig {
logs?: boolean;
metrics?: boolean;
traces?: boolean;
endpoint?: string;
}
export interface ObserveProfileConfig {
sampling?: number;
blocking?: boolean;
sinks?: SinkConfig;
features?: string[];
}
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
/** Features that are only valid in qualify mode */
const QUALIFY_ONLY_FEATURES = new Set([
'chaos',
'stateful',
'scenario',
'outbound-mocks',
'protocol-flow',
]);
/** Valid sampling rate bounds */
const SAMPLING_MIN = 0.0;
const SAMPLING_MAX = 1.0;
// ---------------------------------------------------------------------------
// Validation functions
// ---------------------------------------------------------------------------
/**
* Validate observe configuration for a given profile and environment.
*/
export function validateObserveConfig(
config: Config,
profileName: string | undefined,
env: string,
): ObserveValidationResult {
const errors: string[] = [];
const warnings: string[] = [];
const checks: ObserveCheck[] = [];
// Resolve the effective profile config (preset + profile overrides)
const profileConfig = resolveObserveProfileConfig(config, profileName);
// 1. Check profile exists and is observe mode
const profileCheck = validateProfileMode(config, profileName);
checks.push(profileCheck);
if (profileCheck.status === 'fail') {
errors.push(profileCheck.message);
}
// 2. Check for qualify-only features (uses resolved profile config)
const featureCheck = validateFeatures(profileConfig.features, profileName);
checks.push(featureCheck);
if (featureCheck.status === 'fail') {
errors.push(featureCheck.message);
}
// 3. Validate sampling rate (uses resolved profile config)
const samplingCheck = validateSamplingRate(profileConfig.sampling);
checks.push(samplingCheck);
if (samplingCheck.status === 'fail') {
errors.push(samplingCheck.message);
}
// 4. Validate sink configuration (uses resolved profile config)
const sinkCheck = validateSinkConfig(profileConfig.sinks, env, config);
checks.push(sinkCheck);
if (sinkCheck.status === 'fail') {
errors.push(sinkCheck.message);
} else if (sinkCheck.status === 'warn') {
warnings.push(sinkCheck.message);
}
// 5. Validate non-blocking semantics (uses resolved profile config)
const blockingCheck = validateBlockingSemantics(profileConfig.blocking, env, config);
checks.push(blockingCheck);
if (blockingCheck.status === 'fail') {
errors.push(blockingCheck.message);
}
// 6. Environment policy check: must explicitly allow observe
const envPolicyCheck = validateEnvironmentPolicy(config, env);
checks.push(envPolicyCheck);
if (envPolicyCheck.status === 'fail') {
errors.push(envPolicyCheck.message);
}
// 7. Environment safety check
const envCheck = validateEnvironmentSafety(env, profileConfig);
checks.push(envCheck);
if (envCheck.status === 'warn') {
warnings.push(envCheck.message);
}
// 8. Profile must be configured for observe mode
const profileObserveCheck = validateProfileObserveMode(config, profileName);
checks.push(profileObserveCheck);
if (profileObserveCheck.status === 'fail') {
errors.push(profileObserveCheck.message);
}
return {
valid: errors.length === 0,
errors,
warnings,
checks,
};
}
/**
* Resolve the observe-specific configuration from profile and preset.
* Preset values are applied first, then profile overrides.
*/
function resolveObserveProfileConfig(
config: Config,
profileName: string | undefined,
): ObserveProfileConfig {
const result: ObserveProfileConfig = {};
if (!profileName || !config.profiles) {
return result;
}
const profile = config.profiles[profileName];
if (!profile) {
return result;
}
// Apply preset first if referenced
if (profile.preset && config.presets) {
const preset = config.presets[profile.preset];
if (preset) {
Object.assign(result, presetToObserveConfig(preset));
}
}
// Apply profile overrides
Object.assign(result, profileToObserveConfig(profile));
return result;
}
/**
* Convert preset definition to observe config.
*/
function presetToObserveConfig(preset: PresetDefinition): ObserveProfileConfig {
return {
features: preset.features,
sampling: (preset as Record<string, unknown>).sampling as number | undefined,
blocking: (preset as Record<string, unknown>).blocking as boolean | undefined,
sinks: (preset as Record<string, unknown>).sinks as SinkConfig | undefined,
};
}
/**
* Convert profile definition to observe config.
*/
function profileToObserveConfig(profile: ProfileDefinition): ObserveProfileConfig {
return {
features: profile.features,
sampling: (profile as Record<string, unknown>).sampling as number | undefined,
blocking: (profile as Record<string, unknown>).blocking as boolean | undefined,
sinks: (profile as Record<string, unknown>).sinks as SinkConfig | undefined,
};
}
/**
* Validate that the profile exists.
* Note: mode validation is handled by validateProfileObserveMode.
*/
function validateProfileMode(
config: Config,
profileName: string | undefined,
): ObserveCheck {
if (!profileName) {
return {
name: 'profile-mode',
status: 'pass',
message: 'No profile specified, using default observe configuration',
};
}
if (!config.profiles || !config.profiles[profileName]) {
const available = config.profiles ? Object.keys(config.profiles).join(', ') : 'none';
return {
name: 'profile-mode',
status: 'fail',
message: `Profile "${profileName}" not found. Available profiles: ${available}`,
};
}
return {
name: 'profile-mode',
status: 'pass',
message: `Profile "${profileName}" exists`,
};
}
/**
* Validate that the profile is explicitly configured for observe mode.
*/
function validateProfileObserveMode(
config: Config,
profileName: string | undefined,
): ObserveCheck {
if (!profileName) {
return {
name: 'profile-observe-mode',
status: 'pass',
message: 'No profile specified, mode will be determined by top-level config',
};
}
if (!config.profiles || !config.profiles[profileName]) {
return {
name: 'profile-observe-mode',
status: 'pass',
message: `Profile "${profileName}" not found — will be validated by profile-mode check`,
};
}
const profile = config.profiles[profileName];
const profileMode = profile.mode;
if (profileMode && profileMode !== 'observe') {
return {
name: 'profile-observe-mode',
status: 'fail',
message: `Profile "${profileName}" is configured for "${profileMode}" mode but observe command requires "observe" mode`,
detail: 'Change the profile mode to "observe" or use the appropriate command ' +
`for "${profileMode}" mode (e.g., apophis ${profileMode}).`,
};
}
return {
name: 'profile-observe-mode',
status: 'pass',
message: `Profile "${profileName}" is configured for observe mode`,
};
}
/**
* Validate that no qualify-only features are used in observe mode.
*/
function validateFeatures(
features: string[] | undefined,
profileName: string | undefined,
): ObserveCheck {
if (!features || features.length === 0) {
return {
name: 'feature-restrictions',
status: 'pass',
message: 'No features configured',
};
}
const invalidFeatures = features.filter(f => QUALIFY_ONLY_FEATURES.has(f));
if (invalidFeatures.length > 0) {
const profileRef = profileName ? `Profile "${profileName}"` : 'Configuration';
return {
name: 'feature-restrictions',
status: 'fail',
message: `${profileRef} references qualify-only features that cannot be used in observe mode: ${invalidFeatures.join(', ')}`,
detail: `Remove these features from the profile or preset. Qualify-only features: ${Array.from(QUALIFY_ONLY_FEATURES).join(', ')}`,
};
}
return {
name: 'feature-restrictions',
status: 'pass',
message: `All features are valid for observe mode: ${features.join(', ')}`,
};
}
/**
* Validate sampling rate is within valid bounds [0.0, 1.0].
*/
export function validateSamplingRate(sampling: number | undefined): ObserveCheck {
if (sampling === undefined || sampling === null) {
return {
name: 'sampling-rate',
status: 'pass',
message: 'No sampling rate configured, using default (1.0)',
};
}
if (typeof sampling !== 'number' || Number.isNaN(sampling)) {
return {
name: 'sampling-rate',
status: 'fail',
message: `Sampling rate must be a number, got ${typeof sampling}`,
detail: `Valid range: ${SAMPLING_MIN} to ${SAMPLING_MAX} (inclusive)`,
};
}
if (sampling < SAMPLING_MIN || sampling > SAMPLING_MAX) {
return {
name: 'sampling-rate',
status: 'fail',
message: `Sampling rate ${sampling} is out of bounds`,
detail: `Set sampling to a value between ${SAMPLING_MIN} and ${SAMPLING_MAX} (inclusive). ` +
`A rate of 0.0 disables observation, 1.0 observes all requests.`,
};
}
return {
name: 'sampling-rate',
status: 'pass',
message: `Sampling rate ${sampling} is valid`,
};
}
/**
* Validate sink configuration for the environment.
*/
function validateSinkConfig(
sinks: SinkConfig | undefined,
env: string,
config: Config,
): ObserveCheck {
// Check if environment requires sinks
const envPolicy = config.environments?.[env];
const requireSink = envPolicy?.requireSink ?? false;
if (!sinks || Object.keys(sinks).length === 0) {
if (requireSink) {
return {
name: 'sink-config',
status: 'fail',
message: `Environment "${env}" requires sink configuration but none is provided`,
detail: 'Add sinks to your profile (e.g., sinks: { logs: true }) ' +
'or set requireSink: false in the environment policy.',
};
}
return {
name: 'sink-config',
status: 'warn',
message: 'No sinks configured. Observation data will not be persisted.',
detail: 'Configure at least one sink (logs, metrics, or traces) ' +
'to capture observation data for analysis.',
};
}
const activeSinks = [];
if (sinks.logs) activeSinks.push('logs');
if (sinks.metrics) activeSinks.push('metrics');
if (sinks.traces) activeSinks.push('traces');
if (activeSinks.length === 0) {
return {
name: 'sink-config',
status: 'warn',
message: 'Sinks are configured but none are enabled. Observation data will not be persisted.',
detail: 'Set at least one of logs, metrics, or traces to true in your sink configuration.',
};
}
return {
name: 'sink-config',
status: 'pass',
message: `Active sinks: ${activeSinks.join(', ')}`,
};
}
/**
* Validate non-blocking semantics for the environment.
* Blocking is NEVER allowed in production unless explicitly enabled by a break-glass policy.
*/
function validateBlockingSemantics(
blocking: boolean | undefined,
env: string,
config: Config,
): ObserveCheck {
const isProd = env === 'production' || env === 'prod';
if (blocking === true && isProd) {
// Check for break-glass policy override
const envPolicy = config.environments?.[env];
const allowBlocking = envPolicy?.allowBlocking ?? false;
if (!allowBlocking) {
return {
name: 'blocking-semantics',
status: 'fail',
message: `Blocking behavior is not allowed in production environment "${env}"`,
detail: 'Set blocking: false in your profile, use a non-production environment, ' +
'or set allowBlocking: true in the environment policy for break-glass scenarios.',
};
}
return {
name: 'blocking-semantics',
status: 'pass',
message: `Blocking behavior is enabled in production "${env}" via break-glass policy`,
detail: 'WARNING: blocking observation can severely impact request latency. ' +
'This should only be used during active incident response.',
};
}
if (blocking === true) {
return {
name: 'blocking-semantics',
status: 'pass',
message: `Blocking behavior is enabled in non-production environment "${env}"`,
detail: 'Warning: blocking observation can increase request latency. ' +
'Only enable in environments where latency impact is acceptable.',
};
}
// blocking is false or undefined (default to non-blocking)
return {
name: 'blocking-semantics',
status: 'pass',
message: `Non-blocking semantics confirmed for environment "${env}"`,
detail: 'Observation will run asynchronously without blocking request handling.',
};
}
/**
* Validate environment policy explicitly allows observe mode.
*/
function validateEnvironmentPolicy(
config: Config,
env: string,
): ObserveCheck {
const envPolicy = config.environments?.[env];
if (!envPolicy) {
// No explicit policy for this environment — warn but don't fail
return {
name: 'environment-policy',
status: 'pass',
message: `No environment policy defined for "${env}"`,
detail: 'Observe mode is allowed by default when no policy is configured.',
};
}
const allowObserve = envPolicy.allowObserve;
if (allowObserve === false) {
return {
name: 'environment-policy',
status: 'fail',
message: `Environment policy for "${env}" explicitly blocks observe mode`,
detail: 'Set allowObserve: true in the environment policy to enable observe mode, ' +
'or run in an environment where observe is allowed.',
};
}
return {
name: 'environment-policy',
status: 'pass',
message: `Environment "${env}" explicitly allows observe mode`,
};
}
/**
* Validate environment-specific safety constraints.
*/
function validateEnvironmentSafety(
env: string,
profileConfig: ObserveProfileConfig,
): ObserveCheck {
const isProd = env === 'production' || env === 'prod';
if (isProd) {
const warnings = [];
if (profileConfig.sampling === undefined) {
warnings.push('sampling rate not configured (will use default 1.0)');
}
if (!profileConfig.sinks) {
warnings.push('no sinks configured');
}
if (warnings.length > 0) {
return {
name: 'environment-safety',
status: 'warn',
message: `Production environment "${env}" observe configuration has warnings: ${warnings.join(', ')}`,
detail: 'In production, configure explicit sampling rate and sinks ' +
'to control observation overhead and ensure data capture.',
};
}
}
return {
name: 'environment-safety',
status: 'pass',
message: `Environment "${env}" safety checks passed`,
};
}
// ---------------------------------------------------------------------------
// Exports for testing
// ---------------------------------------------------------------------------
export {
QUALIFY_ONLY_FEATURES,
SAMPLING_MIN,
SAMPLING_MAX,
};
+148
View File
@@ -0,0 +1,148 @@
/**
* S6: Qualify thread - Chaos execution handler
*
* Responsibilities:
* - Run a single route with chaos injection and collect traces
* - Generate deterministic chaos events for CLI qualify mode
* - Uses chaos-v3 pure functions for deterministic adversity
*
* Architecture:
* - Pure execution function that accepts injected dependencies
* - No optional imports — everything is passed via parameters
*/
import { applyChaosToExecution, createChaosEventArbitrary, formatChaosEvents } from '../../../quality/chaos-v3.js'
import { SeededRng } from '../../../infrastructure/seeded-rng.js'
import type {
RouteContract,
EvalContext,
ChaosConfig,
} from '../../../types.js'
import type { QualifyRunnerDeps, ChaosRunResult } from './runner.js'
/**
* Run a single route with chaos injection and collect traces.
* Uses chaos-v3 pure functions for deterministic adversity.
*/
export async function runChaosOnRoute(
deps: QualifyRunnerDeps,
route: RouteContract,
chaosConfig: ChaosConfig,
): Promise<{ ctx: EvalContext; chaosResult: ChaosRunResult }> {
const started = Date.now()
// Generate chaos events using seeded RNG via fast-check
// For CLI qualify, we use a deterministic subset
const rng = new SeededRng(deps.seed)
const contractNames: string[] = []
// Build a minimal request for the route
const request = {
method: route.method,
url: route.path,
headers: {},
query: undefined as Record<string, string> | undefined,
body: undefined as unknown,
}
// Execute the request
const { executeHttp } = await import('../../../infrastructure/http-executor.js')
const ctx = await executeHttp(deps.fastify, route, request, undefined, deps.timeout)
// Generate and apply chaos events
const chaosArb = createChaosEventArbitrary(chaosConfig, contractNames)
// For deterministic CLI runs, we generate a fixed small set of events
// In practice, fast-check would be used in property tests; here we simulate
const events = generateDeterministicChaosEvents(chaosConfig, deps.seed)
const application = applyChaosToExecution(ctx, events)
const chaosResult: ChaosRunResult = {
applied: application.applied,
events: application.events
.filter(e => e.type !== 'none')
.map(e => formatChaosEvents([e])),
route: `${route.method} ${route.path}`,
durationMs: Date.now() - started,
}
return { ctx: application.ctx, chaosResult }
}
/**
* Generate a deterministic set of chaos events for CLI qualify mode.
* Uses seeded RNG for reproducibility.
*/
export function generateDeterministicChaosEvents(config: ChaosConfig, seed: number): import('../../../quality/chaos-v3.js').ChaosEvent[] {
const rng = new SeededRng(seed)
const events: import('../../../quality/chaos-v3.js').ChaosEvent[] = []
// Only inject chaos if probability threshold is met
if (config.probability <= 0 || rng.next() > config.probability) {
return events
}
// Pick one chaos type deterministically
const types: Array<'delay' | 'error' | 'dropout' | 'corruption'> = []
if (config.delay) types.push('delay')
if (config.error) types.push('error')
if (config.dropout) types.push('dropout')
if (config.corruption) types.push('corruption')
if (types.length === 0) return events
const chosen = types[Math.floor(rng.next() * types.length)]
if (!chosen) return events
switch (chosen) {
case 'delay': {
if (config.delay) {
const minMs = config.delay.minMs
const maxMs = config.delay.maxMs
const delayMs = minMs + Math.floor(rng.next() * (maxMs - minMs + 1))
events.push({
type: 'inbound-delay',
target: 'inbound',
delayMs,
})
}
break
}
case 'error': {
if (config.error) {
events.push({
type: 'inbound-error',
target: 'inbound',
statusCode: config.error.statusCode,
body: config.error.body,
})
}
break
}
case 'dropout': {
if (config.dropout) {
events.push({
type: 'inbound-dropout',
target: 'inbound',
statusCode: config.dropout.statusCode ?? 504,
})
}
break
}
case 'corruption': {
if (config.corruption) {
const strategies = ['truncate', 'malformed', 'field-corrupt'] as const
const strategy = strategies[Math.floor(rng.next() * strategies.length)]
events.push({
type: 'inbound-corruption',
target: 'inbound',
corruptionStrategy: strategy,
corruptionField: strategy === 'field-corrupt' ? 'id' : undefined,
})
}
break
}
}
return events
}
+868
View File
@@ -0,0 +1,868 @@
/**
* S6: Qualify thread - Qualify command handler
*
* Responsibilities:
* - Load config and resolve profile
* - Block prod runs by default (policy engine)
* - Run scenario/stateful/chaos based on profile
* - Generate seed if omitted, always print it
* - Rich artifact emission with step traces
* - Handle cleanup failures separately
* - Exit 0 on pass, 1 on qualification failure, 2 on safety violation
*
* Architecture:
* - Dependency injection: all dependencies passed explicitly
* - No optional imports — everything is required or injected
* - Inline comments for documentation
*/
import type { CliContext } from '../../core/context.js'
import { loadConfig } from '../../core/config-loader.js'
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
import { resolveGenerationProfileOverride, GenerationProfileResolutionError } from '../../core/generation-profile.js'
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
import type { CommandResult, Artifact, FailureRecord } from '../../core/types.js'
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
import {
runQualify,
resolveProfileGates,
type QualifyRunResult,
type StepTrace,
type CleanupFailure,
} from './runner.js'
import { SeededRng } from '../../../infrastructure/seeded-rng.js'
import type { ScenarioConfig, TestConfig, RouteContract, ChaosConfig } from '../../../types.js'
import { renderHumanArtifact } from '../../renderers/human.js'
import { renderJson, renderJsonArtifact, renderJsonSummaryArtifact } from '../../renderers/json.js'
import { renderNdjsonArtifact, renderNdjsonSummaryArtifact } from '../../renderers/ndjson.js'
import type { OutputContext } from '../../renderers/shared.js'
import { resolve } from 'node:path'
import { pathToFileURL } from 'node:url'
const ROUTE_IDENTITY_PATTERN = /^[A-Z]+\s+\/\S*$/
function normalizeRouteIdentity(route: string): string {
const normalized = route.trim().replace(/\s+/g, ' ')
const [method, ...pathParts] = normalized.split(' ')
if (!method || pathParts.length === 0) {
return normalized
}
return `${method.toUpperCase()} ${pathParts.join(' ')}`
}
function isReplayCompatibleRoute(route: string): boolean {
return ROUTE_IDENTITY_PATTERN.test(route)
}
function coerceDepth(value: unknown): TestConfig['depth'] {
if (value === 'quick' || value === 'standard' || value === 'thorough') {
return value
}
return 'standard'
}
function coerceTimeout(value: unknown): number | undefined {
return typeof value === 'number' ? value : undefined
}
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface QualifyOptions {
profile?: string
generationProfile?: string
seed?: number
config?: string
cwd?: string
format?: 'human' | 'json' | 'ndjson'
quiet?: boolean
verbose?: boolean
artifactDir?: string
}
interface FastifyAppLike {
ready?: () => Promise<void>
close?: () => Promise<void>
}
// ---------------------------------------------------------------------------
// Seed generation
// ---------------------------------------------------------------------------
/**
* Generate a deterministic seed if none provided.
* Uses current time + process pid + counter for uniqueness.
*/
let seedCounter = 0
export function generateSeed(): number {
seedCounter++
return Date.now() + (process.pid || 0) + seedCounter
}
// ---------------------------------------------------------------------------
// Route discovery helper
// ---------------------------------------------------------------------------
/**
* Discover routes from the Fastify app for chaos execution.
* Injected fastify instance must have routes registered.
*/
async function discoverAppRoutes(fastify: unknown): Promise<RouteContract[]> {
// Cast to access routes
const app = fastify as { routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> }> }
if (!app.routes) return []
return app.routes.map(r => ({
path: r.url,
method: r.method as RouteContract['method'],
category: 'observer',
requires: [],
ensures: [],
invariants: [],
regexPatterns: {},
validateRuntime: false,
schema: r.schema,
}))
}
// ---------------------------------------------------------------------------
// Scenario builder from profile
// ---------------------------------------------------------------------------
/**
* Build scenario configs from profile routes for protocol-lab fixture.
* Creates an OAuth-like multi-step scenario.
*/
function buildScenarioConfigs(routes: string[], seed: number): ScenarioConfig[] {
// For the protocol-lab fixture, build the OAuth scenario
const hasOAuth = routes.some(r => r.includes('/oauth/authorize'))
if (!hasOAuth) return []
const rng = new SeededRng(seed)
const clientId = `client-${Math.floor(rng.next() * 10000)}`
return [{
name: 'oauth-flow',
steps: [
{
name: 'authorize',
request: {
method: 'POST',
url: '/oauth/authorize',
body: {
client_id: clientId,
redirect_uri: 'http://localhost/callback',
scope: 'read',
},
},
expect: ['status:200', 'response_body(this).code != null'],
capture: { code: 'response_body(this).code' },
},
{
name: 'token',
request: {
method: 'POST',
url: '/oauth/token',
body: {
code: '$authorize.code',
client_id: clientId,
client_secret: 'secret',
redirect_uri: 'http://localhost/callback',
},
},
expect: ['status:200', 'response_body(this).access_token != null'],
capture: { accessToken: 'response_body(this).access_token' },
},
{
name: 'user',
request: {
method: 'GET',
url: '/api/user',
headers: {
authorization: 'Bearer $token.accessToken',
},
},
expect: ['status:200', 'response_body(this).id != null'],
},
],
}]
}
// ---------------------------------------------------------------------------
// Artifact builder
// ---------------------------------------------------------------------------
/**
* Build a rich artifact document from qualify results.
* Includes step traces, cleanup failures, and replay info.
*/
export function buildArtifact(
runResult: QualifyRunResult,
options: {
cwd: string
configPath?: string
profile?: string
preset?: string
env: string
seed: number
},
): Artifact {
const failures: FailureRecord[] = []
const warnings: string[] = []
const replayCompatibleExecutedRoutes = (runResult.executedRoutes || [])
.map(normalizeRouteIdentity)
.filter(isReplayCompatibleRoute)
// Collect scenario failures
for (const scenario of runResult.scenarioResults) {
if (!scenario.ok) {
for (let stepIdx = 0; stepIdx < scenario.steps.length; stepIdx++) {
const step = scenario.steps[stepIdx]!
if (!step.ok && step.diagnostics) {
// Use actual HTTP route from step trace for stable replay identity
const trace = runResult.stepTraces.find(
t => t.name === step.name && t.status === 'failed'
)
const route = normalizeRouteIdentity(trace?.route || `${scenario.name} / ${step.name}`)
if (!isReplayCompatibleRoute(route)) {
warnings.push(`Scenario step "${scenario.name}/${step.name}" did not resolve to METHOD /path route identity.`)
}
failures.push({
route,
contract: step.diagnostics.formula || 'scenario-step',
expected: step.diagnostics.expected || 'success',
observed: step.diagnostics.error || 'failure',
seed: runResult.seed,
replayCommand: `apophis replay --artifact <artifact-path-unavailable>`,
category: step.diagnostics.error ? classifyError(step.diagnostics.error) : ErrorTaxonomy.RUNTIME,
diff: step.diagnostics.diff ?? undefined,
actual: step.diagnostics.actual ?? undefined,
})
}
}
}
}
// Collect stateful failures
if (runResult.statefulResult) {
let fallbackRouteIdx = 0
for (const test of runResult.statefulResult.tests) {
if (!test.ok) {
let route = normalizeRouteIdentity(test.name)
if (!isReplayCompatibleRoute(route)) {
route = replayCompatibleExecutedRoutes[fallbackRouteIdx] || route
fallbackRouteIdx++
}
if (!isReplayCompatibleRoute(route)) {
warnings.push(`Stateful failure "${test.name}" did not resolve to METHOD /path route identity.`)
}
failures.push({
route,
contract: test.diagnostics?.formula || 'stateful-test',
expected: test.diagnostics?.expected || 'success',
observed: test.diagnostics?.error || 'failure',
seed: runResult.seed,
replayCommand: `apophis replay --artifact <artifact-path-unavailable>`,
category: test.diagnostics?.error ? classifyError(test.diagnostics.error) : ErrorTaxonomy.RUNTIME,
diff: test.diagnostics?.diff ?? undefined,
actual: test.diagnostics?.actual ?? undefined,
})
}
}
}
const totalTests =
runResult.scenarioResults.reduce((sum, s) => sum + s.steps.length, 0) +
(runResult.statefulResult?.tests.length ?? 0)
const passedTests =
runResult.scenarioResults.reduce((sum, s) => sum + s.summary.passed, 0) +
(runResult.statefulResult?.summary.passed ?? 0)
if (runResult.cleanupFailures.length > 0) {
warnings.push(
`Cleanup failures: ${runResult.cleanupFailures.map(c => `${c.resource}: ${c.error}`).join(', ')}`
)
}
// Build cleanup outcomes from cleanup failures
const cleanupOutcomes = runResult.cleanupFailures.map(cf => ({
resource: cf.resource,
cleaned: false,
error: cf.error,
}))
// Build execution summary from runner result
const executionSummary = runResult.executionSummary
// Build profile gates from the result context
// We need to pass gates through or infer from results
const profileGates = {
scenario: runResult.scenarioResults.length > 0 || executionSummary.scenariosRun > 0,
stateful: (runResult.statefulResult?.tests.length ?? 0) > 0 || executionSummary.statefulTestsRun > 0,
chaos: (runResult.chaosResult !== undefined) || executionSummary.chaosRunsRun > 0,
}
// Deterministic parameters for audit
const deterministicParams = {
seed: runResult.seed,
profileGates,
}
return {
version: 'apophis-artifact/1',
command: 'qualify',
mode: 'qualify',
cwd: options.cwd,
configPath: options.configPath,
profile: options.profile,
preset: options.preset,
env: options.env,
seed: options.seed,
startedAt: new Date(Date.now() - runResult.durationMs).toISOString(),
durationMs: runResult.durationMs,
summary: {
total: totalTests,
passed: passedTests,
failed: failures.length,
},
executionSummary,
executedRoutes: (runResult.executedRoutes || []).map(normalizeRouteIdentity),
skippedRoutes: (runResult.skippedRoutes || []).map(sr => ({
route: sr.route,
executed: false,
reason: sr.reason,
})),
stepTraces: runResult.stepTraces,
cleanupOutcomes,
profileGates,
deterministicParams,
failures,
artifacts: [],
warnings,
exitReason: runResult.passed ? 'success' : 'behavioral_failure',
}
}
function attachReplayCommands(artifact: Artifact, artifactPath: string): void {
for (const failure of artifact.failures) {
failure.replayCommand = `apophis replay --artifact ${artifactPath}`
}
}
async function emitArtifact(
artifact: Artifact,
options: {
command: 'qualify'
cwd: string
preferredDir?: string
force: boolean
},
): Promise<string | undefined> {
if (!options.force && !options.preferredDir) {
return undefined
}
const defaultDir = resolve(options.cwd, 'reports', 'apophis')
const candidateDirs = [options.preferredDir, defaultDir].filter(Boolean) as string[]
const attempted = new Set<string>()
for (const dir of candidateDirs) {
if (attempted.has(dir)) continue
attempted.add(dir)
try {
const { mkdirSync, writeFileSync } = await import('node:fs')
const artifactPath = resolve(dir, `${options.command}-${new Date().toISOString().replace(/[:.]/g, '-')}.json`)
mkdirSync(dir, { recursive: true })
attachReplayCommands(artifact, artifactPath)
writeFileSync(artifactPath, JSON.stringify(artifact, null, 2))
if (!artifact.artifacts.includes(artifactPath)) {
artifact.artifacts.push(artifactPath)
}
return artifactPath
} catch {
// Try fallback directory if available.
}
}
artifact.warnings.push('Failed to write artifact to disk')
return undefined
}
// ---------------------------------------------------------------------------
// Output formatting
// ---------------------------------------------------------------------------
function formatHumanOutput(
result: QualifyRunResult,
options: { profile?: string; seed: number; env: string },
): string {
const lines: string[] = []
lines.push(`Qualify run for environment "${options.env}"`)
if (options.profile) {
lines.push(`Profile: ${options.profile}`)
}
lines.push(`Seed: ${options.seed}`)
lines.push('')
// Scenario results
for (const scenario of result.scenarioResults) {
lines.push(`Scenario: ${scenario.name}`)
for (const step of scenario.steps) {
const icon = step.ok ? '✓' : '✗'
lines.push(` ${icon} ${step.name} (${step.statusCode ?? 'no-status'})`)
if (!step.ok && step.diagnostics) {
lines.push(` Expected: ${step.diagnostics.expected || 'success'}`)
lines.push(` Observed: ${step.diagnostics.error || 'failure'}`)
if (step.diagnostics.actual) {
lines.push(` Actual: ${step.diagnostics.actual}`)
}
if (step.diagnostics.diff) {
lines.push(` Diff:`)
for (const line of String(step.diagnostics.diff).split('\n')) {
lines.push(` ${line}`)
}
}
}
}
lines.push('')
}
// Stateful results
if (result.statefulResult) {
lines.push(`Stateful: ${result.statefulResult.summary.passed} passed, ${result.statefulResult.summary.failed} failed`)
lines.push('')
}
// Chaos results
if (result.chaosResult) {
lines.push(`Chaos: ${result.chaosResult.applied ? 'applied' : 'none'}`)
if (result.chaosResult.events.length > 0) {
for (const event of result.chaosResult.events) {
lines.push(` ${event}`)
}
}
lines.push('')
}
// Step traces
if (result.stepTraces.length > 0) {
lines.push('Step traces:')
for (const trace of result.stepTraces.slice(0, 20)) {
const icon = trace.status === 'passed' ? '✓' : trace.status === 'skipped' ? '⊘' : '✗'
lines.push(` ${icon} ${trace.name} (${trace.durationMs}ms)`)
}
if (result.stepTraces.length > 20) {
lines.push(` ... and ${result.stepTraces.length - 20} more`)
}
lines.push('')
}
// Cleanup failures
if (result.cleanupFailures.length > 0) {
lines.push('Cleanup failures (reported separately):')
for (const cf of result.cleanupFailures) {
lines.push(`${cf.resource}: ${cf.error}`)
}
lines.push('')
}
// Per-profile gate execution counts
lines.push('Profile gate execution counts:')
lines.push(` Scenario: ${result.executionSummary.scenariosRun} run`)
lines.push(` Stateful: ${result.executionSummary.statefulTestsRun} tests run`)
lines.push(` Chaos: ${result.executionSummary.chaosRunsRun} runs run`)
lines.push('')
// Executed routes
if (result.executedRoutes.length > 0) {
lines.push(`Executed routes (${result.executedRoutes.length}):`)
for (const route of result.executedRoutes) {
lines.push(` ${route}`)
}
lines.push('')
}
// Skipped routes
if (result.skippedRoutes.length > 0) {
lines.push(`Skipped routes (${result.skippedRoutes.length}):`)
for (const sr of result.skippedRoutes) {
lines.push(` ${sr.route}: ${sr.reason}`)
}
lines.push('')
}
// Summary
if (result.passed) {
lines.push('All qualifications passed.')
} else {
lines.push('Qualification failed.')
lines.push(`Replay: apophis replay --artifact <artifact-path>`)
}
return lines.join('\n')
}
// ---------------------------------------------------------------------------
// Main command handler
// ---------------------------------------------------------------------------
/**
* Main qualify command handler.
*
* Flow:
* 1. Load and resolve config
* 2. Run policy engine checks (block prod by default)
* 3. Generate seed if omitted, always print it
* 4. Resolve profile gates (scenario/stateful/chaos)
* 5. Build scenario configs from profile routes
* 6. Run execution modes
* 7. Build rich artifact with step traces
* 8. Handle cleanup failures separately
* 9. Return appropriate exit code
*/
export async function qualifyCommand(
options: QualifyOptions,
ctx: CliContext,
): Promise<CommandResult> {
const {
profile,
generationProfile,
seed: explicitSeed,
config: configPath,
cwd,
artifactDir,
} = options
const workingDir = cwd || ctx.cwd
const format = options.format || ctx.options.format || 'human'
// Detect environment
const env = detectEnvironment()
try {
// 1. Load config
const loadResult = await loadConfig({
cwd: workingDir,
configPath,
profileName: profile,
env,
})
if (!loadResult.configPath) {
return {
exitCode: USAGE_ERROR,
message: 'No config found. Run "apophis init" to create one.',
}
}
const config = loadResult.config
const resolvedGenerationProfile = resolveGenerationProfileOverride(generationProfile, config)
// 2. Run policy engine checks
const policyEngine = new PolicyEngine({
config,
env,
mode: 'qualify',
profileName: profile || undefined,
presetName: loadResult.presetName || undefined,
})
const policyResult = policyEngine.check()
if (!policyResult.allowed) {
const message = [
'Policy check failed:',
...policyResult.errors.map(e => `${e}`),
].join('\n')
return {
exitCode: USAGE_ERROR,
message,
}
}
// 3. Generate seed if omitted
const seed = explicitSeed ?? generateSeed()
if (!ctx.options.quiet && format === 'human') {
console.log(`Seed: ${seed}`)
}
// 4. Resolve profile gates
const profileDef = profile ? config.profiles?.[profile] : undefined
const gates = resolveProfileGates(profileDef?.features)
// 5. Build scenario configs from profile routes
const routes = profileDef?.routes ?? []
const scenarios = buildScenarioConfigs(routes, seed)
// 6. Build stateful config
const presetName = profileDef?.preset
const preset = presetName ? config.presets?.[presetName] : undefined
const presetDepth = coerceDepth((preset as { depth?: unknown } | undefined)?.depth)
const presetTimeout = coerceTimeout((preset as { timeout?: unknown } | undefined)?.timeout)
const statefulConfig: TestConfig | undefined = gates.stateful
? {
depth: presetDepth,
generationProfile: resolvedGenerationProfile,
seed,
timeout: presetTimeout,
routes: profileDef?.routes,
}
: undefined
// 7. Build chaos config
const chaosConfig: ChaosConfig | undefined = gates.chaos && preset?.chaos
? {
probability: 0.5,
delay: { probability: 0.3, minMs: 100, maxMs: 500 },
error: { probability: 0.2, statusCode: 503 },
dropout: { probability: 0.2, statusCode: 504 },
corruption: { probability: 0.1 },
}
: undefined
// 8. Load the Fastify app for execution
// Try to import the app from the fixture
let fastify: FastifyAppLike | undefined
try {
const appPath = resolve(workingDir, 'app.js')
const appUrl = pathToFileURL(appPath)
appUrl.searchParams.set('apophisRun', String(Date.now()))
const appModule = await import(appUrl.href)
fastify = (appModule.default || appModule) as FastifyAppLike
if (fastify && typeof fastify.ready === 'function') {
await fastify.ready()
}
} catch (err) {
// App not available — return a result indicating no app to test
if (process.env.APOPHIS_DEBUG === '1') {
console.error('Failed to load app:', err)
}
return {
exitCode: USAGE_ERROR,
message: 'No Fastify app found. Ensure app.js exports a Fastify instance.',
}
}
try {
// 9. Discover routes for chaos
const appRoutes = await discoverAppRoutes(fastify)
// 10. Run qualify execution
const deps = {
fastify: fastify as any,
seed,
timeout: presetTimeout,
}
const runResult = await runQualify(deps, gates, scenarios, statefulConfig, chaosConfig, appRoutes)
// 11. Build artifact first so we can reference it for guardrails
const artifact = buildArtifact(runResult, {
cwd: workingDir,
configPath: loadResult.configPath,
profile: profile || undefined,
preset: presetName,
env,
seed,
})
// 12. Signal quality guardrails — fail if zero checks executed
const execSummary = runResult.executionSummary
const warnings: string[] = [...artifact.warnings]
if (execSummary.totalExecuted === 0) {
await emitArtifact(artifact, {
command: 'qualify',
cwd: workingDir,
preferredDir: artifactDir || config.artifactDir,
force: true,
})
return {
exitCode: BEHAVIORAL_FAILURE,
message: 'Qualify failed: zero checks executed. No scenarios, stateful tests, or chaos runs were performed. Verify profile gates and app configuration.',
artifact,
warnings: artifact.warnings,
}
}
// Warn if execution counts are suspiciously low
if (gates.scenario && execSummary.scenariosRun === 0) {
warnings.push('WARNING: scenario gate enabled but zero scenarios executed. Check route configuration.')
}
if (gates.stateful && execSummary.statefulTestsRun === 0) {
warnings.push('WARNING: stateful gate enabled but zero stateful tests executed. Check app routes and schema.')
}
if (gates.chaos && execSummary.chaosRunsRun === 0) {
warnings.push('WARNING: chaos gate enabled but zero chaos runs executed. Check chaos config and route availability.')
}
// 12. Write artifact if configured or on failure
const shouldEmitArtifact = Boolean(artifactDir || config.artifactDir || !runResult.passed)
await emitArtifact(artifact, {
command: 'qualify',
cwd: workingDir,
preferredDir: artifactDir || config.artifactDir,
force: shouldEmitArtifact,
})
// 13. Format output based on format option
const outputCtx: OutputContext = {
isTTY: ctx.isTTY,
isCI: ctx.isCI,
colorMode: ctx.options.color,
}
let message = ''
if (!ctx.options.quiet) {
if (format === 'json') {
message = renderJsonArtifact(artifact)
} else if (format === 'json-summary') {
message = renderJsonSummaryArtifact(artifact)
} else if (format === 'ndjson') {
// For ndjson, we don't return a message string; events are streamed
message = ''
} else if (format === 'ndjson-summary') {
// Concise ndjson: only summary events
message = ''
} else {
// human format
message = renderHumanArtifact(artifact, outputCtx)
}
}
return {
exitCode: runResult.passed ? SUCCESS : BEHAVIORAL_FAILURE,
artifact,
message,
warnings: artifact.warnings,
}
} finally {
if (fastify && typeof fastify.close === 'function') {
try {
await fastify.close()
} catch (closeErr) {
if (process.env.APOPHIS_DEBUG === '1') {
console.error('Failed to close Fastify app after qualify run:', closeErr)
}
}
}
}
} catch (error) {
if (error instanceof GenerationProfileResolutionError) {
return {
exitCode: USAGE_ERROR,
message: error.message,
}
}
const message = error instanceof Error ? error.message : String(error)
return {
exitCode: INTERNAL_ERROR,
message: `Internal error in qualify command: ${message}`,
}
}
}
// ---------------------------------------------------------------------------
// CLI adapter
// ---------------------------------------------------------------------------
/**
* Adapter that bridges the CLI framework (cac) to the qualify command handler.
* This function signature matches what the CLI core expects.
*/
export async function handleQualify(
args: string[],
ctx: CliContext,
): Promise<number> {
const options: QualifyOptions = {
profile: ctx.options.profile || undefined,
generationProfile: ctx.options.generationProfile,
seed: undefined,
config: ctx.options.config || undefined,
cwd: ctx.cwd,
format: ctx.options.format as QualifyOptions['format'],
quiet: ctx.options.quiet,
verbose: ctx.options.verbose,
artifactDir: ctx.options.artifactDir || undefined,
}
const seedIdx = args.indexOf('--seed')
if (seedIdx !== -1 && args[seedIdx + 1]) {
const parsed = parseInt(args[seedIdx + 1]!, 10)
if (!isNaN(parsed)) {
options.seed = parsed
}
}
const generationProfileIdx = args.indexOf('--generation-profile')
if (generationProfileIdx !== -1 && args[generationProfileIdx + 1]) {
options.generationProfile = args[generationProfileIdx + 1]
}
const result = await qualifyCommand(options, ctx)
const format = options.format || ctx.options.format || 'human'
const machineMode = format === 'json' || format === 'ndjson' || format === 'json-summary' || format === 'ndjson-summary'
if (!ctx.options.quiet) {
if (format === 'json') {
if (result.artifact) {
console.log(renderJsonArtifact(result.artifact))
} else {
console.log(renderJson({
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}))
}
} else if (format === 'json-summary') {
if (result.artifact) {
console.log(renderJsonSummaryArtifact(result.artifact))
} else {
console.log(renderJson({
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}))
}
} else if (format === 'ndjson') {
if (result.artifact) {
renderNdjsonArtifact(result.artifact)
} else {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'qualify',
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}) + '\n')
}
} else if (format === 'ndjson-summary') {
if (result.artifact) {
renderNdjsonSummaryArtifact(result.artifact)
} else {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'qualify',
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}) + '\n')
}
} else if (result.message) {
console.log(result.message)
}
}
// Print warnings in human mode only
if (!machineMode && result.warnings && result.warnings.length > 0 && !ctx.options.quiet) {
for (const warning of result.warnings) {
console.warn(`Warning: ${warning}`)
}
}
return result.exitCode
}
+255
View File
@@ -0,0 +1,255 @@
/**
* S6: Qualify thread - Runner for scenario, stateful, and chaos execution
*
* Responsibilities:
* - Scenario execution (multi-step flows with capture/rebind)
* - Stateful execution (model-based property testing)
* - Chaos execution (adversity injection via chaos-v3)
* - Profile gating logic (determine which execution modes to run)
* - Step trace collection for rich artifacts
* - Cleanup failure tracking (reported separately)
*
* Architecture:
* - Pure execution functions that accept injected dependencies
* - No optional imports — everything is passed via constructor/parameters
* - Step traces collected as arrays and returned in result
*/
import { runScenarioWithTraces } from './scenario-handler.js'
import { runStatefulWithTraces } from './stateful-handler.js'
import { runChaosOnRoute } from './chaos-handler.js'
import { SeededRng } from '../../../infrastructure/seeded-rng.js'
import type {
ScenarioConfig,
ScenarioResult,
TestConfig,
TestSuite,
RouteContract,
ChaosConfig,
FastifyInjectInstance,
} from '../../../types.js'
import type { ExtensionRegistry } from '../../../extension/types.js'
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface StepTrace {
step: number
name: string
route: string
durationMs: number
status: 'passed' | 'failed' | 'skipped'
error?: string
}
export interface QualifyRunResult {
passed: boolean
scenarioResults: ScenarioResult[]
statefulResult?: TestSuite
chaosResult?: ChaosRunResult
stepTraces: StepTrace[]
cleanupFailures: CleanupFailure[]
durationMs: number
seed: number
executionSummary: {
totalPlanned: number
totalExecuted: number
totalPassed: number
totalFailed: number
scenariosRun: number
statefulTestsRun: number
chaosRunsRun: number
totalSteps: number
}
executedRoutes: string[]
skippedRoutes: { route: string; reason: string }[]
}
export interface ChaosRunResult {
applied: boolean
events: string[]
route: string
durationMs: number
}
export interface CleanupFailure {
resource: string
error: string
}
export interface QualifyRunnerDeps {
fastify: FastifyInjectInstance
extensionRegistry?: ExtensionRegistry
seed: number
timeout?: number
}
// ---------------------------------------------------------------------------
// Profile gating logic
// ---------------------------------------------------------------------------
export interface ProfileGates {
scenario: boolean
stateful: boolean
chaos: boolean
}
/**
* Determine which execution modes to enable based on profile features.
* Default: all enabled if no features specified.
*/
export function resolveProfileGates(features?: string[]): ProfileGates {
if (!features || features.length === 0) {
return { scenario: true, stateful: true, chaos: true }
}
return {
scenario: features.includes('scenario') || features.includes('protocol-flow'),
stateful: features.includes('stateful'),
chaos: features.includes('chaos'),
}
}
// ---------------------------------------------------------------------------
// Main qualify runner
// ---------------------------------------------------------------------------
/**
* Run all qualify execution modes based on profile gates.
* Collects step traces, handles cleanup failures separately.
*/
export async function runQualify(
deps: QualifyRunnerDeps,
gates: ProfileGates,
scenarios: ScenarioConfig[],
statefulConfig?: TestConfig,
chaosConfig?: ChaosConfig,
routes?: RouteContract[],
): Promise<QualifyRunResult> {
const started = Date.now()
const scenarioResults: ScenarioResult[] = []
const allTraces: StepTrace[] = []
const cleanupFailures: CleanupFailure[] = []
let statefulResult: TestSuite | undefined
let chaosResult: ChaosRunResult | undefined
// Run scenarios
if (gates.scenario) {
for (const scenarioConfig of scenarios) {
const { result, traces } = await runScenarioWithTraces(deps, scenarioConfig)
scenarioResults.push(result)
allTraces.push(...traces)
}
}
// Run stateful tests
if (gates.stateful && statefulConfig) {
const { result, traces } = await runStatefulWithTraces(deps, statefulConfig)
statefulResult = result
allTraces.push(...traces)
}
// Run chaos on routes
if (gates.chaos && chaosConfig && routes && routes.length > 0) {
// Pick one route deterministically for CLI chaos demo
const rng = new SeededRng(deps.seed)
const route = routes[Math.floor(rng.next() * routes.length)]
if (route) {
const { chaosResult: cr } = await runChaosOnRoute(deps, route, chaosConfig)
chaosResult = cr
}
}
// Simulate cleanup tracking
// In real usage, cleanupManager would be injected and tracked
// For now, cleanup failures are empty unless injected by caller
const durationMs = Date.now() - started
// Determine overall pass/fail
const scenarioPassed = scenarioResults.every(r => r.ok)
const statefulPassed = !statefulResult || statefulResult.summary.failed === 0
const chaosPassed = !chaosResult || chaosResult.applied // chaos "passes" if it applied
// Count execution metrics
const scenariosRun = scenarioResults.length
const statefulTestsRun = statefulResult?.tests.length ?? 0
const chaosRunsRun = chaosResult ? 1 : 0
const totalSteps = allTraces.length
const totalExecuted = scenariosRun + statefulTestsRun + chaosRunsRun
const totalPassed = scenarioResults.reduce((sum, r) => sum + r.summary.passed, 0) +
(statefulResult?.summary.passed ?? 0) +
(chaosResult?.applied ? 1 : 0)
const totalFailed = scenarioResults.reduce((sum, r) => sum + r.summary.failed, 0) +
(statefulResult?.summary.failed ?? 0)
// Track executed and skipped routes for transparency
const executedRoutes: string[] = []
const skippedRoutes: { route: string; reason: string }[] = []
// Track scenario routes
for (const scenario of scenarioResults) {
for (const step of scenario.steps) {
const trace = allTraces.find(t => t.name === step.name)
if (trace) {
executedRoutes.push(trace.route)
}
}
}
// Track stateful test routes
if (statefulResult) {
for (const test of statefulResult.tests) {
executedRoutes.push(test.name)
}
}
// Track chaos route
if (chaosResult) {
executedRoutes.push(chaosResult.route)
}
// Track skipped routes from profile filters
if (routes) {
const executedSet = new Set(executedRoutes)
for (const route of routes) {
const routeStr = `${route.method} ${route.path}`
if (!executedSet.has(routeStr)) {
let reason = 'Not selected for execution'
if (!gates.scenario && !gates.stateful && !gates.chaos) {
reason = 'All profile gates disabled'
} else if (gates.scenario && !scenarios.some(s => s.steps.some(st => st.request.url === route.path))) {
reason = 'No scenario covers this route'
} else if (gates.stateful && !statefulConfig) {
reason = 'Stateful config missing or invalid'
} else if (gates.chaos && !chaosConfig) {
reason = 'Chaos config missing or invalid'
}
skippedRoutes.push({ route: routeStr, reason })
}
}
}
return {
passed: scenarioPassed && statefulPassed && chaosPassed,
scenarioResults,
statefulResult,
chaosResult,
stepTraces: allTraces,
cleanupFailures,
durationMs,
seed: deps.seed,
executionSummary: {
totalPlanned: scenarios.length + (statefulConfig ? 1 : 0) + (chaosConfig && routes && routes.length > 0 ? 1 : 0),
totalExecuted,
totalPassed,
totalFailed,
scenariosRun,
statefulTestsRun,
chaosRunsRun,
totalSteps,
},
executedRoutes: [...new Set(executedRoutes)],
skippedRoutes,
}
}
@@ -0,0 +1,55 @@
/**
* S6: Qualify thread - Scenario execution handler
*
* Responsibilities:
* - Run scenario configs and collect step traces
* - Wrap the scenario-runner with trace collection
*
* Architecture:
* - Pure execution function that accepts injected dependencies
* - No optional imports — everything is passed via parameters
*/
import { runScenario } from '../../../test/scenario-runner.js'
import type {
ScenarioConfig,
ScenarioResult,
} from '../../../types.js'
import type { QualifyRunnerDeps, StepTrace } from './runner.js'
/**
* Run a scenario config and collect step traces.
* Returns the scenario result plus per-step traces.
*/
export async function runScenarioWithTraces(
deps: QualifyRunnerDeps,
config: ScenarioConfig,
): Promise<{ result: ScenarioResult; traces: StepTrace[] }> {
const scopeHeaders: Record<string, string> = {}
const result = await runScenario(deps.fastify, config, scopeHeaders, deps.extensionRegistry)
const traces: StepTrace[] = result.steps.map((step, idx) => {
const trace: StepTrace = {
step: idx + 1,
name: step.name,
route: `${config.steps[idx]?.request.method ?? 'UNKNOWN'} ${config.steps[idx]?.request.url ?? 'UNKNOWN'}`,
durationMs: 0, // scenario-runner doesn't track per-step timing; use total
status: step.ok ? 'passed' : 'failed',
}
if (!step.ok && step.diagnostics) {
trace.error = typeof step.diagnostics.error === 'string'
? step.diagnostics.error
: JSON.stringify(step.diagnostics.error)
}
return trace
})
// Distribute total time across steps roughly
const perStepMs = result.summary.timeMs / Math.max(result.steps.length, 1)
for (const trace of traces) {
trace.durationMs = perStepMs
}
return { result, traces }
}
@@ -0,0 +1,55 @@
/**
* S6: Qualify thread - Stateful execution handler
*
* Responsibilities:
* - Run stateful tests with the given config
* - Wrap the existing stateful runner with trace collection
*
* Architecture:
* - Pure execution function that accepts injected dependencies
* - No optional imports — everything is passed via parameters
*/
import { runStatefulTests } from '../../../test/stateful-runner.js'
import type {
TestConfig,
TestSuite,
} from '../../../types.js'
import type { QualifyRunnerDeps, StepTrace } from './runner.js'
/**
* Run stateful tests with the given config.
* Wraps the existing stateful runner.
*/
export async function runStatefulWithTraces(
deps: QualifyRunnerDeps,
config: TestConfig,
): Promise<{ result: TestSuite; traces: StepTrace[] }> {
const started = Date.now()
const result = await runStatefulTests(
deps.fastify,
config,
undefined, // cleanupManager — injected if needed by caller
undefined, // scopeRegistry
deps.extensionRegistry,
undefined, // pluginContractRegistry
undefined, // outboundContractRegistry
)
const traces: StepTrace[] = result.tests.map((test, idx) => ({
step: idx + 1,
name: test.name,
route: test.name, // stateful tests name includes route
durationMs: 0,
status: test.ok ? 'passed' : test.directive ? 'skipped' : 'failed',
error: test.diagnostics?.error,
}))
const perStepMs = (Date.now() - started) / Math.max(traces.length, 1)
for (const trace of traces) {
trace.durationMs = perStepMs
}
return { result, traces }
}
+569
View File
@@ -0,0 +1,569 @@
/**
* S7: Replay thread - Replay command handler
*
* Responsibilities:
* - Load artifact from --artifact path
* - Validate artifact schema version
* - Check CLI version compatibility
* - Re-run the failing route/contract with the same seed
* - Handle source code changes since artifact (warn but attempt)
* - Handle missing/corrupted artifacts
* - Handle route no longer existing
* - Fast startup (must feel instant)
* - Exit 0 if replay reproduces same failure, 1 if different, 2 on error
*
* Architecture:
* - Dependency injection: all dependencies passed explicitly
* - No optional imports — everything is required or injected
* - Inline comments for documentation
* - Reuses verify runner for actual replay execution
*/
import type { CliContext } from '../../core/context.js'
import { loadConfig } from '../../core/config-loader.js'
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
import type { CommandResult, Artifact, FailureRecord } from '../../core/types.js'
import { runVerify } from '../verify/runner.js'
import { loadArtifact, type ArtifactLoadResult } from './loader.js'
import { renderJson } from '../../renderers/json.js'
import type { OutputContext } from '../../renderers/shared.js'
import { executeHttp } from '../../../infrastructure/http-executor.js'
import { parse } from '../../../formula/parser.js'
import { evaluateAsync } from '../../../formula/evaluator.js'
import { createOperationResolver } from '../../../formula/runtime.js'
import type { EvalContext, RouteContract } from '../../../types.js'
import { resolve } from 'node:path'
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface ReplayOptions {
artifact: string
config?: string
cwd?: string
format?: 'human' | 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary'
quiet?: boolean
verbose?: boolean
route?: string
}
export interface ReplayResult {
exitCode: number
message?: string
warnings?: string[]
reproduced: boolean
originalFailure?: FailureRecord
newFailure?: FailureRecord
}
// ---------------------------------------------------------------------------
// Human output formatting
// ---------------------------------------------------------------------------
/**
* Format human-readable output for replay results.
*/
function formatHumanOutput(result: ReplayResult, artifact: Artifact): string {
const lines: string[] = []
const sourceDriftDetected = (result.warnings || []).some(w =>
w.includes('Source code has changed since artifact was created') ||
w.includes('modified since artifact was created') ||
w.includes('Artifact cwd no longer exists')
)
if (result.reproduced) {
lines.push('Replay reproduced the original failure.')
lines.push('')
lines.push('Original failure')
lines.push(` Route: ${result.originalFailure?.route}`)
lines.push(` Contract: ${result.originalFailure?.contract}`)
lines.push(` Expected: ${result.originalFailure?.expected}`)
lines.push(` Observed: ${result.originalFailure?.observed}`)
lines.push(` Seed: ${artifact.seed}`)
} else if (result.newFailure) {
lines.push('Replay produced a different result.')
lines.push('')
lines.push('Original failure')
lines.push(` Route: ${result.originalFailure?.route}`)
lines.push(` Contract: ${result.originalFailure?.contract}`)
lines.push('')
lines.push('New result')
lines.push(` Route: ${result.newFailure.route}`)
lines.push(` Contract: ${result.newFailure.contract}`)
lines.push(` Expected: ${result.newFailure.expected}`)
lines.push(` Observed: ${result.newFailure.observed}`)
lines.push(` Seed: ${artifact.seed}`)
} else {
lines.push('Replay passed — failure no longer reproduces.')
lines.push('')
lines.push('Original failure')
lines.push(` Route: ${result.originalFailure?.route}`)
lines.push(` Contract: ${result.originalFailure?.contract}`)
lines.push(` Seed: ${artifact.seed}`)
}
// Add trust labeling and stabilization guidance when replay does not exactly match.
if (!result.reproduced) {
lines.push('')
lines.push('Replay confidence')
if (sourceDriftDetected) {
lines.push(' Degraded: source drift detected since artifact creation; exact reproduction is not guaranteed.')
} else {
lines.push(' Degraded: same-seed replay diverged without source drift; likely runtime/data nondeterminism.')
}
lines.push('')
lines.push('Stabilization guidance:')
lines.push(' 1. Ensure the app database/state is reset to a known baseline')
lines.push(' 2. Run with --seed for explicit control')
lines.push(' 3. Freeze time/randomness in app code and isolate external dependencies')
lines.push(' 4. Disable chaos/stateful gates in profile if not needed for this failure')
}
if (result.warnings && result.warnings.length > 0) {
lines.push('')
lines.push('Warnings')
for (const warning of result.warnings) {
lines.push(`${warning}`)
}
}
return lines.join('\n')
}
// ---------------------------------------------------------------------------
// Direct contract execution (bypasses route discovery)
// ---------------------------------------------------------------------------
/**
* Execute a contract directly against a Fastify instance without route discovery.
* Used by replay when the app doesn't have APOPHIS plugin pre-registered.
*/
async function executeContractDirect(
fastify: any,
route: string,
contract: string,
seed: number,
): Promise<{ success: boolean; observed?: string }> {
// Parse route into method and path
const parts = route.split(' ')
const method = parts[0] || 'GET'
const path = parts.slice(1).join(' ')
// Check if route exists using hasRoute
const hasRoute = typeof fastify.hasRoute === 'function' &&
fastify.hasRoute({ url: path, method })
if (!hasRoute) {
return { success: false, observed: `Route "${route}" no longer exists` }
}
// Build a minimal route contract
const routeContract: RouteContract = {
method: method as RouteContract['method'],
path,
category: 'observer',
schema: {},
requires: [],
ensures: [contract],
invariants: [],
regexPatterns: {},
validateRuntime: false,
}
// Build request
const headers: Record<string, string> = {}
// Execute request
try {
const ctx = await executeHttp(fastify, routeContract, {
method,
url: path,
headers,
query: {},
})
// Build eval context
const evalCtx: EvalContext = {
...ctx,
operationResolver: createOperationResolver(fastify, headers, ctx),
}
// Parse and evaluate contract
const parsed = parse(contract)
const result = await evaluateAsync(parsed.ast, evalCtx)
if (!result.success || !result.value) {
return {
success: false,
observed: result.success ? String(result.value) : result.error,
}
}
return { success: true }
} catch (error) {
return {
success: false,
observed: error instanceof Error ? error.message : String(error),
}
}
}
// ---------------------------------------------------------------------------
// Replay execution
// ---------------------------------------------------------------------------
/**
* Run the replay by re-executing verify with the same seed and route filter.
*
* Flow:
* 1. Load the Fastify app from artifact.cwd
* 2. Run verify with the artifact's seed and route filter
* 3. Compare results to the original failure
* 4. Return whether the failure was reproduced
*/
async function executeReplay(
artifact: Artifact,
failure: FailureRecord,
artifactPath: string,
ctx: CliContext,
options?: { sourceChanged?: boolean },
): Promise<ReplayResult> {
const workingDir = artifact.cwd
const warnings: string[] = []
// Load the Fastify app
let fastify: unknown
try {
const { loadApp } = await import('../../core/app-loader.js')
const loaded = await loadApp(workingDir)
fastify = loaded.fastify
if (fastify && typeof (fastify as any).ready === 'function') {
// Only register APOPHIS plugin if not already registered
// The fixture apps already register it, so re-registering throws
const hasApophis = (fastify as any).apophis !== undefined
const canRegister = typeof (fastify as any).register === 'function'
if (!hasApophis && canRegister) {
const { apophisPlugin } = await import('../../../plugin/index.js')
if (typeof apophisPlugin === 'function') {
await (fastify as any).register(apophisPlugin, { runtime: 'off' })
}
}
await (fastify as any).ready()
}
} catch (err) {
const errorMessage = err instanceof Error ? err.message : String(err)
return {
exitCode: USAGE_ERROR,
message: `Cannot load Fastify app from ${workingDir}/app.js: ${errorMessage}`,
warnings,
reproduced: false,
originalFailure: failure,
}
}
// Try to run verify first (works if app has APOPHIS plugin)
let runResult = await runVerify({
fastify: fastify as any,
seed: artifact.seed || 42,
routeFilters: [failure.route],
})
// If no routes matched, or route found but no contracts (plugin not registered before routes),
// try direct contract execution
if (runResult.noRoutesMatched || runResult.noContractsFound) {
const directResult = await executeContractDirect(
fastify as any,
failure.route,
failure.contract,
artifact.seed || 42,
)
if (!directResult.success) {
// Check if it's a route-not-found error
if (directResult.observed?.includes('no longer exists')) {
return {
exitCode: USAGE_ERROR,
message: `Route "${failure.route}" no longer exists in the application.\n` +
`The source code has drifted since the artifact was created.`,
warnings: [...warnings, `Route "${failure.route}" no longer exists`],
reproduced: false,
originalFailure: failure,
}
}
// Same failure reproduced via direct execution
return {
exitCode: BEHAVIORAL_FAILURE,
message: formatHumanOutput({
exitCode: BEHAVIORAL_FAILURE,
reproduced: true,
originalFailure: failure,
warnings,
}, artifact),
warnings,
reproduced: true,
originalFailure: failure,
}
}
// Direct execution passed — failure no longer reproduces
return {
exitCode: SUCCESS,
message: formatHumanOutput({
exitCode: SUCCESS,
reproduced: false,
originalFailure: failure,
warnings,
}, artifact),
warnings,
reproduced: false,
originalFailure: failure,
}
}
// Check if the same failure was reproduced
const reproducedFailure = runResult.failures.find(f =>
f.route === failure.route && f.contract === failure.contract
)
if (reproducedFailure) {
// Same failure reproduced
return {
exitCode: BEHAVIORAL_FAILURE,
message: formatHumanOutput({
exitCode: BEHAVIORAL_FAILURE,
reproduced: true,
originalFailure: failure,
warnings,
}, artifact),
warnings,
reproduced: true,
originalFailure: failure,
}
}
// Check if there are different failures
if (runResult.failures.length > 0) {
const newFailure = runResult.failures[0]
if (!newFailure) {
return {
exitCode: SUCCESS,
message: formatHumanOutput({
exitCode: SUCCESS,
reproduced: false,
originalFailure: failure,
warnings,
}, artifact),
warnings,
reproduced: false,
originalFailure: failure,
}
}
return {
exitCode: BEHAVIORAL_FAILURE,
message: formatHumanOutput({
exitCode: BEHAVIORAL_FAILURE,
reproduced: false,
originalFailure: failure,
newFailure: {
route: newFailure.route,
contract: newFailure.contract,
expected: newFailure.expected,
observed: newFailure.observed,
seed: artifact.seed || 42,
replayCommand: `apophis replay --artifact ${artifactPath}`,
},
warnings,
}, artifact),
warnings,
reproduced: false,
originalFailure: failure,
newFailure: {
route: newFailure.route,
contract: newFailure.contract,
expected: newFailure.expected,
observed: newFailure.observed,
seed: artifact.seed || 42,
replayCommand: `apophis replay --artifact ${artifactPath}`,
},
}
}
// No failures — the bug was fixed
if (!options?.sourceChanged) {
warnings.push('Replay diverged with same seed and no source drift detected. Likely runtime/data nondeterminism.')
}
return {
exitCode: SUCCESS,
message: formatHumanOutput({
exitCode: SUCCESS,
reproduced: false,
originalFailure: failure,
warnings,
}, artifact),
warnings,
reproduced: false,
originalFailure: failure,
}
}
// ---------------------------------------------------------------------------
// Main command handler
// ---------------------------------------------------------------------------
/**
* Main replay command handler.
*
* Flow:
* 1. Load and validate artifact
* 2. Check CLI version compatibility
* 3. Detect source code changes (warn but continue)
* 4. Load Fastify app and re-run verify with same seed
* 5. Compare results to original failure
* 6. Return appropriate exit code
*
* Exit codes:
* - 0: Replay passed (failure no longer reproduces)
* - 1: Same failure reproduced OR different failure found
* - 2: Error (missing artifact, corrupted, route no longer exists, etc.)
*/
export async function replayCommand(
options: ReplayOptions,
ctx: CliContext,
): Promise<CommandResult> {
const { artifact: artifactPath, config: configPath, cwd } = options
const workingDir = cwd || ctx.cwd
const resolvedArtifactPath = resolve(workingDir, artifactPath)
try {
// 1. Load and validate artifact
const loadResult = loadArtifact({
artifactPath,
cwd: workingDir,
routeFilter: options.route,
})
if (!loadResult.success) {
return {
exitCode: USAGE_ERROR,
message: loadResult.message,
warnings: loadResult.warnings,
}
}
const artifact = loadResult.artifact!
const failure = loadResult.failure!
const warnings = [...loadResult.warnings]
// 2. Execute replay
const replayResult = await executeReplay(artifact, failure, resolvedArtifactPath, ctx, {
sourceChanged: loadResult.sourceChanged,
})
// Merge warnings
if (replayResult.warnings) {
warnings.push(...replayResult.warnings)
}
return {
exitCode: replayResult.exitCode as import('../../core/types.js').ExitCode,
message: replayResult.message,
warnings: warnings.length > 0 ? warnings : undefined,
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error)
return {
exitCode: INTERNAL_ERROR,
message: `Internal error in replay command: ${message}`,
}
}
}
// ---------------------------------------------------------------------------
// CLI adapter
// ---------------------------------------------------------------------------
/**
* Adapter that bridges the CLI framework (cac) to the replay command handler.
* This function signature matches what the CLI core expects.
*/
export async function handleReplay(
args: string[],
ctx: CliContext,
): Promise<number> {
const options: ReplayOptions = {
artifact: '',
config: ctx.options.config || undefined,
cwd: ctx.cwd,
format: ctx.options.format as ReplayOptions['format'],
quiet: ctx.options.quiet,
verbose: ctx.options.verbose,
}
// Parse command-specific flags from args (passed by CLI dispatcher)
const artifactIdx = args.indexOf('--artifact')
if (artifactIdx !== -1 && args[artifactIdx + 1]) {
options.artifact = args[artifactIdx + 1]!
}
const routeIdx = args.indexOf('--route')
if (routeIdx !== -1 && args[routeIdx + 1]) {
options.route = args[routeIdx + 1]!
}
if (!options.artifact) {
const format = options.format || ctx.options.format || 'human'
if (format === 'json') {
console.log(renderJson({
exitCode: USAGE_ERROR,
error: 'Error: --artifact is required',
}))
} else if (format === 'ndjson') {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'replay',
exitCode: USAGE_ERROR,
error: 'Error: --artifact is required',
}) + '\n')
} else {
console.error('Error: --artifact is required')
}
return USAGE_ERROR
}
const result = await replayCommand(options, ctx)
// Output result based on format
if (!ctx.options.quiet && result.message) {
const format = options.format || ctx.options.format || 'human'
if (format === 'json') {
console.log(renderJson({
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}))
} else if (format === 'ndjson') {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'replay',
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}) + '\n')
} else {
console.log(result.message)
}
}
// Print warnings in human mode only
const format = options.format || ctx.options.format || 'human'
if (format === 'human' && result.warnings && result.warnings.length > 0 && !ctx.options.quiet) {
for (const warning of result.warnings) {
console.warn(`Warning: ${warning}`)
}
}
return result.exitCode
}
+424
View File
@@ -0,0 +1,424 @@
/**
* S7: Replay thread - Artifact loader and validation
*
* Responsibilities:
* - Load artifact from filesystem
* - Validate artifact schema version
* - Check CLI version compatibility
* - Detect source code changes since artifact
* - Provide degraded replay guidance
*
* Architecture:
* - Pure functions with dependency injection
* - No optional imports — everything is required or injected
* - Inline comments for documentation
*/
import { readFileSync, existsSync, statSync } from 'node:fs';
import { resolve, dirname } from 'node:path';
import type { Artifact, FailureRecord } from '../../core/types.js';
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
/** Supported artifact schema version */
const SUPPORTED_ARTIFACT_VERSION = 'apophis-artifact/1';
/** Current CLI version for compatibility checks */
const CLI_VERSION = '2.0.0';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
/**
* Result of loading and validating an artifact.
*/
export interface ArtifactLoadResult {
/** Whether the load was successful */
success: boolean;
/** The loaded artifact (if successful) */
artifact?: Artifact;
/** The failure record to replay (if successful and artifact has failures) */
failure?: FailureRecord;
/** Human-readable message about the result */
message: string;
/** Warnings about degraded replay conditions */
warnings: string[];
/** Whether the artifact is compatible with this CLI version */
compatible: boolean;
/** Whether source code has changed since the artifact was created */
sourceChanged: boolean;
}
/**
* Options for loading an artifact.
*/
export interface LoadArtifactOptions {
/** Absolute or relative path to the artifact file */
artifactPath: string;
/** Current working directory for resolving relative paths */
cwd: string;
/** CLI version to check compatibility against (injected) */
cliVersion?: string;
/** Optional route filter to select a specific failure */
routeFilter?: string;
}
// ---------------------------------------------------------------------------
// Artifact loading
// ---------------------------------------------------------------------------
/**
* Load an artifact file from disk.
* Returns the parsed artifact or throws with a clear message.
*/
export function loadArtifactFile(artifactPath: string, cwd: string): Artifact {
const resolvedPath = resolve(cwd, artifactPath);
if (!existsSync(resolvedPath)) {
throw new ArtifactLoadError(
`Artifact not found: ${resolvedPath}`,
'missing',
resolvedPath,
);
}
let content: string;
try {
content = readFileSync(resolvedPath, 'utf-8');
} catch (err) {
throw new ArtifactLoadError(
`Cannot read artifact at ${resolvedPath}: ${err instanceof Error ? err.message : String(err)}`,
'unreadable',
resolvedPath,
);
}
let parsed: unknown;
try {
parsed = JSON.parse(content);
} catch (err) {
throw new ArtifactLoadError(
`Artifact is corrupted (invalid JSON) at ${resolvedPath}: ${err instanceof Error ? err.message : String(err)}`,
'corrupted',
resolvedPath,
);
}
if (!parsed || typeof parsed !== 'object') {
throw new ArtifactLoadError(
`Artifact is corrupted (not an object) at ${resolvedPath}`,
'corrupted',
resolvedPath,
);
}
return parsed as Artifact;
}
// ---------------------------------------------------------------------------
// Schema validation
// ---------------------------------------------------------------------------
/**
* Validate that an artifact matches the expected schema.
* Checks version, required fields, and basic structure.
*/
export function validateArtifactSchema(artifact: unknown): { valid: boolean; errors: string[] } {
const errors: string[] = [];
if (!artifact || typeof artifact !== 'object') {
errors.push('Artifact must be an object');
return { valid: false, errors };
}
const obj = artifact as Record<string, unknown>;
// Check version
if (!obj.version || typeof obj.version !== 'string') {
errors.push('Missing or invalid "version" field');
} else if (obj.version !== SUPPORTED_ARTIFACT_VERSION) {
errors.push(
`Unsupported artifact version: "${obj.version}". ` +
`Expected: "${SUPPORTED_ARTIFACT_VERSION}"`,
);
}
// Check required fields
const requiredFields = ['command', 'cwd', 'startedAt', 'durationMs', 'summary'];
for (const field of requiredFields) {
if (!(field in obj)) {
errors.push(`Missing required field: "${field}"`);
}
}
// Check summary structure
if (obj.summary && typeof obj.summary === 'object') {
const summary = obj.summary as Record<string, unknown>;
const summaryFields = ['total', 'passed', 'failed'];
for (const field of summaryFields) {
if (typeof summary[field] !== 'number') {
errors.push(`Summary field "${field}" must be a number`);
}
}
}
// Check failures array
if (obj.failures !== undefined && !Array.isArray(obj.failures)) {
errors.push('Field "failures" must be an array');
}
return { valid: errors.length === 0, errors };
}
// ---------------------------------------------------------------------------
// CLI version compatibility
// ---------------------------------------------------------------------------
/**
* Check if the CLI version is compatible with the artifact.
* Artifacts from newer CLI versions may not be replayable.
*/
export function checkCliCompatibility(
artifact: Artifact,
cliVersion: string = CLI_VERSION,
): { compatible: boolean; message?: string } {
// For now, we only support exact version match
// In the future, this could support semver ranges
const artifactCliVersion = (artifact as unknown as Record<string, unknown>).cliVersion as string | undefined;
if (!artifactCliVersion) {
// No CLI version in artifact — assume compatible but warn
return {
compatible: true,
message: 'Artifact does not specify CLI version. Replay may behave differently.',
};
}
if (artifactCliVersion === cliVersion) {
return { compatible: true };
}
// Parse major versions
const artifactMajor = artifactCliVersion.split('.')[0];
const cliMajor = cliVersion.split('.')[0];
if (artifactMajor !== cliMajor) {
return {
compatible: false,
message:
`CLI version mismatch: artifact was created with v${artifactCliVersion}, ` +
`but current CLI is v${cliVersion}. Major version differences may prevent replay.`,
};
}
// Same major, different minor/patch — warn but allow
return {
compatible: true,
message:
`CLI version mismatch: artifact was created with v${artifactCliVersion}, ` +
`current CLI is v${cliVersion}. Replay should work but may differ slightly.`,
};
}
// ---------------------------------------------------------------------------
// Source code change detection
// ---------------------------------------------------------------------------
/**
* Detect if source code has changed since the artifact was created.
* Uses artifact mtime vs source file mtimes as a heuristic.
*/
export function detectSourceChanges(
artifact: Artifact,
artifactPath: string,
): { changed: boolean; details: string[] } {
const details: string[] = [];
try {
const artifactStat = statSync(artifactPath);
const artifactMtime = artifactStat.mtime;
// Check if cwd exists and get its stats
const cwd = artifact.cwd;
if (!existsSync(cwd)) {
return {
changed: true,
details: ['Artifact cwd no longer exists: ' + cwd],
};
}
// Try to find the app.js file in the cwd
const appPath = resolve(cwd, 'app.js');
if (existsSync(appPath)) {
const appStat = statSync(appPath);
if (appStat.mtime > artifactMtime) {
details.push('app.js has been modified since artifact was created');
}
}
// Check config file if referenced
if (artifact.configPath) {
const configPath = resolve(cwd, artifact.configPath);
if (existsSync(configPath)) {
const configStat = statSync(configPath);
if (configStat.mtime > artifactMtime) {
details.push('Config file has been modified since artifact was created');
}
}
}
} catch {
// If we can't stat files, assume no changes (fail open)
}
return {
changed: details.length > 0,
details,
};
}
// ---------------------------------------------------------------------------
// Route existence check
// ---------------------------------------------------------------------------
/**
* Check if the route from a failure record still exists in the current app.
* This is a heuristic — the actual check happens during replay execution.
*/
export function checkRouteExists(
failure: FailureRecord,
availableRoutes: string[],
): boolean {
return availableRoutes.includes(failure.route);
}
// ---------------------------------------------------------------------------
// Main loader
// ---------------------------------------------------------------------------
/**
* Load and validate an artifact for replay.
*
* Flow:
* 1. Load artifact file from disk
* 2. Validate schema
* 3. Check CLI version compatibility
* 4. Detect source code changes
* 5. Extract failure to replay
* 6. Return result with warnings
*/
export function loadArtifact(options: LoadArtifactOptions): ArtifactLoadResult {
const { artifactPath, cwd, cliVersion = CLI_VERSION, routeFilter } = options;
const warnings: string[] = [];
// 1. Load artifact file
let artifact: Artifact;
try {
artifact = loadArtifactFile(artifactPath, cwd);
} catch (err) {
if (err instanceof ArtifactLoadError) {
return {
success: false,
message: err.message,
warnings: [],
compatible: false,
sourceChanged: false,
};
}
throw err;
}
// 2. Validate schema
const validation = validateArtifactSchema(artifact);
if (!validation.valid) {
return {
success: false,
message: 'Artifact validation failed:\n' + validation.errors.map(e => ' ✗ ' + e).join('\n'),
warnings: [],
compatible: false,
sourceChanged: false,
};
}
// 3. Check CLI version compatibility
const compatibility = checkCliCompatibility(artifact, cliVersion);
if (!compatibility.compatible) {
return {
success: false,
message: compatibility.message!,
warnings: [],
compatible: false,
sourceChanged: false,
};
}
if (compatibility.message) {
warnings.push(compatibility.message);
}
// 4. Detect source code changes
const resolvedPath = resolve(cwd, artifactPath);
const sourceChanges = detectSourceChanges(artifact, resolvedPath);
if (sourceChanges.changed) {
warnings.push(...sourceChanges.details);
warnings.push('Source code has changed since artifact was created. Replay confidence is degraded and results may differ.');
warnings.push('Stabilize replay by checking out the same revision or rebuilding the fixture state used by the original run.');
}
// 5. Extract failure to replay
// If routeFilter is provided, find matching failure; otherwise use first failure
let failure: FailureRecord | undefined;
if (routeFilter) {
failure = artifact.failures.find(f => f.route === routeFilter);
if (!failure) {
return {
success: false,
message: `No failure found for route "${routeFilter}". Available routes: ${artifact.failures.map(f => f.route).join(', ')}`,
warnings,
compatible: compatibility.compatible,
sourceChanged: sourceChanges.changed,
};
}
} else {
failure = artifact.failures[0];
}
if (!failure) {
return {
success: false,
message: 'Artifact contains no failures to replay.',
warnings,
compatible: compatibility.compatible,
sourceChanged: sourceChanges.changed,
};
}
return {
success: true,
artifact,
failure,
message: `Loaded artifact: ${artifact.command} run with seed ${artifact.seed} (${artifact.summary.failed} failure(s))`,
warnings,
compatible: compatibility.compatible,
sourceChanged: sourceChanges.changed,
};
}
// ---------------------------------------------------------------------------
// Error type
// ---------------------------------------------------------------------------
/**
* Error type for artifact loading failures.
*/
export class ArtifactLoadError extends Error {
constructor(
message: string,
public readonly code: 'missing' | 'unreadable' | 'corrupted' | 'incompatible',
public readonly path: string,
) {
super(message);
this.name = 'ArtifactLoadError';
}
}
+803
View File
@@ -0,0 +1,803 @@
/**
* S4: Verify thread - Deterministic contract verification command
*
* Responsibilities:
* - Load config and resolve profile
* - Discover routes from Fastify app
* - Filter routes by --routes flag (supports wildcards/patterns)
* - Run deterministic contract verification
* - Generate seed if omitted, always print it
* - Produce canonical failure output matching golden snapshot
* - Emit artifact JSON
* - Print replay command
* - Support --changed for git-based filtering
* - Exit 0 on pass, 1 on behavioral failure, 2 on config error
*
* Architecture:
* - Dependency injection: all dependencies passed explicitly
* - No optional imports — everything is required or injected
* - Inline comments for documentation
*/
import type { CliContext } from '../../core/context.js'
import { loadConfig, findWorkspacePackages } from '../../core/config-loader.js'
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
import { resolveGenerationProfileOverride, GenerationProfileResolutionError } from '../../core/generation-profile.js'
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
import type { CommandResult, Artifact, FailureRecord, RouteResult, WorkspaceRun, WorkspaceResult } from '../../core/types.js'
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
import { runVerify, type VerifyRunResult } from './runner.js'
import { renderCanonicalFailure, renderHumanArtifact } from '../../renderers/human.js'
import { renderJson, renderJsonArtifact, renderJsonSummaryArtifact } from '../../renderers/json.js'
import { renderNdjsonArtifact, renderNdjsonSummaryArtifact } from '../../renderers/ndjson.js'
import type { OutputContext } from '../../renderers/shared.js'
import { resolve, basename } from 'node:path'
const ROUTE_IDENTITY_PATTERN = /^[A-Z]+\s+\/\S*$/
function normalizeRouteIdentity(route: string): string {
const normalized = route.trim().replace(/\s+/g, ' ')
const [method, ...pathParts] = normalized.split(' ')
if (!method || pathParts.length === 0) {
return normalized
}
return `${method.toUpperCase()} ${pathParts.join(' ')}`
}
function isReplayCompatibleRoute(route: string): boolean {
return ROUTE_IDENTITY_PATTERN.test(route)
}
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface VerifyOptions {
profile?: string
generationProfile?: string
routes?: string
seed?: number
changed?: boolean
config?: string
cwd?: string
format?: 'human' | 'json' | 'ndjson'
quiet?: boolean
verbose?: boolean
artifactDir?: string
}
// ---------------------------------------------------------------------------
// Seed generation
// ---------------------------------------------------------------------------
/**
* Generate a deterministic seed if none provided.
* Uses current time + process pid for uniqueness.
*/
export function generateSeed(): number {
return Date.now() + (process.pid || 0)
}
// ---------------------------------------------------------------------------
// Route filter parsing
// ---------------------------------------------------------------------------
/**
* Parse --routes flag into filter patterns.
* Supports comma-separated patterns with wildcards.
*/
function parseRouteFilters(routesFlag: string | undefined): string[] | undefined {
if (!routesFlag) return undefined
return routesFlag.split(',').map(r => r.trim()).filter(Boolean)
}
// ---------------------------------------------------------------------------
// Artifact builder
// ---------------------------------------------------------------------------
/**
* Build artifact document from verify results.
*/
function buildArtifact(
runResult: VerifyRunResult,
options: {
cwd: string
configPath?: string
profile?: string
preset?: string
env: string
seed: number
routeFilters?: string[]
},
): Artifact {
const warnings: string[] = []
const failures: FailureRecord[] = runResult.failures.map(f => {
const route = normalizeRouteIdentity(f.route)
if (!isReplayCompatibleRoute(route)) {
warnings.push(`Failure route "${f.route}" is not in METHOD /path format; replay matching may be less precise.`)
}
return {
route,
contract: f.contract,
expected: f.expected,
observed: f.observed,
seed: options.seed,
replayCommand: `apophis replay --artifact ${f.artifactPath || '<artifact-path-unavailable>'}`,
category: f.observed ? classifyError(f.observed) : ErrorTaxonomy.RUNTIME,
}
})
if (runResult.noContractsFound) {
warnings.push('No behavioral contracts found. Schema-only routes are not enough for verify. Add x-ensures or x-requires to route schemas. See docs/getting-started.md for examples.')
}
if (runResult.noRoutesMatched) {
warnings.push(`No routes matched the filter. Available routes: ${runResult.availableRoutes?.join(', ') || 'none'}`)
}
if (runResult.notGitRepo) {
warnings.push('--changed requires a git repository. Current directory is not inside a git repo.')
}
if (runResult.noRelevantChanges) {
warnings.push('No relevant changes detected. Git shows no modified files that match any route.')
}
if (runResult.failures.length > 0) {
const profileFlag = options.profile ? ` --profile ${options.profile}` : ''
const routesFlag = options.routeFilters && options.routeFilters.length > 0
? ` --routes "${options.routeFilters.join(',')}"`
: ''
warnings.push(`Deterministic rerun: apophis verify --seed ${options.seed}${profileFlag}${routesFlag}`)
warnings.push('If rerun output differs with same seed, stabilize app state/data and isolate time/external dependencies.')
}
return {
version: 'apophis-artifact/1',
cliVersion: '2.0.0',
command: 'verify',
mode: 'verify',
cwd: options.cwd,
configPath: options.configPath,
profile: options.profile,
preset: options.preset,
env: options.env,
seed: options.seed,
startedAt: new Date(Date.now() - runResult.durationMs).toISOString(),
durationMs: runResult.durationMs,
summary: {
total: runResult.total,
passed: runResult.passedCount,
failed: runResult.failed,
},
deterministicParams: {
seed: options.seed,
routeFilters: options.routeFilters ?? [],
},
failures,
artifacts: runResult.artifactPaths,
warnings,
exitReason: runResult.passed ? 'success' : 'behavioral_failure',
}
}
function attachReplayCommands(artifact: Artifact, artifactPath: string): void {
for (const failure of artifact.failures) {
failure.replayCommand = `apophis replay --artifact ${artifactPath}`
}
}
async function emitArtifact(
artifact: Artifact,
options: {
command: 'verify'
cwd: string
preferredDir?: string
force: boolean
},
): Promise<string | undefined> {
if (!options.force && !options.preferredDir) {
return undefined
}
const defaultDir = resolve(options.cwd, 'reports', 'apophis')
const candidateDirs = [options.preferredDir, defaultDir].filter(Boolean) as string[]
const attempted = new Set<string>()
for (const dir of candidateDirs) {
if (attempted.has(dir)) continue
attempted.add(dir)
try {
const { mkdirSync, writeFileSync } = await import('node:fs')
const artifactPath = resolve(dir, `${options.command}-${new Date().toISOString().replace(/[:.]/g, '-')}.json`)
mkdirSync(dir, { recursive: true })
attachReplayCommands(artifact, artifactPath)
writeFileSync(artifactPath, JSON.stringify(artifact, null, 2))
if (!artifact.artifacts.includes(artifactPath)) {
artifact.artifacts.push(artifactPath)
}
return artifactPath
} catch {
// Try fallback directory if available.
}
}
artifact.warnings.push('Failed to write artifact to disk')
return undefined
}
// ---------------------------------------------------------------------------
// Human output formatting
// ---------------------------------------------------------------------------
/**
* Format canonical failure output matching golden snapshot.
*/
function formatHumanFailure(failure: FailureRecord, profile?: string): string {
const lines: string[] = []
lines.push('Contract violation')
lines.push(failure.route)
lines.push(`Profile: ${profile || 'default'}`)
lines.push(`Seed: ${failure.seed}`)
lines.push('')
lines.push('Expected')
lines.push(` ${failure.contract}`)
lines.push('')
lines.push('Observed')
lines.push(` ${failure.observed}`)
lines.push('')
lines.push('Why this matters')
lines.push(` The resource created by ${failure.route.split(' ')[1]} is not retrievable.`)
lines.push('')
lines.push('Replay')
lines.push(` ${failure.replayCommand}`)
lines.push('')
lines.push('Next')
lines.push(` Check the create/read consistency for ${failure.route} and GET ${failure.route.split(' ')[1]}/{id}.`)
return lines.join('\n')
}
/**
* Format human-readable output for verify results.
*/
function formatHumanOutput(
runResult: VerifyRunResult,
options: { profile?: string; seed: number; env: string; routeFilters?: string[] },
): string {
const lines: string[] = []
if (runResult.notGitRepo) {
lines.push(`--changed requires a git repository.`)
lines.push(`Current directory is not inside a git repo.`)
lines.push('')
lines.push('Next:')
lines.push(` Initialize git with \`git init\`, or run verify without --changed.`)
lines.push('')
return lines.join('\n')
}
if (runResult.noRelevantChanges) {
lines.push(`No relevant changes detected.`)
lines.push(`Git shows no modified files that match any route.`)
lines.push('')
return lines.join('\n')
}
if (runResult.noRoutesMatched) {
lines.push(`No routes matched the filter.`)
lines.push(`Filters applied: ${options.routeFilters?.join(', ') || 'none'}`)
lines.push(`Available routes:`)
for (const r of runResult.availableRoutes || []) {
lines.push(` ${r}`)
}
lines.push('')
lines.push('Next:')
lines.push(` Adjust --routes filter or add routes to your app.`)
lines.push('')
return lines.join('\n')
}
if (runResult.noContractsFound) {
lines.push('No behavioral contracts found.')
lines.push('')
lines.push('APOPHIS discovered routes, but none have behavioral contracts.')
lines.push('Schema-only routes (with response schemas) are not enough.')
lines.push('You must add x-ensures or x-requires clauses that check behavior.')
lines.push('')
lines.push('Example — add this to your route schema:')
lines.push(' "x-ensures": [')
lines.push(' "response_code(GET /users/{response_body(this).id}) == 200"')
lines.push(' ]')
lines.push('')
lines.push('Next steps:')
lines.push(' 1. Open your route file (e.g., app.js or src/routes/users.js)')
lines.push(' 2. Find the route you want to test')
lines.push(' 3. Add an "x-ensures" array inside the schema object')
lines.push(' 4. Run: apophis verify --profile quick --routes "POST /users"')
lines.push('')
lines.push('For more examples, see docs/getting-started.md')
lines.push('')
return lines.join('\n')
}
// Print failures using canonical format
for (const failure of runResult.failures) {
const failureRecord: FailureRecord = {
route: failure.route,
contract: failure.contract,
expected: failure.expected,
observed: failure.observed,
seed: options.seed,
replayCommand: `apophis replay --artifact ${failure.artifactPath || 'reports/apophis/failure-*.json'}`,
}
lines.push(formatHumanFailure(failureRecord, options.profile))
lines.push('')
}
// Summary
if (runResult.passed) {
lines.push(`All ${runResult.total} contract(s) passed.`)
} else {
lines.push(`Failed: ${runResult.failed} of ${runResult.total} contract(s) failed.`)
}
lines.push(`Seed: ${options.seed}`)
// Replay command on failure
if (!runResult.passed && runResult.failures.length > 0) {
lines.push('')
lines.push('Replay')
lines.push(` apophis replay --artifact <path-to-artifact>`)
lines.push('')
lines.push('Determinism')
lines.push(` This run used seed ${options.seed}.`)
lines.push(` Same seed + same app state = same results.`)
lines.push(` If results differ on re-run, the app has nondeterministic behavior.`)
lines.push(` Stabilize: reset app state, mock external services, avoid time-dependent logic.`)
}
return lines.join('\n')
}
// ---------------------------------------------------------------------------
// Main command handler
// ---------------------------------------------------------------------------
/**
* Main verify command handler.
*
* Flow:
* 1. Load and resolve config
* 2. Run policy engine checks
* 3. Generate seed if omitted, always print it
* 4. Parse route filters
* 5. Load Fastify app and discover routes
* 6. Run deterministic contract verification
* 7. Build artifact
* 8. Format output
* 9. Write artifact if artifactDir specified
* 10. Return appropriate exit code
*/
export async function verifyCommand(
options: VerifyOptions,
ctx: CliContext,
): Promise<CommandResult> {
const {
profile,
generationProfile,
routes: routesFlag,
seed: explicitSeed,
changed,
config: configPath,
cwd,
artifactDir,
} = options
const workingDir = cwd || ctx.cwd
const format = options.format || ctx.options.format || 'human'
// Detect environment
const env = detectEnvironment()
try {
// 1. Load config
const loadResult = await loadConfig({
cwd: workingDir,
configPath,
profileName: profile,
env,
})
if (!loadResult.configPath) {
return {
exitCode: USAGE_ERROR,
message: 'No config found. Run "apophis init" to create one.',
}
}
const config = loadResult.config
const resolvedGenerationProfile = resolveGenerationProfileOverride(generationProfile, config)
// 2a. Resolve profile — if explicitly requested but missing, list available ones
if (profile && !config.profiles?.[profile]) {
const available = Object.keys(config.profiles ?? {}).join(', ') || 'none'
return {
exitCode: USAGE_ERROR,
message: `Unknown profile "${profile}". Available profiles: ${available}.\n\nNext:\n Run \`apophis init\` to scaffold a new profile, or use one of the profiles listed above.`,
}
}
// 2. Run policy engine checks
const policyEngine = new PolicyEngine({
config,
env,
mode: 'verify',
profileName: profile || undefined,
presetName: loadResult.presetName || undefined,
})
const policyResult = policyEngine.check()
if (!policyResult.allowed) {
const message = [
'Policy check failed:',
...policyResult.errors.map(e => `${e}`),
].join('\n')
return {
exitCode: USAGE_ERROR,
message,
}
}
// 3. Generate seed if omitted
const seed = explicitSeed ?? generateSeed()
if (!ctx.options.quiet && format === 'human') {
console.log(`Seed: ${seed}`)
}
// 4. Parse route filters
const routeFilters = parseRouteFilters(routesFlag)
// 5. Load the Fastify app
let fastify: unknown
try {
const { loadApp } = await import('../../core/app-loader.js')
const loaded = await loadApp(workingDir)
fastify = loaded.fastify
if (fastify && typeof (fastify as any).ready === 'function') {
await (fastify as any).ready()
}
} catch (err) {
const errorMessage = err instanceof Error ? err.message : String(err)
return {
exitCode: USAGE_ERROR,
message: `No Fastify app found. Ensure app.js exports a Fastify instance.\n\nError: ${errorMessage}\n\nNext:\n Run \`apophis init\` to scaffold a working app.js and config.`,
}
}
// 6. Run verify execution
const runResult = await runVerify({
fastify: fastify as any,
seed,
generationProfile: resolvedGenerationProfile,
timeout: typeof config.presets?.[loadResult.presetName || '']?.timeout === 'number'
? (config.presets[loadResult.presetName || ''] as { timeout?: number }).timeout
: undefined,
routeFilters,
changed,
profileRoutes: config.profiles?.[profile || '']?.routes,
})
// 7. Build artifact
const artifact = buildArtifact(runResult, {
cwd: workingDir,
configPath: loadResult.configPath,
profile: profile || undefined,
preset: loadResult.presetName || undefined,
env,
seed,
routeFilters,
})
// 8. Write artifact if configured or on failure
const shouldEmitArtifact = Boolean(artifactDir || config.artifactDir || !runResult.passed)
await emitArtifact(artifact, {
command: 'verify',
cwd: workingDir,
preferredDir: artifactDir || config.artifactDir,
force: shouldEmitArtifact,
})
// 9. Format output based on format option
const outputCtx: OutputContext = {
isTTY: ctx.isTTY,
isCI: ctx.isCI,
colorMode: ctx.options.color,
}
let message: string
if (format === 'json') {
message = renderJsonArtifact(artifact)
} else if (format === 'json-summary') {
message = renderJsonSummaryArtifact(artifact)
} else if (format === 'ndjson') {
// For ndjson, we don't return a message string; events are streamed
message = ''
} else if (format === 'ndjson-summary') {
// Concise ndjson: only summary events
message = ''
} else {
// human format
message = renderHumanArtifact(artifact, outputCtx)
}
// Determine exit code
let exitCode: number = SUCCESS
if (runResult.noRoutesMatched || runResult.noContractsFound || runResult.notGitRepo) {
exitCode = USAGE_ERROR
} else if (!runResult.passed) {
exitCode = BEHAVIORAL_FAILURE
}
return {
exitCode: exitCode as import('../../core/types.js').ExitCode,
artifact,
message,
warnings: artifact.warnings,
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error)
// Config validation errors are usage errors, not internal errors
if (error instanceof Error && error.name === 'ConfigValidationError') {
return {
exitCode: USAGE_ERROR,
message: `Config validation failed: ${message}`,
}
}
if (error instanceof GenerationProfileResolutionError) {
return {
exitCode: USAGE_ERROR,
message,
}
}
return {
exitCode: INTERNAL_ERROR,
message: `Internal error in verify command: ${message}`,
}
}
}
// ---------------------------------------------------------------------------
// CLI adapter
// ---------------------------------------------------------------------------
/**
* Adapter that bridges the CLI framework (cac) to the verify command handler.
* This function signature matches what the CLI core expects.
*/
export async function handleVerify(
args: string[],
ctx: CliContext,
): Promise<number> {
const options: VerifyOptions = {
profile: ctx.options.profile || undefined,
generationProfile: ctx.options.generationProfile,
routes: undefined,
seed: undefined,
changed: false,
config: ctx.options.config || undefined,
cwd: ctx.cwd,
format: ctx.options.format as VerifyOptions['format'],
quiet: ctx.options.quiet,
verbose: ctx.options.verbose,
artifactDir: ctx.options.artifactDir || undefined,
}
// Parse command-specific flags from args (passed by CLI dispatcher)
const routesIdx = args.indexOf('--routes')
if (routesIdx !== -1 && args[routesIdx + 1]) {
options.routes = args[routesIdx + 1]
}
const seedIdx = args.indexOf('--seed')
if (seedIdx !== -1 && args[seedIdx + 1]) {
const parsed = parseInt(args[seedIdx + 1]!, 10)
if (!isNaN(parsed)) {
options.seed = parsed
}
}
options.seed = options.seed as number | undefined
if (args.includes('--changed')) {
options.changed = true
}
const generationProfileIdx = args.indexOf('--generation-profile')
if (generationProfileIdx !== -1 && args[generationProfileIdx + 1]) {
options.generationProfile = args[generationProfileIdx + 1]
}
const workspaceMode = args.includes('--workspace')
if (workspaceMode) {
const packages = findWorkspacePackages(ctx.cwd)
if (packages.length === 0) {
if (!ctx.options.quiet) {
console.error('No workspace packages found. Ensure workspaces are defined in root package.json or pnpm-workspace.yaml.')
}
return USAGE_ERROR
}
const runs: WorkspaceRun[] = []
let overallExitCode = SUCCESS
const allWarnings: string[] = []
for (const pkgPath of packages) {
const pkgName = basename(pkgPath)
const pkgOptions = { ...options, cwd: pkgPath }
const pkgCtx: CliContext = { ...ctx, cwd: pkgPath }
const pkgResult = await verifyCommand(pkgOptions, pkgCtx)
if (pkgResult.artifact) {
pkgResult.artifact.package = pkgName
runs.push({ package: pkgName, cwd: pkgPath, artifact: pkgResult.artifact })
}
if (pkgResult.exitCode !== SUCCESS) {
overallExitCode = pkgResult.exitCode
}
if (pkgResult.warnings) {
allWarnings.push(...pkgResult.warnings.map(w => `[${pkgName}] ${w}`))
}
}
const workspaceResult: WorkspaceResult = {
exitCode: overallExitCode as import('../../core/types.js').ExitCode,
runs,
warnings: allWarnings,
}
const format = options.format || ctx.options.format || 'human'
if (!ctx.options.quiet) {
if (format === 'json') {
console.log(renderJson({
exitCode: workspaceResult.exitCode,
runs: workspaceResult.runs.map(r => ({
package: r.package,
cwd: r.cwd,
artifact: r.artifact,
})),
warnings: workspaceResult.warnings,
}))
} else if (format === 'json-summary') {
console.log(renderJson({
exitCode: workspaceResult.exitCode,
runs: workspaceResult.runs.map(r => ({
package: r.package,
cwd: r.cwd,
summary: r.artifact.summary,
exitReason: r.artifact.exitReason,
})),
warnings: workspaceResult.warnings,
}))
} else if (format === 'ndjson') {
for (const run of workspaceResult.runs) {
process.stdout.write(JSON.stringify({
type: 'workspace.run.completed',
package: run.package,
cwd: run.cwd,
summary: run.artifact.summary,
exitReason: run.artifact.exitReason,
}) + '\n')
}
process.stdout.write(JSON.stringify({
type: 'workspace.completed',
exitCode: workspaceResult.exitCode,
packages: workspaceResult.runs.length,
}) + '\n')
} else if (format === 'ndjson-summary') {
for (const run of workspaceResult.runs) {
process.stdout.write(JSON.stringify({
type: 'workspace.run.completed',
package: run.package,
cwd: run.cwd,
summary: run.artifact.summary,
exitReason: run.artifact.exitReason,
}) + '\n')
}
process.stdout.write(JSON.stringify({
type: 'workspace.completed',
exitCode: workspaceResult.exitCode,
packages: workspaceResult.runs.length,
}) + '\n')
} else {
// Human format
const lines: string[] = []
lines.push('Workspace verify results')
lines.push('')
for (const run of workspaceResult.runs) {
const a = run.artifact
const status = a.exitReason === 'success' ? '✓' : '✗'
lines.push(` ${status} ${run.package}: ${a.summary.passed}/${a.summary.total} passed`)
if (a.summary.failed > 0) {
lines.push(` ${a.summary.failed} failed`)
}
}
lines.push('')
lines.push(`Overall: ${workspaceResult.exitCode === SUCCESS ? 'passed' : 'failed'}`)
console.log(lines.join('\n'))
}
}
if (format !== 'json' && format !== 'ndjson' && format !== 'json-summary' && format !== 'ndjson-summary' && allWarnings.length > 0 && !ctx.options.quiet) {
for (const warning of allWarnings) {
console.warn(`Warning: ${warning}`)
}
}
return workspaceResult.exitCode
}
const result = await verifyCommand(options, ctx)
const format = options.format || ctx.options.format || 'human'
const machineMode = format === 'json' || format === 'ndjson' || format === 'json-summary' || format === 'ndjson-summary'
if (!ctx.options.quiet) {
if (format === 'json') {
if (result.artifact) {
console.log(renderJsonArtifact(result.artifact))
} else {
console.log(renderJson({
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}))
}
} else if (format === 'json-summary') {
if (result.artifact) {
console.log(renderJsonSummaryArtifact(result.artifact))
} else {
console.log(renderJson({
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}))
}
} else if (format === 'ndjson') {
if (result.artifact) {
renderNdjsonArtifact(result.artifact)
} else {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'verify',
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}) + '\n')
}
} else if (format === 'ndjson-summary') {
if (result.artifact) {
renderNdjsonSummaryArtifact(result.artifact)
} else {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'verify',
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}) + '\n')
}
} else if (result.message) {
console.log(result.message)
}
}
// Print warnings in human mode only
if (!machineMode && result.warnings && result.warnings.length > 0 && !ctx.options.quiet) {
for (const warning of result.warnings) {
console.warn(`Warning: ${warning}`)
}
}
return result.exitCode
}
+490
View File
@@ -0,0 +1,490 @@
/**
* S4: Verify thread - Runner for deterministic contract verification
*
* Responsibilities:
* - Route discovery from Fastify app
* - Route filtering by patterns and git changes
* - Contract execution using existing plugin/evaluator code
* - Deterministic execution with seed
* - Result aggregation
*
* Architecture:
* - Pure execution functions that accept injected dependencies
* - Reuses existing APOPHIS plugin and formula code
* - No reimplementation of parser/evaluator
*/
import { discoverRoutes } from '../../../domain/discovery.js'
import { extractContract } from '../../../domain/contract.js'
import { executeHttp } from '../../../infrastructure/http-executor.js'
import { parse } from '../../../formula/parser.js'
import { evaluateAsync } from '../../../formula/evaluator.js'
import { createOperationResolver } from '../../../formula/runtime.js'
import type { EvalContext, RouteContract, FastifyInjectInstance } from '../../../types.js'
import type { RouteResult } from '../../core/types.js'
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface VerifyFailure {
route: string
contract: string
expected: string
observed: string
artifactPath?: string
}
export interface VerifyRunResult {
passed: boolean
total: number
passedCount: number
failed: number
failures: VerifyFailure[]
durationMs: number
noRoutesMatched: boolean
noContractsFound: boolean
notGitRepo?: boolean
noRelevantChanges?: boolean
availableRoutes?: string[]
artifactPaths: string[]
}
export interface VerifyRunnerDeps {
fastify: FastifyInjectInstance
seed: number
generationProfile?: 'quick' | 'standard' | 'thorough'
timeout?: number
routeFilters?: string[]
changed?: boolean
profileRoutes?: string[]
}
// ---------------------------------------------------------------------------
// Route discovery
// ---------------------------------------------------------------------------
/**
* Discover routes from a Fastify instance.
* Uses the existing discovery module.
*/
export async function discoverAppRoutes(fastify: FastifyInjectInstance): Promise<RouteContract[]> {
return discoverRoutes(fastify)
}
/**
* Check if specific routes exist in a Fastify instance using hasRoute.
* Used when the APOPHIS plugin wasn't registered before routes.
*/
export async function discoverSpecificRoutes(
fastify: FastifyInjectInstance,
routePatterns: string[],
): Promise<RouteContract[]> {
if (typeof fastify.hasRoute !== 'function') {
return []
}
const routes: RouteContract[] = []
const seen = new Set<string>()
for (const pattern of routePatterns) {
// Parse pattern like "GET /users" or "POST /api/*"
const parts = pattern.split(' ')
const method = parts[0] || 'GET'
const path = parts.slice(1).join(' ')
// For exact routes (no wildcards), check if route exists
if (!pattern.includes('*') && !pattern.includes('?')) {
try {
if (fastify.hasRoute({ url: path, method })) {
const key = `${method} ${path}`
if (!seen.has(key)) {
seen.add(key)
routes.push({
method: method as RouteContract['method'],
path,
category: 'observer',
schema: {},
requires: [],
ensures: [],
invariants: [],
regexPatterns: {},
validateRuntime: false,
})
}
}
} catch {
// Route doesn't exist
}
}
}
return routes
}
// ---------------------------------------------------------------------------
// Route filtering
// ---------------------------------------------------------------------------
/**
* Check if a route matches a filter pattern.
* Supports wildcards: * matches any characters.
*/
function matchRoutePattern(route: string, pattern: string): boolean {
// Convert pattern to regex
const regexPattern = pattern
.replace(/\*/g, '.*')
.replace(/\?/g, '.')
const regex = new RegExp(`^${regexPattern}$`, 'i')
return regex.test(route)
}
/**
* Filter routes by patterns.
*/
function filterRoutesByPatterns(routes: RouteContract[], patterns: string[]): RouteContract[] {
return routes.filter(route => {
const routeStr = `${route.method} ${route.path}`
return patterns.some(pattern => matchRoutePattern(routeStr, pattern))
})
}
/**
* Check if cwd is inside a git repository.
*/
async function isGitRepo(cwd: string): Promise<boolean> {
try {
const { execSync } = await import('node:child_process')
execSync('git rev-parse --git-dir', { cwd, encoding: 'utf-8', stdio: 'pipe' })
return true
} catch {
return false
}
}
/**
* Get git-modified files for --changed filtering.
*/
async function getGitChangedFiles(cwd: string): Promise<string[]> {
try {
const { execSync } = await import('node:child_process')
const output = execSync('git diff --name-only HEAD', { cwd, encoding: 'utf-8' })
return output.split('\n').filter(Boolean)
} catch {
return []
}
}
/**
* Filter routes to only those modified in git.
*/
async function filterChangedRoutes(
routes: RouteContract[],
cwd: string,
): Promise<RouteContract[]> {
const changedFiles = await getGitChangedFiles(cwd)
// Map route paths to potential file paths (heuristic)
return routes.filter(route => {
const routePath = route.path
// Check if any changed file might contain this route
return changedFiles.some(file => {
// Simple heuristic: check if route path segments appear in file path
const segments = routePath.split('/').filter(Boolean)
return segments.some(segment => file.includes(segment))
})
})
}
// ---------------------------------------------------------------------------
// Contract execution
// ---------------------------------------------------------------------------
/**
* Build a request for a route.
*/
function buildRouteRequest(route: RouteContract): {
method: string
url: string
body?: unknown
headers: Record<string, string>
} {
const headers: Record<string, string> = {
'content-type': 'application/json',
}
// Build body from schema if available
let body: unknown = undefined
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
if (bodySchema && route.method === 'POST') {
body = buildExampleBody(bodySchema)
}
return {
method: route.method,
url: route.path,
body,
headers,
}
}
/**
* Build an example body from JSON Schema.
*/
function buildExampleBody(schema: Record<string, unknown>): unknown {
if (schema.type === 'object' && schema.properties) {
const obj: Record<string, unknown> = {}
const properties = schema.properties as Record<string, Record<string, unknown>>
for (const [key, propSchema] of Object.entries(properties)) {
obj[key] = buildExampleValue(propSchema)
}
return obj
}
return undefined
}
/**
* Build an example value from a property schema.
*/
function buildExampleValue(schema: Record<string, unknown>): unknown {
if (schema.type === 'string') {
if (schema.enum && Array.isArray(schema.enum) && schema.enum.length > 0) {
return schema.enum[0]
}
return 'test'
}
if (schema.type === 'number' || schema.type === 'integer') {
return 1
}
if (schema.type === 'boolean') {
return true
}
if (schema.type === 'array') {
return []
}
if (schema.type === 'object' && schema.properties) {
return buildExampleBody(schema)
}
return undefined
}
/**
* Execute a single contract for a route.
* Returns the evaluation context and any failure.
*/
async function executeContract(
fastify: FastifyInjectInstance,
route: RouteContract,
contract: string,
timeout?: number,
variant?: { name: string; headers?: Record<string, string> },
): Promise<{ ctx: EvalContext; failure?: VerifyFailure }> {
const request = buildRouteRequest(route)
// Merge variant headers if provided
const headers = variant?.headers
? { ...request.headers, ...variant.headers }
: request.headers
// Execute the primary request
const ctx = await executeHttp(fastify, route, {
method: request.method,
url: request.url,
body: request.body,
headers,
query: {},
}, undefined, timeout)
// Build eval context with operation resolver for cross-operation calls
const evalCtx: EvalContext = {
...ctx,
operationResolver: createOperationResolver(fastify, headers, ctx),
}
// Parse and evaluate the contract
try {
const parsed = parse(contract)
const result = await evaluateAsync(parsed.ast, evalCtx)
if (!result.success || !result.value) {
return {
ctx: evalCtx,
failure: {
route: variant && variant.name !== 'default'
? `[variant:${variant.name}] ${route.method} ${route.path}`
: `${route.method} ${route.path}`,
contract,
expected: 'true',
observed: result.success ? String(result.value) : result.error,
},
}
}
return { ctx: evalCtx }
} catch (error) {
return {
ctx: evalCtx,
failure: {
route: variant && variant.name !== 'default'
? `[variant:${variant.name}] ${route.method} ${route.path}`
: `${route.method} ${route.path}`,
contract,
expected: 'true',
observed: error instanceof Error ? error.message : String(error),
},
}
}
}
// ---------------------------------------------------------------------------
// Main verify runner
// ---------------------------------------------------------------------------
/**
* Run deterministic contract verification.
*
* Flow:
* 1. Discover routes from Fastify app
* 2. Apply route filters (patterns, changed, profile routes)
* 3. Check for behavioral contracts
* 4. Execute each contract deterministically
* 5. Aggregate results
*/
export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult> {
const started = Date.now()
const { fastify, routeFilters, changed, profileRoutes } = deps
// 1. Discover routes
let allRoutes = await discoverAppRoutes(fastify)
// If no routes discovered (plugin not registered before routes),
// try to discover specific routes from filters
if (allRoutes.length === 0 && (routeFilters?.length || profileRoutes?.length)) {
const patternsToCheck = [
...(routeFilters || []),
...(profileRoutes || []),
]
allRoutes = await discoverSpecificRoutes(fastify, patternsToCheck)
}
const availableRoutes = allRoutes.map(r => `${r.method} ${r.path}`)
// 2. Apply filters
let routes = allRoutes
// Apply profile routes filter first
if (profileRoutes && profileRoutes.length > 0) {
routes = filterRoutesByPatterns(routes, profileRoutes)
}
// Apply --routes flag filter
if (routeFilters && routeFilters.length > 0) {
routes = filterRoutesByPatterns(routes, routeFilters)
}
// Apply --changed filter
if (changed) {
const cwd = process.cwd()
const inGit = await isGitRepo(cwd)
if (!inGit) {
return {
passed: false,
total: 0,
passedCount: 0,
failed: 0,
failures: [],
durationMs: Date.now() - started,
noRoutesMatched: false,
noContractsFound: false,
availableRoutes,
artifactPaths: [],
notGitRepo: true,
}
}
routes = await filterChangedRoutes(routes, cwd)
}
// Check if any routes matched
if (routes.length === 0) {
return {
passed: false,
total: 0,
passedCount: 0,
failed: 0,
failures: [],
durationMs: Date.now() - started,
noRoutesMatched: true,
noContractsFound: false,
availableRoutes,
artifactPaths: [],
}
}
// 3. Check for behavioral contracts
const routesWithContracts = routes.filter(route =>
route.ensures.length > 0 || route.requires.length > 0
)
if (routesWithContracts.length === 0) {
return {
passed: false,
total: 0,
passedCount: 0,
failed: 0,
failures: [],
durationMs: Date.now() - started,
noRoutesMatched: false,
noContractsFound: true,
availableRoutes,
artifactPaths: [],
}
}
// 4. Execute contracts (with variant expansion)
const failures: VerifyFailure[] = []
let total = 0
let passedCount = 0
for (const route of routesWithContracts) {
const contracts = [...route.requires, ...route.ensures]
const variants = route.variants && route.variants.length > 0
? route.variants
: [{ name: 'default' }]
for (const variant of variants) {
for (const contract of contracts) {
total++
const result = await executeContract(fastify, route, contract, deps.timeout, variant)
if (result.failure) {
failures.push(result.failure)
} else {
passedCount++
}
}
}
}
const durationMs = Date.now() - started
// Sort failures deterministically by route then contract for stable output
const sortedFailures = failures.sort((a, b) => {
const routeCmp = a.route.localeCompare(b.route)
if (routeCmp !== 0) return routeCmp
return a.contract.localeCompare(b.contract)
})
return {
passed: failures.length === 0,
total,
passedCount,
failed: failures.length,
failures: sortedFailures,
durationMs,
noRoutesMatched: false,
noContractsFound: false,
availableRoutes,
artifactPaths: [],
}
}