chore: crush git history - reborn from consolidation on 2026-03-10

This commit is contained in:
John Dvorak
2026-03-10 00:00:00 -07:00
commit d278c4b105
313 changed files with 87549 additions and 0 deletions
@@ -0,0 +1,36 @@
/**
* APOPHIS configuration for broken-behavior fixture.
*/
export default {
mode: "verify",
profiles: {
quick: {
name: "quick",
mode: "verify",
preset: "safe-ci",
routes: ["POST /users"],
},
},
presets: {
"safe-ci": {
name: "safe-ci",
depth: "quick",
timeout: 5000,
parallel: false,
chaos: false,
observe: false,
},
},
environments: {
local: {
name: "local",
allowVerify: true,
allowObserve: true,
allowQualify: false,
allowChaos: false,
allowBlocking: true,
requireSink: false,
},
},
};
+100
View File
@@ -0,0 +1,100 @@
/**
* Broken behavior fixture: POST /users returns 201 but GET /users/{id} returns 404.
* This is the canonical "wow" failure for APOPHIS CLI acceptance tests.
*/
import Fastify from "fastify";
import apophisPlugin from "../../../index.js";
const app = Fastify({ logger: false });
// Register swagger (required by APOPHIS)
await app.register(import("@fastify/swagger"), {
openapi: {
info: { title: "Broken API", version: "1.0.0" },
},
});
// Register APOPHIS plugin for route discovery
await app.register(apophisPlugin, { runtime: "off" });
app.post(
"/users",
{
schema: {
description: "Create a user",
body: {
type: "object",
required: ["name"],
properties: {
name: { type: "string" },
},
},
response: {
201: {
type: "object",
properties: {
id: { type: "string" },
name: { type: "string" },
},
},
},
// Behavioral contract: created resource must be retrievable
"x-ensures": [
"response_code(GET /users/{response_body(this).id}) == 200",
],
},
},
async (request, reply) => {
const { name } = request.body;
const id = `usr-${Date.now()}`;
reply.status(201);
return { id, name };
}
);
app.get(
"/users/:id",
{
schema: {
description: "Get a user by ID",
params: {
type: "object",
required: ["id"],
properties: {
id: { type: "string" },
},
},
response: {
200: {
type: "object",
properties: {
id: { type: "string" },
name: { type: "string" },
},
},
404: {
type: "object",
properties: {
error: { type: "string" },
},
},
},
},
},
async (request, reply) => {
const { id } = request.params;
// BUG: Always returns 404, even for resources that were just created
reply.status(404);
return { error: `User ${id} not found` };
}
);
export default app;
// Start server if run directly
if (process.argv[1] === new URL(import.meta.url).pathname) {
await app.ready();
await app.listen({ port: 3000 });
console.log("Broken behavior app running on http://localhost:3000");
}
@@ -0,0 +1,13 @@
{
"name": "@apophis/fixture-broken-behavior",
"version": "1.0.0",
"type": "module",
"scripts": {
"start": "node app.js",
"test": "node --test"
},
"dependencies": {
"fastify": "^5.0.0",
"@fastify/swagger": "^9.0.0"
}
}
@@ -0,0 +1,40 @@
/**
* LEGACY APOPHIS configuration (old-style, for migration tests).
* This uses deprecated field names that should be detected by `apophis migrate`.
*/
export default {
// Deprecated: 'mode' used to be 'testMode'
testMode: "verify",
// Deprecated: 'profiles' used to be 'testProfiles'
testProfiles: {
quick: {
name: "quick",
// Deprecated: 'preset' used to be 'usesPreset'
usesPreset: "safe-ci",
// Deprecated: 'routes' used to be 'routeFilter'
routeFilter: ["GET /legacy"],
},
},
// Deprecated: 'presets' used to be 'testPresets'
testPresets: {
"safe-ci": {
name: "safe-ci",
// Deprecated: 'depth' used to be 'testDepth'
testDepth: "quick",
// Deprecated: 'timeout' used to be 'maxDuration'
maxDuration: 5000,
},
},
// Deprecated: 'environments' used to be 'envPolicies'
envPolicies: {
local: {
name: "local",
// Deprecated: 'allowVerify' used to be 'canVerify'
canVerify: true,
},
},
};
+25
View File
@@ -0,0 +1,25 @@
/**
* Legacy config fixture: old-style config for migration tests.
* Uses deprecated field names and structure.
*/
import Fastify from "fastify";
const app = Fastify({ logger: false });
await app.register(import("@fastify/swagger"), {
openapi: {
info: { title: "Legacy App", version: "1.0.0" },
},
});
app.get("/legacy", async () => ({ status: "legacy" }));
export default app;
// Start server if run directly
if (process.argv[1] === new URL(import.meta.url).pathname) {
await app.ready();
await app.listen({ port: 3000 });
console.log("Legacy config app running on http://localhost:3000");
}
@@ -0,0 +1,13 @@
{
"name": "@apophis/fixture-legacy-config",
"version": "1.0.0",
"type": "module",
"scripts": {
"start": "node app.js",
"test": "node --test"
},
"dependencies": {
"fastify": "^5.0.0",
"@fastify/swagger": "^9.0.0"
}
}
@@ -0,0 +1,30 @@
/**
* Root-level APOPHIS config for monorepo.
* Packages can override with their own configs.
*/
export default {
mode: "verify",
profiles: {
"api-quick": {
name: "api-quick",
mode: "verify",
preset: "safe-ci",
},
"web-quick": {
name: "web-quick",
mode: "verify",
preset: "safe-ci",
},
},
presets: {
"safe-ci": {
name: "safe-ci",
depth: "quick",
timeout: 5000,
parallel: false,
chaos: false,
observe: false,
},
},
};
@@ -0,0 +1,12 @@
{
"name": "@apophis/fixture-monorepo",
"version": "1.0.0",
"private": true,
"type": "module",
"workspaces": [
"packages/*"
],
"scripts": {
"test": "npm run test --workspaces"
}
}
@@ -0,0 +1,43 @@
/**
* API package in monorepo fixture.
*/
import Fastify from "fastify";
const app = Fastify({ logger: false });
await app.register(import("@fastify/swagger"), {
openapi: {
info: { title: "API Package", version: "1.0.0" },
},
});
app.get("/health", async () => ({ status: "ok" }));
app.post(
"/users",
{
schema: {
body: {
type: "object",
required: ["name"],
properties: { name: { type: "string" } },
},
"x-ensures": [
"response_code(GET /users/{response_body(this).id}) == 200",
],
},
},
async (request, reply) => {
const id = `usr-${Date.now()}`;
reply.status(201);
return { id, name: request.body.name };
}
);
app.get("/users/:id", async (request) => ({
id: request.params.id,
name: "Test User",
}));
export default app;
@@ -0,0 +1,12 @@
{
"name": "@apophis/fixture-monorepo-api",
"version": "1.0.0",
"type": "module",
"scripts": {
"start": "node app.js"
},
"dependencies": {
"fastify": "^5.0.0",
"@fastify/swagger": "^9.0.0"
}
}
@@ -0,0 +1,17 @@
/**
* Web package in monorepo fixture.
*/
import Fastify from "fastify";
const app = Fastify({ logger: false });
await app.register(import("@fastify/swagger"), {
openapi: {
info: { title: "Web Package", version: "1.0.0" },
},
});
app.get("/", async () => ({ message: "Hello from web" }));
export default app;
@@ -0,0 +1,12 @@
{
"name": "@apophis/fixture-monorepo-web",
"version": "1.0.0",
"type": "module",
"scripts": {
"start": "node app.js"
},
"dependencies": {
"fastify": "^5.0.0",
"@fastify/swagger": "^9.0.0"
}
}
@@ -0,0 +1,36 @@
/**
* APOPHIS configuration for observe-config fixture.
*/
export default {
mode: "observe",
profiles: {
"staging-observe": {
name: "staging-observe",
mode: "observe",
preset: "observe-safe",
routes: ["/health", "/events"],
},
},
presets: {
"observe-safe": {
name: "observe-safe",
depth: "quick",
timeout: 5000,
parallel: false,
chaos: false,
observe: true,
},
},
environments: {
staging: {
name: "staging",
allowVerify: true,
allowObserve: true,
allowQualify: false,
allowChaos: false,
allowBlocking: false,
requireSink: true,
},
},
};
@@ -0,0 +1,55 @@
/**
* Observe config fixture: app with observe configuration and sink setup.
*/
import Fastify from "fastify";
const app = Fastify({ logger: false });
await app.register(import("@fastify/swagger"), {
openapi: {
info: { title: "Observe App", version: "1.0.0" },
},
});
app.get("/health", async () => ({ status: "ok" }));
app.post(
"/events",
{
schema: {
description: "Record an event",
body: {
type: "object",
required: ["type", "payload"],
properties: {
type: { type: "string" },
payload: { type: "object" },
},
},
response: {
201: {
type: "object",
properties: {
id: { type: "string" },
received: { type: "boolean" },
},
},
},
},
},
async (request, reply) => {
const id = `evt-${Date.now()}`;
reply.status(201);
return { id, received: true };
}
);
export default app;
// Start server if run directly
if (process.argv[1] === new URL(import.meta.url).pathname) {
await app.ready();
await app.listen({ port: 3000 });
console.log("Observe config app running on http://localhost:3000");
}
@@ -0,0 +1,13 @@
{
"name": "@apophis/fixture-observe-config",
"version": "1.0.0",
"type": "module",
"scripts": {
"start": "node app.js",
"test": "node --test"
},
"dependencies": {
"fastify": "^5.0.0",
"@fastify/swagger": "^9.0.0"
}
}
@@ -0,0 +1,42 @@
/**
* Fastify app that attempts duplicate APOPHIS plugin registration.
* Doctor should detect the duplicate and warn, not fail hard.
*/
import Fastify from "fastify";
import apophisPlugin from "/home/johndvorak/Business/workspace/Apophis/dist/index.js";
const app = Fastify({ logger: false });
await app.register(import("@fastify/swagger"), {
openapi: {
info: { title: "Duplicate Plugin Test", version: "1.0.0" },
},
});
// First registration
await app.register(apophisPlugin, { runtime: "off" });
// Second registration (duplicate) - this should be handled gracefully
// In real Fastify this would throw "decorator already added"
// But doctor should detect pre-registration and skip its own attempt
app.get(
"/health",
{
schema: {
description: "Health check",
response: {
200: {
type: "object",
properties: {
status: { type: "string" },
},
},
},
},
},
async () => ({ status: "ok" })
);
export default app;
@@ -0,0 +1,9 @@
{
"name": "@apophis/fixture-plugin-duplicate",
"version": "1.0.0",
"type": "module",
"dependencies": {
"fastify": "^5.0.0",
"@fastify/swagger": "^9.0.0"
}
}
@@ -0,0 +1,36 @@
/**
* Fastify app without APOPHIS plugin registered.
* Doctor should detect plugin is missing and warn.
*/
import Fastify from "fastify";
const app = Fastify({ logger: false });
await app.register(import("@fastify/swagger"), {
openapi: {
info: { title: "No Plugin Test", version: "1.0.0" },
},
});
// NOTE: APOPHIS plugin is NOT registered here
app.get(
"/health",
{
schema: {
description: "Health check",
response: {
200: {
type: "object",
properties: {
status: { type: "string" },
},
},
},
},
},
async () => ({ status: "ok" })
);
export default app;
@@ -0,0 +1,9 @@
{
"name": "@apophis/fixture-plugin-not-registered",
"version": "1.0.0",
"type": "module",
"dependencies": {
"fastify": "^5.0.0",
"@fastify/swagger": "^9.0.0"
}
}
@@ -0,0 +1,33 @@
import Fastify from "fastify";
import apophisPlugin from "/home/johndvorak/Business/workspace/Apophis/dist/index.js";
const app = Fastify({ logger: false });
await app.register(import("@fastify/swagger"), {
openapi: {
info: { title: "Pre-registered Plugin Test", version: "1.0.0" },
},
});
// Plugin is already registered here - doctor should detect this
await app.register(apophisPlugin, { runtime: "off" });
app.get(
"/health",
{
schema: {
description: "Health check",
response: {
200: {
type: "object",
properties: {
status: { type: "string" },
},
},
},
},
},
async () => ({ status: "ok" })
);
export default app;
@@ -0,0 +1,9 @@
{
"name": "@apophis/fixture-plugin-pre-registered",
"version": "1.0.0",
"type": "module",
"dependencies": {
"fastify": "^5.0.0",
"@fastify/swagger": "^9.0.0"
}
}
@@ -0,0 +1,36 @@
/**
* APOPHIS configuration for protocol-lab fixture.
*/
export default {
mode: "qualify",
profiles: {
"oauth-nightly": {
name: "oauth-nightly",
mode: "qualify",
preset: "deep",
routes: ["POST /oauth/authorize", "POST /oauth/token", "GET /api/user"],
},
},
presets: {
deep: {
name: "deep",
depth: "deep",
timeout: 30000,
parallel: false,
chaos: true,
observe: false,
},
},
environments: {
local: {
name: "local",
allowVerify: true,
allowObserve: true,
allowQualify: true,
allowChaos: true,
allowBlocking: true,
requireSink: false,
},
},
};
+169
View File
@@ -0,0 +1,169 @@
/**
* Protocol lab fixture: OAuth-like multi-step flow app.
* Demonstrates stateful testing with multi-step protocols.
*/
import Fastify from "fastify";
const app = Fastify({ logger: false });
// In-memory token store (for demo only)
const tokens = new Map();
const authCodes = new Map();
await app.register(import("@fastify/swagger"), {
openapi: {
info: { title: "Protocol Lab", version: "1.0.0" },
},
});
// Step 1: Request authorization code
app.post(
"/oauth/authorize",
{
schema: {
description: "Request authorization code",
body: {
type: "object",
required: ["client_id", "redirect_uri"],
properties: {
client_id: { type: "string" },
redirect_uri: { type: "string" },
scope: { type: "string" },
},
},
response: {
200: {
type: "object",
properties: {
code: { type: "string" },
state: { type: "string" },
},
},
},
},
},
async (request, reply) => {
const { client_id, redirect_uri } = request.body;
const code = `auth-${Date.now()}`;
authCodes.set(code, { client_id, redirect_uri, used: false });
return { code, state: "xyz" };
}
);
// Step 2: Exchange code for token
app.post(
"/oauth/token",
{
schema: {
description: "Exchange authorization code for access token",
body: {
type: "object",
required: ["code", "client_id", "client_secret"],
properties: {
code: { type: "string" },
client_id: { type: "string" },
client_secret: { type: "string" },
redirect_uri: { type: "string" },
},
},
response: {
200: {
type: "object",
properties: {
access_token: { type: "string" },
token_type: { type: "string" },
expires_in: { type: "number" },
},
},
400: {
type: "object",
properties: {
error: { type: "string" },
},
},
},
},
},
async (request, reply) => {
const { code, client_id, client_secret } = request.body;
const auth = authCodes.get(code);
if (!auth || auth.used) {
reply.status(400);
return { error: "invalid_grant" };
}
if (auth.client_id !== client_id) {
reply.status(400);
return { error: "invalid_client" };
}
auth.used = true;
const token = `tok-${Date.now()}`;
tokens.set(token, { client_id, createdAt: Date.now() });
return {
access_token: token,
token_type: "Bearer",
expires_in: 3600,
};
}
);
// Step 3: Use token
app.get(
"/api/user",
{
schema: {
description: "Get current user with access token",
headers: {
type: "object",
required: ["authorization"],
properties: {
authorization: { type: "string" },
},
},
response: {
200: {
type: "object",
properties: {
id: { type: "string" },
client_id: { type: "string" },
},
},
401: {
type: "object",
properties: {
error: { type: "string" },
},
},
},
},
},
async (request, reply) => {
const auth = request.headers.authorization;
if (!auth || !auth.startsWith("Bearer ")) {
reply.status(401);
return { error: "invalid_token" };
}
const token = auth.slice(7);
const data = tokens.get(token);
if (!data) {
reply.status(401);
return { error: "invalid_token" };
}
return { id: `user-${token}`, client_id: data.client_id };
}
);
export default app;
// Start server if run directly
if (process.argv[1] === new URL(import.meta.url).pathname) {
await app.ready();
await app.listen({ port: 3000 });
console.log("Protocol lab app running on http://localhost:3000");
}
@@ -0,0 +1,13 @@
{
"name": "@apophis/fixture-protocol-lab",
"version": "1.0.0",
"type": "module",
"scripts": {
"start": "node app.js",
"test": "node --test"
},
"dependencies": {
"fastify": "^5.0.0",
"@fastify/swagger": "^9.0.0"
}
}
@@ -0,0 +1,36 @@
/**
* APOPHIS configuration for tiny-fastify fixture.
*/
export default {
mode: "verify",
profiles: {
quick: {
name: "quick",
mode: "verify",
preset: "safe-ci",
routes: ["POST /users"],
},
},
presets: {
"safe-ci": {
name: "safe-ci",
depth: "quick",
timeout: 5000,
parallel: false,
chaos: false,
observe: false,
},
},
environments: {
local: {
name: "local",
allowVerify: true,
allowObserve: true,
allowQualify: false,
allowChaos: false,
allowBlocking: true,
requireSink: false,
},
},
};
+99
View File
@@ -0,0 +1,99 @@
/**
* Minimal Fastify app with one route and one behavioral contract.
* This is the "hello world" fixture for APOPHIS CLI.
*/
import Fastify from "fastify";
const app = Fastify({ logger: false });
// Register swagger (required by APOPHIS)
await app.register(import("@fastify/swagger"), {
openapi: {
info: { title: "Tiny API", version: "1.0.0" },
},
});
let apophisPlugin;
try {
({ default: apophisPlugin } = await import("../../../index.js"));
} catch {
({ default: apophisPlugin } = await import("../../../../dist/index.js"));
}
await app.register(apophisPlugin, { runtime: "off" });
app.post(
"/users",
{
schema: {
description: "Create a user",
body: {
type: "object",
required: ["name"],
properties: {
name: { type: "string" },
},
},
response: {
201: {
type: "object",
properties: {
id: { type: "string" },
name: { type: "string" },
},
},
},
// Behavioral contract: created resource must be retrievable
"x-ensures": [
"response_code(GET /users/{response_body(this).id}) == 200",
],
},
},
async (request, reply) => {
const { name } = request.body;
const id = `usr-${Date.now()}`;
reply.status(201);
return { id, name };
}
);
app.get(
"/users/:id",
{
schema: {
description: "Get a user by ID",
params: {
type: "object",
required: ["id"],
properties: {
id: { type: "string" },
},
},
response: {
200: {
type: "object",
properties: {
id: { type: "string" },
name: { type: "string" },
},
},
},
},
},
async (request, reply) => {
const { id } = request.params;
// In a real app, this would fetch from DB
// For this fixture, we always return the user
return { id, name: "Test User" };
}
);
export default app;
// Start server if run directly
if (process.argv[1] === new URL(import.meta.url).pathname) {
await app.ready();
await app.listen({ port: 3000 });
console.log("Tiny Fastify app running on http://localhost:3000");
}
@@ -0,0 +1,13 @@
{
"name": "@apophis/fixture-tiny-fastify",
"version": "1.0.0",
"type": "module",
"scripts": {
"start": "node app.js",
"test": "node --test"
},
"dependencies": {
"fastify": "^5.0.0",
"@fastify/swagger": "^9.0.0"
}
}
@@ -0,0 +1,21 @@
export default {
mode: "verify",
profiles: {
quick: {
name: "quick",
mode: "verify",
preset: "safe-ci",
routes: ["GET /health"],
},
},
presets: {
"safe-ci": {
name: "safe-ci",
depth: "quick",
timeout: 5000,
parallel: false,
chaos: false,
observe: false,
},
},
};
@@ -0,0 +1,32 @@
import Fastify from "fastify";
import apophisPlugin from "../../../index.js";
const app = Fastify({ logger: false });
await app.register(import("@fastify/swagger"), {
openapi: {
info: { title: "Verify No Contracts", version: "1.0.0" },
},
});
await app.register(apophisPlugin, { runtime: "off" });
app.get(
"/health",
{
schema: {
description: "Health check route with schema only",
response: {
200: {
type: "object",
properties: {
status: { type: "string" },
},
},
},
},
},
async () => ({ status: "ok" }),
);
export default app;
@@ -0,0 +1,13 @@
{
"name": "@apophis/fixture-verify-no-contracts",
"version": "1.0.0",
"type": "module",
"scripts": {
"start": "node app.js",
"test": "node --test"
},
"dependencies": {
"fastify": "^5.0.0",
"@fastify/swagger": "^9.0.0"
}
}
@@ -0,0 +1,21 @@
export default {
mode: "verify",
profiles: {
quick: {
name: "quick",
mode: "verify",
preset: "safe-ci",
routes: ["GET /broken"],
},
},
presets: {
"safe-ci": {
name: "safe-ci",
depth: "quick",
timeout: 5000,
parallel: false,
chaos: false,
observe: false,
},
},
};
@@ -0,0 +1,33 @@
import Fastify from "fastify";
import apophisPlugin from "../../../index.js";
const app = Fastify({ logger: false });
await app.register(import("@fastify/swagger"), {
openapi: {
info: { title: "Verify Parse Fail", version: "1.0.0" },
},
});
await app.register(apophisPlugin, { runtime: "off" });
app.get(
"/broken",
{
schema: {
description: "Route with invalid behavioral contract",
"x-ensures": ["this is not a valid contract!!!"],
response: {
200: {
type: "object",
properties: {
status: { type: "string" },
},
},
},
},
},
async () => ({ status: "ok" }),
);
export default app;
@@ -0,0 +1,13 @@
{
"name": "@apophis/fixture-verify-parse-fail",
"version": "1.0.0",
"type": "module",
"scripts": {
"start": "node app.js",
"test": "node --test"
},
"dependencies": {
"fastify": "^5.0.0",
"@fastify/swagger": "^9.0.0"
}
}
@@ -0,0 +1,21 @@
export default {
mode: "verify",
profiles: {
quick: {
name: "quick",
mode: "verify",
preset: "safe-ci",
routes: ["GET /slow"],
},
},
presets: {
"safe-ci": {
name: "safe-ci",
depth: "quick",
timeout: 5000,
parallel: false,
chaos: false,
observe: false,
},
},
};
@@ -0,0 +1,37 @@
import Fastify from "fastify";
import apophisPlugin from "../../../index.js";
const app = Fastify({ logger: false });
await app.register(import("@fastify/swagger"), {
openapi: {
info: { title: "Verify Timeout Route", version: "1.0.0" },
},
});
await app.register(apophisPlugin, { runtime: "off" });
app.get(
"/slow",
{
schema: {
description: "Slow route with timeout metadata",
"x-timeout": 1,
"x-ensures": ["response_code(this) == 200"],
response: {
200: {
type: "object",
properties: {
ok: { type: "boolean" },
},
},
},
},
},
async () => {
await new Promise((resolvePromise) => setTimeout(resolvePromise, 100));
return { ok: true };
},
);
export default app;
@@ -0,0 +1,13 @@
{
"name": "@apophis/fixture-verify-timeout-route",
"version": "1.0.0",
"type": "module",
"scripts": {
"start": "node app.js",
"test": "node --test"
},
"dependencies": {
"fastify": "^5.0.0",
"@fastify/swagger": "^9.0.0"
}
}
+17
View File
@@ -0,0 +1,17 @@
Usage: apophis doctor [options]
Validate config, environment safety, docs/example correctness.
Options:
-h, --help Display this help message
--config <path> Path to config file
--cwd <path> Working directory
--format <human|json|ndjson> Output format
--color <auto|always|never> Color mode
--quiet Suppress non-essential output
--verbose Verbose output
--artifact-dir <path> Directory for artifacts
Examples:
apophis doctor
apophis doctor --verbose
+24
View File
@@ -0,0 +1,24 @@
Usage: apophis [options] [command]
Options:
-v, --version Display version number
-h, --help Display this help message
--config <path> Path to config file
--profile <name> Profile name from config
--cwd <path> Working directory
--format <human|json|ndjson> Output format (default: human)
--color <auto|always|never> Color mode (default: auto)
--quiet Suppress non-essential output
--verbose Verbose output
--artifact-dir <path> Directory for artifacts
Commands:
init Scaffold config, scripts, and example usage
verify Run deterministic contract verification
observe Validate runtime observe configuration and reporting setup
qualify Run scenario, stateful, protocol, or chaos-driven qualification
replay Replay a failure using seed and stored trace
doctor Validate config, environment safety, docs/example correctness
migrate Check and rewrite deprecated config or API usage
For more help on a command, run: apophis <command> --help
+21
View File
@@ -0,0 +1,21 @@
Usage: apophis migrate [options]
Check and rewrite deprecated config or API usage.
Options:
-h, --help Display this help message
--check Detect legacy config without rewriting
--dry-run Show exact rewrites without writing
--write Perform rewrites
--config <path> Path to config file
--cwd <path> Working directory
--format <human|json|ndjson> Output format
--color <auto|always|never> Color mode
--quiet Suppress non-essential output
--verbose Verbose output
--artifact-dir <path> Directory for artifacts
Examples:
apophis migrate --check
apophis migrate --dry-run
apophis migrate --write
+19
View File
@@ -0,0 +1,19 @@
Usage: apophis observe [options]
Validate runtime observe configuration and reporting setup.
Options:
-h, --help Display this help message
--profile <name> Profile name from config
--check-config Only validate config, do not activate
--config <path> Path to config file
--cwd <path> Working directory
--format <human|json|ndjson> Output format
--color <auto|always|never> Color mode
--quiet Suppress non-essential output
--verbose Verbose output
--artifact-dir <path> Directory for artifacts
Examples:
apophis observe --profile staging-observe
apophis observe --check-config
+21
View File
@@ -0,0 +1,21 @@
Usage: apophis qualify [options]
Run scenario, stateful, protocol, or chaos-driven qualification.
Options:
-h, --help Display this help message
--profile <name> Profile name from config
--seed <number> Deterministic seed for reproducible runs
--scenario <name> Scenario name to run
--chaos Enable chaos mode
--config <path> Path to config file
--cwd <path> Working directory
--format <human|json|ndjson> Output format
--color <auto|always|never> Color mode
--quiet Suppress non-essential output
--verbose Verbose output
--artifact-dir <path> Directory for artifacts
Examples:
apophis qualify --profile oauth-nightly --seed 42
apophis qualify --profile chaos-nightly --chaos
+17
View File
@@ -0,0 +1,17 @@
Usage: apophis replay [options]
Replay a failure using seed and stored trace.
Options:
-h, --help Display this help message
--artifact <path> Path to artifact file (required)
--config <path> Path to config file
--cwd <path> Working directory
--format <human|json|ndjson> Output format
--color <auto|always|never> Color mode
--quiet Suppress non-essential output
--verbose Verbose output
--artifact-dir <path> Directory for artifacts
Examples:
apophis replay --artifact reports/apophis/failure-2026-04-28T12-30-22Z.json
+19
View File
@@ -0,0 +1,19 @@
Contract violation
POST /users
Profile: quick
Seed: 42
Expected
response_code(GET /users/{response_body(this).id}) == 200
Observed
GET /users/usr-123 returned 404
Why this matters
The resource created by POST /users is not retrievable.
Replay
apophis replay --artifact reports/apophis/failure-2026-04-28T12-30-22Z.json
Next
Check the create/read consistency for POST /users and GET /users/{id}.
+22
View File
@@ -0,0 +1,22 @@
Usage: apophis verify [options]
Run deterministic contract verification against your Fastify routes.
Options:
-h, --help Display this help message
--profile <name> Profile name from config
--routes <filter> Comma-separated route filters (e.g., "POST /users,GET /users/*")
--seed <number> Deterministic seed for reproducible runs
--changed Filter to routes modified in git
--config <path> Path to config file
--cwd <path> Working directory
--format <human|json|ndjson> Output format
--color <auto|always|never> Color mode
--quiet Suppress non-essential output
--verbose Verbose output
--artifact-dir <path> Directory for artifacts
Examples:
apophis verify --profile quick
apophis verify --profile quick --routes "POST /users"
apophis verify --changed
+367
View File
@@ -0,0 +1,367 @@
/**
* S8: Doctor thread - Config validation checks
*
* Checks:
* - Config file exists and is loadable
* - Unknown keys rejection with exact path
* - Legacy config detection (deprecated field names)
* - Mixed legacy/new config style detection
*/
import {
loadConfig,
loadConfigFile,
discoverConfig,
ConfigValidationError,
type Config,
type LoadConfigResult,
} from '../../../core/config-loader.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface ConfigCheckResult {
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
remediation?: string;
mode: 'all' | 'verify' | 'observe' | 'qualify';
}
export interface ConfigCheckOptions {
cwd: string;
configPath?: string;
}
// ---------------------------------------------------------------------------
// Legacy field detection
// ---------------------------------------------------------------------------
/**
* Map of deprecated field names to their modern equivalents.
*/
const LEGACY_FIELDS: Record<string, string> = {
testMode: 'mode',
testProfiles: 'profiles',
testPresets: 'presets',
envPolicies: 'environments',
usesPreset: 'preset',
routeFilter: 'routes',
testDepth: 'depth',
maxDuration: 'timeout',
canVerify: 'allowVerify',
};
/**
* Recursively scan an object for legacy field names.
* Returns array of { path, legacyKey, modernKey } tuples.
*/
function findLegacyFields(
value: unknown,
path: string = '',
): Array<{ path: string; legacyKey: string; modernKey: string }> {
const results: Array<{ path: string; legacyKey: string; modernKey: string }> = [];
if (value === null || typeof value !== 'object') {
return results;
}
const obj = value as Record<string, unknown>;
for (const key of Object.keys(obj)) {
const currentPath = path ? `${path}.${key}` : key;
// Check if this key is legacy
if (LEGACY_FIELDS[key]) {
results.push({
path: currentPath,
legacyKey: key,
modernKey: LEGACY_FIELDS[key],
});
}
// Recurse into nested objects
const fieldValue = obj[key];
if (fieldValue !== null && typeof fieldValue === 'object' && !Array.isArray(fieldValue)) {
results.push(...findLegacyFields(fieldValue, currentPath));
}
}
return results;
}
/**
* Check if config contains legacy field names.
*/
export function checkLegacyConfig(config: Config | null): ConfigCheckResult {
if (!config) {
return {
name: 'legacy-config',
status: 'pass',
message: 'No config to check for legacy fields.',
mode: 'all',
};
}
const legacyFields = findLegacyFields(config);
if (legacyFields.length > 0) {
const details = legacyFields
.map(f => ` ${f.path}: "${f.legacyKey}" → "${f.modernKey}"`)
.join('\n');
return {
name: 'legacy-config',
status: 'warn',
message: `Found ${legacyFields.length} legacy field(s) in config.`,
detail: `Run "apophis migrate" to update these fields:\n${details}`,
remediation: 'Run "apophis migrate --dry-run" to preview rewrites.',
mode: 'all',
};
}
return {
name: 'legacy-config',
status: 'pass',
message: 'No legacy config fields detected.',
mode: 'all',
};
}
/**
* Check for mixed legacy and new config styles.
* This happens when some fields use old names and others use new names.
*/
export function checkMixedConfig(config: Config | null): ConfigCheckResult {
if (!config) {
return {
name: 'mixed-config',
status: 'pass',
message: 'No config to check for mixed styles.',
mode: 'all',
};
}
const legacyFields = findLegacyFields(config);
const hasLegacy = legacyFields.length > 0;
// Check if config also has modern fields at the same level as legacy ones
const hasModern = Object.keys(config).some(key => !LEGACY_FIELDS[key] && key !== 'name');
if (hasLegacy && hasModern) {
const legacyTopLevel = Object.keys(config).filter(key => LEGACY_FIELDS[key]);
const modernTopLevel = Object.keys(config).filter(key => !LEGACY_FIELDS[key] && key !== 'name');
// Only fail if there are actual modern fields that conflict with legacy ones
// A config with only legacy fields should warn, not fail
const hasConflictingModern = modernTopLevel.length > 0 &&
legacyTopLevel.some(lf => LEGACY_FIELDS[lf] !== undefined && modernTopLevel.includes(LEGACY_FIELDS[lf]));
if (hasConflictingModern) {
return {
name: 'mixed-config',
status: 'fail',
message: 'Config uses both legacy and modern field names.',
detail:
`Legacy fields: ${legacyTopLevel.join(', ')}\n` +
`Modern fields: ${modernTopLevel.join(', ')}\n` +
`Run "apophis migrate" to unify your config to the modern schema.`,
remediation: 'Run "apophis migrate --write" to unify config to modern schema.',
mode: 'all',
};
}
// Has both legacy and other modern fields - still warn but don't fail
return {
name: 'mixed-config',
status: 'warn',
message: 'Config contains legacy field names alongside modern fields.',
detail:
`Legacy fields: ${legacyTopLevel.join(', ')}\n` +
`Run "apophis migrate" to update to the modern schema.`,
remediation: 'Run "apophis migrate --dry-run" to preview rewrites.',
mode: 'all',
};
}
if (hasLegacy) {
return {
name: 'mixed-config',
status: 'warn',
message: 'Config uses legacy field names only.',
detail: 'Run "apophis migrate" to update to the modern schema.',
remediation: 'Run "apophis migrate --write" to update to modern schema.',
mode: 'all',
};
}
return {
name: 'mixed-config',
status: 'pass',
message: 'Config uses consistent modern field names.',
mode: 'all',
};
}
// ---------------------------------------------------------------------------
// Unknown key check
// ---------------------------------------------------------------------------
/**
* Check config for unknown keys by loading with strict validation.
*/
export async function checkUnknownKeys(options: ConfigCheckOptions): Promise<ConfigCheckResult> {
const { cwd, configPath } = options;
try {
const loadResult = await loadConfig({
cwd,
configPath,
});
if (!loadResult.configPath) {
return {
name: 'unknown-keys',
status: 'warn',
message: 'No config file found. Skipping unknown key check.',
detail: 'Run "apophis init" to create a config file.',
remediation: 'Run "apophis init --preset safe-ci" to scaffold a config.',
mode: 'all',
};
}
return {
name: 'unknown-keys',
status: 'pass',
message: 'Config keys are valid.',
mode: 'all',
};
} catch (error) {
if (error instanceof ConfigValidationError) {
return {
name: 'unknown-keys',
status: 'fail',
message: `Unknown config key at ${error.path}`,
detail: `Key "${error.key}" is not recognized by the APOPHIS config schema.`,
remediation: `Remove "${error.key}" from your config or check the docs for valid keys.`,
mode: 'all',
};
}
const message = error instanceof Error ? error.message : String(error);
return {
name: 'unknown-keys',
status: 'fail',
message: `Config validation failed: ${message}`,
remediation: 'Check your config file syntax and ensure it exports a valid object.',
mode: 'all',
};
}
}
// ---------------------------------------------------------------------------
// Config load check
// ---------------------------------------------------------------------------
/**
* Check if config can be loaded successfully.
*/
export async function checkConfigLoad(options: ConfigCheckOptions): Promise<ConfigCheckResult> {
const { cwd, configPath } = options;
try {
const loadResult = await loadConfig({
cwd,
configPath,
});
if (!loadResult.configPath) {
return {
name: 'config-load',
status: 'warn',
message: 'No config file found.',
detail: 'APOPHIS will use defaults. Run "apophis init" to create a config.',
remediation: 'Run "apophis init --preset safe-ci" to scaffold a config.',
mode: 'all',
};
}
return {
name: 'config-load',
status: 'pass',
message: `Config loaded from ${loadResult.configPath}`,
mode: 'all',
};
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return {
name: 'config-load',
status: 'fail',
message: `Failed to load config: ${message}`,
remediation: 'Check your config file syntax and ensure it exports a valid object.',
mode: 'all',
};
}
}
// ---------------------------------------------------------------------------
// Raw config loader (without validation)
// ---------------------------------------------------------------------------
/**
* Load raw config without schema validation.
* Used for legacy detection when validation would fail on legacy keys.
*/
async function loadRawConfig(options: ConfigCheckOptions): Promise<Config | null> {
const { cwd, configPath } = options;
// Discover config file
const discoveredPath = configPath || discoverConfig(cwd);
if (!discoveredPath) {
return null;
}
return await loadConfigFile(discoveredPath);
}
// ---------------------------------------------------------------------------
// Main config check runner
// ---------------------------------------------------------------------------
/**
* Run all config checks.
*/
export async function runConfigChecks(options: ConfigCheckOptions): Promise<ConfigCheckResult[]> {
const results: ConfigCheckResult[] = [];
// 1. Check config can be loaded
results.push(await checkConfigLoad(options));
// 2. Check for unknown keys
results.push(await checkUnknownKeys(options));
// 3. Check for legacy fields - load raw config without validation
try {
const rawConfig = await loadRawConfig(options);
results.push(checkLegacyConfig(rawConfig));
results.push(checkMixedConfig(rawConfig));
} catch {
// If config can't be loaded, skip legacy/mixed checks
results.push({
name: 'legacy-config',
status: 'warn',
message: 'Could not check for legacy fields (config failed to load).',
mode: 'all',
});
results.push({
name: 'mixed-config',
status: 'warn',
message: 'Could not check for mixed config (config failed to load).',
mode: 'all',
});
}
return results;
}
@@ -0,0 +1,242 @@
/**
* S8: Doctor thread - Dependency checks
*
* Checks:
* - Node.js version compatibility
* - Fastify installation and version
* - @fastify/swagger installation and version
* - Peer dependency completeness
*/
import { readFileSync, existsSync } from 'node:fs';
import { resolve } from 'node:path';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface DependencyCheckResult {
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
remediation?: string;
mode: 'all' | 'verify' | 'observe' | 'qualify';
}
export interface DependencyCheckOptions {
cwd: string;
nodeVersion: string;
}
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
const MIN_NODE_VERSION = 18;
const REQUIRED_PEER_DEPS = ['fastify', '@fastify/swagger'];
// ---------------------------------------------------------------------------
// Node.js version check
// ---------------------------------------------------------------------------
/**
* Parse major version from Node.js version string.
*/
function parseNodeMajor(version: string): number {
const match = version.match(/v?(\d+)/);
return match && match[1] ? parseInt(match[1], 10) : 0;
}
/**
* Check if Node.js version meets minimum requirement.
*/
export function checkNodeVersion(nodeVersion: string): DependencyCheckResult {
const major = parseNodeMajor(nodeVersion);
if (major < MIN_NODE_VERSION) {
return {
name: 'node-version',
status: 'fail',
message: `Node.js ${nodeVersion} is not supported. Minimum required: ${MIN_NODE_VERSION}.x`,
detail: `APOPHIS requires Node.js ${MIN_NODE_VERSION} or higher for ESM and modern features.`,
remediation: `Upgrade Node.js to ${MIN_NODE_VERSION}.x or higher (use nvm, fnm, or your package manager).`,
mode: 'all',
};
}
return {
name: 'node-version',
status: 'pass',
message: `Node.js ${nodeVersion} meets minimum requirement (${MIN_NODE_VERSION}+)`,
mode: 'all',
};
}
// ---------------------------------------------------------------------------
// Package.json dependency checks
// ---------------------------------------------------------------------------
/**
* Load and parse package.json from cwd.
*/
function loadPackageJson(cwd: string): Record<string, unknown> | null {
const pkgPath = resolve(cwd, 'package.json');
if (!existsSync(pkgPath)) {
return null;
}
try {
return JSON.parse(readFileSync(pkgPath, 'utf-8'));
} catch {
return null;
}
}
/**
* Check if a dependency is installed (declared in package.json).
*/
function hasDependency(pkg: Record<string, unknown>, name: string): boolean {
const deps = pkg.dependencies as Record<string, string> | undefined;
const devDeps = pkg.devDependencies as Record<string, string> | undefined;
return !!(deps?.[name] || devDeps?.[name]);
}
/**
* Get installed version range for a dependency.
*/
function getDependencyVersion(pkg: Record<string, unknown>, name: string): string | undefined {
const deps = pkg.dependencies as Record<string, string> | undefined;
const devDeps = pkg.devDependencies as Record<string, string> | undefined;
return deps?.[name] || devDeps?.[name];
}
/**
* Check Fastify installation and version.
*/
export function checkFastify(pkg: Record<string, unknown> | null): DependencyCheckResult {
if (!pkg) {
return {
name: 'fastify',
status: 'fail',
message: 'No package.json found. Cannot check Fastify installation.',
detail: 'Ensure you are running from a project root with a package.json file.',
remediation: 'Run npm init -y in your project root, then install dependencies.',
mode: 'all',
};
}
if (!hasDependency(pkg, 'fastify')) {
return {
name: 'fastify',
status: 'fail',
message: 'Fastify is not installed.',
detail: 'Install it with: npm install fastify@^5.0.0',
remediation: 'npm install fastify@^5.0.0',
mode: 'all',
};
}
const version = getDependencyVersion(pkg, 'fastify');
// Check if version is 5.x (recommended)
if (version != null && !version.includes('5')) {
return {
name: 'fastify',
status: 'warn',
message: `Fastify ${version} is installed. APOPHIS is tested with Fastify 5.x.`,
detail: 'Consider upgrading to fastify@^5.0.0 for best compatibility.',
mode: 'all',
};
}
return {
name: 'fastify',
status: 'pass',
message: `Fastify ${version || 'installed'} is present.`,
mode: 'all',
};
}
/**
* Check @fastify/swagger installation.
*/
export function checkSwagger(pkg: Record<string, unknown> | null): DependencyCheckResult {
if (!pkg) {
return {
name: '@fastify/swagger',
status: 'fail',
message: 'No package.json found. Cannot check @fastify/swagger installation.',
detail: 'Ensure you are running from a project root with a package.json file.',
remediation: 'Run npm init -y in your project root, then install dependencies.',
mode: 'all',
};
}
if (!hasDependency(pkg, '@fastify/swagger')) {
return {
name: '@fastify/swagger',
status: 'fail',
message: '@fastify/swagger is not installed.',
detail: 'APOPHIS requires @fastify/swagger for route discovery. Install with: npm install @fastify/swagger@^9.0.0',
remediation: 'npm install @fastify/swagger@^9.0.0',
mode: 'all',
};
}
const version = getDependencyVersion(pkg, '@fastify/swagger');
return {
name: '@fastify/swagger',
status: 'pass',
message: `@fastify/swagger ${version || 'installed'} is present.`,
mode: 'all',
};
}
// ---------------------------------------------------------------------------
// Main dependency check runner
// ---------------------------------------------------------------------------
/**
* Run all dependency checks.
*/
export function runDependencyChecks(options: DependencyCheckOptions): DependencyCheckResult[] {
const { cwd, nodeVersion } = options;
const pkg = loadPackageJson(cwd);
const results: DependencyCheckResult[] = [];
// Node version
results.push(checkNodeVersion(nodeVersion));
// Fastify
results.push(checkFastify(pkg));
// Swagger
results.push(checkSwagger(pkg));
// Check for other missing peer deps
if (pkg) {
const missing = REQUIRED_PEER_DEPS.filter(dep => !hasDependency(pkg, dep));
if (missing.length > 0) {
results.push({
name: 'peer-dependencies',
status: 'fail',
message: `Missing peer dependencies: ${missing.join(', ')}`,
detail: 'Install missing packages to ensure full APOPHIS functionality.',
remediation: `npm install ${missing.join(' ')}`,
mode: 'all',
});
} else {
results.push({
name: 'peer-dependencies',
status: 'pass',
message: 'All required peer dependencies are installed.',
mode: 'all',
});
}
}
return results;
}
+265
View File
@@ -0,0 +1,265 @@
/**
* S8: Doctor thread - Docs and example smoke checks
*
* Checks:
* - Docs examples match current config schema
* - README/APOPHIS.md exists and is readable
* - In CI mode: fail if docs drift from reality
*/
import { readFileSync, existsSync, readdirSync } from 'node:fs';
import { resolve } from 'node:path';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface DocsCheckResult {
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
remediation?: string;
mode?: 'all' | 'verify' | 'observe' | 'qualify';
}
export interface DocsCheckOptions {
cwd: string;
isCI: boolean;
}
// ---------------------------------------------------------------------------
// README / APOPHIS.md check
// ---------------------------------------------------------------------------
/**
* Check if project has documentation files.
*/
export function checkDocsExist(options: DocsCheckOptions): DocsCheckResult {
const { cwd } = options;
const readmePath = resolve(cwd, 'README.md');
const apophisPath = resolve(cwd, 'APOPHIS.md');
const hasReadme = existsSync(readmePath);
const hasApophis = existsSync(apophisPath);
if (hasApophis) {
return {
name: 'docs-exist',
status: 'pass',
message: 'APOPHIS.md documentation found.',
mode: 'all',
};
}
if (hasReadme) {
return {
name: 'docs-exist',
status: 'pass',
message: 'README.md found (no APOPHIS.md).',
detail: 'Consider creating APOPHIS.md for APOPHIS-specific documentation.',
mode: 'all',
};
}
return {
name: 'docs-exist',
status: 'warn',
message: 'No README.md or APOPHIS.md found.',
detail: 'Documentation helps team members understand your APOPHIS setup.',
remediation: 'Create APOPHIS.md with setup instructions for your team.',
mode: 'all',
};
}
// ---------------------------------------------------------------------------
// Config schema drift check
// ---------------------------------------------------------------------------
/**
* Known legacy field names that should not appear in docs.
*/
const LEGACY_FIELD_NAMES = [
'testMode',
'testProfiles',
'testPresets',
'envPolicies',
'usesPreset',
'routeFilter',
'testDepth',
'maxDuration',
'canVerify',
];
/**
* Check if docs contain legacy field names (indicating stale docs).
*/
export function checkDocsSchemaDrift(options: DocsCheckOptions): DocsCheckResult {
const { cwd, isCI } = options;
const docsFiles = findDocsFiles(cwd);
if (docsFiles.length === 0) {
return {
name: 'docs-schema-drift',
status: 'warn',
message: 'No documentation files found to check for schema drift.',
mode: 'all',
};
}
const drift: Array<{ file: string; legacyFields: string[] }> = [];
for (const file of docsFiles) {
try {
const content = readFileSync(file, 'utf-8');
const foundLegacy = LEGACY_FIELD_NAMES.filter(field => content.includes(field));
if (foundLegacy.length > 0) {
drift.push({ file, legacyFields: foundLegacy });
}
} catch {
// Skip unreadable files
}
}
if (drift.length > 0) {
const details = drift
.map(d => ` ${d.file}: ${d.legacyFields.join(', ')}`)
.join('\n');
return {
name: 'docs-schema-drift',
status: isCI ? 'fail' : 'warn',
message: `Found ${drift.length} documentation file(s) with legacy field names.`,
detail: `Update docs to use current config schema:\n${details}\n\nRun "apophis migrate --dry-run" to see rewrites.`,
remediation: 'Update docs to use current field names, or run "apophis migrate --dry-run" to see rewrites.',
mode: 'all',
};
}
return {
name: 'docs-schema-drift',
status: 'pass',
message: 'No schema drift detected in documentation.',
mode: 'all',
};
}
/**
* Find documentation files in the project.
*/
function findDocsFiles(cwd: string): string[] {
const files: string[] = [];
const candidates = [
'README.md',
'APOPHIS.md',
'docs',
];
for (const candidate of candidates) {
const fullPath = resolve(cwd, candidate);
if (existsSync(fullPath)) {
if (candidate.endsWith('.md')) {
files.push(fullPath);
} else {
// It's a directory, scan for .md files
try {
const entries = readdirSync(fullPath, { withFileTypes: true });
for (const entry of entries) {
if (entry.isFile() && entry.name.endsWith('.md')) {
files.push(resolve(fullPath, entry.name));
}
}
} catch {
// Skip unreadable directories
}
}
}
}
return files;
}
// ---------------------------------------------------------------------------
// Example code check
// ---------------------------------------------------------------------------
/**
* Check if docs contain runnable examples that match current API.
*/
export function checkExamplesValid(options: DocsCheckOptions): DocsCheckResult {
const { cwd } = options;
const apophisPath = resolve(cwd, 'APOPHIS.md');
if (!existsSync(apophisPath)) {
return {
name: 'examples-valid',
status: 'pass',
message: 'No APOPHIS.md to check for examples.',
mode: 'all',
};
}
try {
const content = readFileSync(apophisPath, 'utf-8');
// Check for common example patterns
const hasVerifyExample = content.includes('apophis verify');
const hasObserveExample = content.includes('apophis observe');
const hasQualifyExample = content.includes('apophis qualify');
const issues: string[] = [];
if (!hasVerifyExample) {
issues.push('No verify example found.');
}
if (!hasObserveExample) {
issues.push('No observe example found.');
}
if (issues.length > 0) {
return {
name: 'examples-valid',
status: 'warn',
message: 'APOPHIS.md is missing some command examples.',
detail: issues.join('\n'),
remediation: 'Add examples for verify, observe, and qualify commands to APOPHIS.md.',
mode: 'all',
};
}
return {
name: 'examples-valid',
status: 'pass',
message: 'APOPHIS.md contains examples for core commands.',
mode: 'all',
};
} catch {
return {
name: 'examples-valid',
status: 'warn',
message: 'Could not read APOPHIS.md to check examples.',
mode: 'all',
};
}
}
// ---------------------------------------------------------------------------
// Main docs check runner
// ---------------------------------------------------------------------------
/**
* Run all docs checks.
*/
export function runDocsChecks(options: DocsCheckOptions): DocsCheckResult[] {
const results: DocsCheckResult[] = [];
results.push(checkDocsExist(options));
results.push(checkDocsSchemaDrift(options));
results.push(checkExamplesValid(options));
return results;
}
+282
View File
@@ -0,0 +1,282 @@
/**
* S8: Doctor thread - Route discovery checks
*
* Checks:
* - Can we discover routes from the Fastify app?
* - Are routes properly registered with swagger?
* - Is the app file loadable?
*/
import { existsSync } from 'node:fs';
import { resolve } from 'node:path';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface RouteCheckResult {
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
remediation?: string;
mode: 'all' | 'verify' | 'observe' | 'qualify';
}
export interface RouteCheckOptions {
cwd: string;
configPath?: string;
}
// ---------------------------------------------------------------------------
// App file detection
// ---------------------------------------------------------------------------
const APP_CANDIDATES = [
'app.js',
'app.ts',
'server.js',
'server.ts',
'index.js',
'index.ts',
'src/app.js',
'src/app.ts',
'src/server.js',
'src/server.ts',
'src/index.js',
'src/index.ts',
];
/**
* Find the Fastify app entrypoint file.
*/
function findAppFile(cwd: string): string | null {
for (const candidate of APP_CANDIDATES) {
const fullPath = resolve(cwd, candidate);
if (existsSync(fullPath)) {
return candidate;
}
}
return null;
}
/**
* Check if app file exists and is readable.
*/
export function checkAppFile(options: RouteCheckOptions): RouteCheckResult {
const appFile = findAppFile(options.cwd);
if (!appFile) {
return {
name: 'app-file',
status: 'warn',
message: 'No Fastify app file found.',
detail: `Searched for: ${APP_CANDIDATES.join(', ')}. ` +
'APOPHIS needs an app.js or similar to discover routes.',
remediation: 'Create an app.js or server.js that exports a Fastify instance.',
mode: 'all',
};
}
return {
name: 'app-file',
status: 'pass',
message: `Found Fastify app: ${appFile}`,
mode: 'all',
};
}
// ---------------------------------------------------------------------------
// Route discovery check
// ---------------------------------------------------------------------------
/**
* Attempt to load the app and discover routes.
*/
export async function checkRouteDiscovery(options: RouteCheckOptions): Promise<RouteCheckResult> {
const appFile = findAppFile(options.cwd);
if (!appFile) {
return {
name: 'route-discovery',
status: 'warn',
message: 'Skipping route discovery (no app file found).',
mode: 'all',
};
}
try {
const appPath = resolve(options.cwd, appFile);
const appModule = await import(appPath);
const app = appModule.default || appModule;
// Check if it looks like a Fastify instance
if (!app || typeof app !== 'object') {
return {
name: 'route-discovery',
status: 'fail',
message: `App file ${appFile} does not export a valid object.`,
detail: 'Ensure the app file exports a Fastify instance as default.',
remediation: 'Export your Fastify instance as default: export default app;',
mode: 'all',
};
}
// Try to register APOPHIS plugin for route capture
// Skip if already registered to avoid "decorator already added" errors
const isAlreadyRegistered = app.hasDecorator && typeof app.hasDecorator === 'function' && app.hasDecorator('apophis');
if (!isAlreadyRegistered) {
try {
const apophisPlugin = (await import('../../../../index.js')).default;
if (typeof apophisPlugin === 'function' && typeof app.register === 'function') {
await app.register(apophisPlugin, { runtime: 'off' });
}
} catch (err) {
const errMessage = err instanceof Error ? err.message : String(err);
// If decorator already added, the plugin is pre-registered — that's fine
if (errMessage.includes("decorator 'apophis' has already been added")) {
// Plugin is already registered, proceed with discovery
}
// Otherwise, plugin registration is optional for discovery
}
}
// Try to ready the app so routes are registered
if (typeof app.ready === 'function') {
await app.ready();
}
// Check for routes
let routeCount = 0;
// Fastify 5+ routes access
if (app.routes && typeof app.routes === 'function') {
const routes = app.routes();
routeCount = Array.isArray(routes) ? routes.length : 0;
}
// Fallback: check if we can get routes via inject or other methods
if (routeCount === 0 && app.hasRoute) {
// We can't enumerate, but we can at least verify the app is functional
routeCount = -1; // Unknown but app seems functional
}
if (routeCount === 0) {
return {
name: 'route-discovery',
status: 'warn',
message: `App loaded from ${appFile} but no routes were discovered.`,
detail: 'Ensure routes are registered before exporting the app. ' +
'APOPHIS discovers routes via the onRoute hook.',
remediation: 'Register routes before exporting the app, or ensure the APOPHIS plugin is registered.',
mode: 'all',
};
}
if (routeCount < 0) {
return {
name: 'route-discovery',
status: 'pass',
message: `App loaded from ${appFile}. Route enumeration not available (app is functional).`,
detail: 'Route count could not be determined, but the app appears to be a valid Fastify instance.',
mode: 'all',
};
}
return {
name: 'route-discovery',
status: 'pass',
message: `Discovered ${routeCount} route(s) from ${appFile}.`,
mode: 'all',
};
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
// If the error is a module not found, treat as warn (dependencies may not be installed in test env)
if (message.includes('Cannot find module') || message.includes('Cannot resolve')) {
return {
name: 'route-discovery',
status: 'warn',
message: `Could not load app from ${appFile}: ${message}`,
detail: 'Dependencies may not be installed. Run npm install to resolve.',
remediation: 'Run npm install to install missing dependencies.',
mode: 'all',
};
}
return {
name: 'route-discovery',
status: 'fail',
message: `Failed to load app from ${appFile}: ${message}`,
detail: 'Check that the app file exports a valid Fastify instance and all imports resolve.',
remediation: 'Verify all imports in your app file are correct and the file exports a Fastify instance.',
mode: 'all',
};
}
}
// ---------------------------------------------------------------------------
// Swagger registration check
// ---------------------------------------------------------------------------
/**
* Check if @fastify/swagger is registered in the app.
*/
export async function checkSwaggerRegistration(options: RouteCheckOptions): Promise<RouteCheckResult> {
const appFile = findAppFile(options.cwd);
if (!appFile) {
return {
name: 'swagger-registration',
status: 'warn',
message: 'Skipping swagger check (no app file found).',
mode: 'all',
};
}
try {
const appPath = resolve(options.cwd, appFile);
const content = (await import('node:fs')).readFileSync(appPath, 'utf-8');
if (content.includes('@fastify/swagger') || content.includes('fastify-swagger')) {
return {
name: 'swagger-registration',
status: 'pass',
message: `@fastify/swagger appears to be imported in ${appFile}.`,
mode: 'all',
};
}
return {
name: 'swagger-registration',
status: 'warn',
message: `@fastify/swagger not found in ${appFile}.`,
detail: 'APOPHIS requires @fastify/swagger for route discovery. ' +
'Register it with: await app.register(import("@fastify/swagger"), { openapi: { info: { title: "API", version: "1.0.0" } } });',
remediation: 'npm install @fastify/swagger@^9.0.0 and register it in your app file.',
mode: 'all',
};
} catch {
return {
name: 'swagger-registration',
status: 'warn',
message: `Could not read ${appFile} to check swagger registration.`,
mode: 'all',
};
}
}
// ---------------------------------------------------------------------------
// Main route check runner
// ---------------------------------------------------------------------------
/**
* Run all route discovery checks.
*/
export async function runRouteChecks(options: RouteCheckOptions): Promise<RouteCheckResult[]> {
const results: RouteCheckResult[] = [];
results.push(checkAppFile(options));
results.push(await checkRouteDiscovery(options));
results.push(await checkSwaggerRegistration(options));
return results;
}
+230
View File
@@ -0,0 +1,230 @@
/**
* S8: Doctor thread - Safety checks
*
* Checks:
* - Qualify mode in unsafe environment
* - Environment policy validation
* - Mixed config style safety
*/
import { PolicyEngine, detectEnvironment } from '../../../core/policy-engine.js';
import type { Config } from '../../../core/config-loader.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface SafetyCheckResult {
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
remediation?: string;
mode: 'all' | 'verify' | 'observe' | 'qualify';
}
export interface SafetyCheckOptions {
cwd: string;
config: Config;
env?: string;
modeFilter?: 'verify' | 'observe' | 'qualify' | undefined;
}
// ---------------------------------------------------------------------------
// Qualify in unsafe environment
// ---------------------------------------------------------------------------
/**
* Check if qualify mode would be allowed in the current environment.
*/
export function checkQualifySafety(options: SafetyCheckOptions): SafetyCheckResult {
const { config, env: explicitEnv } = options;
const env = explicitEnv || detectEnvironment();
// Use the policy engine's qualify-specific safety check directly
// The PolicyEngine.check() runs ALL checks including mode-allowed, profile features, etc.
// We only care about whether qualify is blocked in this environment.
const engine = new PolicyEngine({
config,
env,
mode: 'qualify',
});
const result = engine.check();
if (!result.allowed) {
// Find the specific error about qualify being blocked
const qualifyBlockError = result.errors.find(e =>
e.includes('blocked') || e.includes('not allowed') || e.includes('Qualify')
);
if (qualifyBlockError) {
return {
name: 'qualify-safety',
status: 'fail',
message: `Qualify mode is blocked in environment "${env}".`,
detail: qualifyBlockError,
remediation: 'Run in a local or test environment, or update environment policy to allow qualify.',
mode: 'qualify',
};
}
// Other errors (profile features, etc.) are warnings in doctor context
return {
name: 'qualify-safety',
status: 'warn',
message: `Qualify mode has warnings in environment "${env}".`,
detail: result.errors.join('\n') + '\n' + result.warnings.join('\n'),
mode: 'qualify',
};
}
// Even if allowed, there may be warnings
if (result.warnings.length > 0) {
return {
name: 'qualify-safety',
status: 'warn',
message: `Qualify mode is allowed in environment "${env}" with warnings.`,
detail: result.warnings.join('\n'),
mode: 'qualify',
};
}
return {
name: 'qualify-safety',
status: 'pass',
message: `Qualify mode is allowed in environment "${env}".`,
mode: 'qualify',
};
}
// ---------------------------------------------------------------------------
// Environment policy validation
// ---------------------------------------------------------------------------
/**
* Check if environment policies are well-formed.
*/
export function checkEnvironmentPolicies(options: SafetyCheckOptions): SafetyCheckResult {
const { config } = options;
if (!config.environments || Object.keys(config.environments).length === 0) {
return {
name: 'environment-policies',
status: 'pass',
message: 'No environment policies configured. Using defaults.',
detail: 'Default policies: local/test allow all, production blocks qualify/chaos.',
mode: 'all',
};
}
const issues: string[] = [];
for (const [envName, policy] of Object.entries(config.environments)) {
if (!policy.name) {
issues.push(`Environment "${envName}" is missing a name field.`);
}
// Check for inconsistent policy settings
if (policy.allowQualify && policy.blockQualify) {
issues.push(`Environment "${envName}" has both allowQualify and blockQualify set.`);
}
}
if (issues.length > 0) {
return {
name: 'environment-policies',
status: 'fail',
message: `Found ${issues.length} issue(s) in environment policies.`,
detail: issues.join('\n'),
remediation: 'Fix the listed issues in your config environments section.',
mode: 'all',
};
}
return {
name: 'environment-policies',
status: 'pass',
message: `Environment policies are well-formed (${Object.keys(config.environments).length} defined).`,
mode: 'all',
};
}
// ---------------------------------------------------------------------------
// Production safety
// ---------------------------------------------------------------------------
/**
* Check production-specific safety concerns.
*/
export function checkProductionSafety(options: SafetyCheckOptions): SafetyCheckResult {
const { config, env: explicitEnv } = options;
const env = explicitEnv || detectEnvironment();
const isProd = env === 'production' || env === 'prod';
if (!isProd) {
return {
name: 'production-safety',
status: 'pass',
message: `Not in production environment (current: ${env}).`,
mode: 'all',
};
}
const warnings: string[] = [];
// Check if chaos is somehow enabled in prod
const prodPolicy = config.environments?.production || config.environments?.prod;
if (prodPolicy?.allowChaos) {
warnings.push('Chaos is explicitly allowed in production. Ensure this is intentional.');
}
// Check if blocking is enabled in prod
if (prodPolicy?.allowBlocking) {
warnings.push('Blocking behavior is explicitly allowed in production. Ensure this is intentional.');
}
if (warnings.length > 0) {
return {
name: 'production-safety',
status: 'warn',
message: 'Production environment has potentially unsafe settings.',
detail: warnings.join('\n'),
remediation: 'Review your production environment policy and disable chaos/blocking unless intentional.',
mode: 'all',
};
}
return {
name: 'production-safety',
status: 'pass',
message: 'Production environment safety checks passed.',
mode: 'all',
};
}
// ---------------------------------------------------------------------------
// Main safety check runner
// ---------------------------------------------------------------------------
/**
* Run all safety checks, filtering by mode if requested.
* When modeFilter is 'observe', skip qualify-specific checks to avoid noisy failures.
*/
export function runSafetyChecks(options: SafetyCheckOptions): SafetyCheckResult[] {
const results: SafetyCheckResult[] = [];
const { modeFilter } = options;
// Qualify-safety check: run when no filter, or when filtering for qualify
// Skip when filtering for observe (observe users don't care about qualify safety)
// Also skip when filtering for verify (verify users don't care about qualify safety)
if (modeFilter !== 'observe' && modeFilter !== 'verify') {
results.push(checkQualifySafety(options));
}
results.push(checkEnvironmentPolicies(options));
results.push(checkProductionSafety(options));
return results;
}
+491
View File
@@ -0,0 +1,491 @@
/**
* S8: Doctor thread - Main command handler
*
* Responsibilities:
* - Run all diagnostic checks (dependencies, config, routes, safety, docs)
* - Aggregate results with clear pass/fail output
* - Monorepo per-package reporting
* - Exit 0 if all pass, 2 if any fail
* - Mode-scoped checks: --mode verify|observe|qualify filters checks
* - Explicit --config honored uniformly
* - Warnings do not fail unless --strict is passed
*/
import type { CliContext } from '../../core/context.js';
import { loadConfig, detectMonorepo, findWorkspacePackages } from '../../core/config-loader.js';
import { detectEnvironment } from '../../core/policy-engine.js';
import { SUCCESS, USAGE_ERROR } from '../../core/exit-codes.js';
import type { WorkspaceResult, WorkspaceRun } from '../../core/types.js';
import { runWorkspace, formatWorkspaceHuman, formatWorkspaceJson, formatWorkspaceNdjson } from '../../core/workspace-runner.js';
import { runDependencyChecks } from './checks/dependencies.js';
import { runConfigChecks } from './checks/config.js';
import { runRouteChecks } from './checks/routes.js';
import { runSafetyChecks } from './checks/safety.js';
import { runDocsChecks } from './checks/docs.js';
import { renderJson } from '../../renderers/json.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export type DoctorMode = 'verify' | 'observe' | 'qualify' | undefined;
export interface DoctorOptions {
config?: string;
cwd?: string;
format?: 'human' | 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary';
quiet?: boolean;
verbose?: boolean;
mode?: DoctorMode;
strict?: boolean;
}
export interface DoctorCheck {
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
remediation?: string;
mode?: 'all' | 'verify' | 'observe' | 'qualify';
package?: string;
}
export interface DoctorResult {
exitCode: number;
message?: string;
checks: DoctorCheck[];
summary: {
total: number;
passed: number;
failed: number;
warnings: number;
};
}
// ---------------------------------------------------------------------------
// Check filtering
// ---------------------------------------------------------------------------
function shouldRunCheck(checkMode: string | undefined, modeFilter: DoctorMode): boolean {
if (!modeFilter) return true;
if (!checkMode || checkMode === 'all') return true;
return checkMode === modeFilter;
}
// ---------------------------------------------------------------------------
// Monorepo detection
// ---------------------------------------------------------------------------
/**
* Find all packages in a monorepo.
*/
import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
import { resolve } from 'node:path';
function findMonorepoPackages(cwd: string): string[] {
const pkgPath = resolve(cwd, 'package.json');
if (!existsSync(pkgPath)) {
return [];
}
try {
const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
const workspaces = pkg.workspaces;
if (!workspaces || !Array.isArray(workspaces)) {
return [];
}
const packages: string[] = [];
for (const pattern of workspaces) {
if (pattern.endsWith('/*')) {
const dir = pattern.slice(0, -2);
const dirPath = resolve(cwd, dir);
if (existsSync(dirPath)) {
const entries = readdirSync(dirPath, { withFileTypes: true });
for (const entry of entries) {
if (entry.isDirectory()) {
packages.push(resolve(dirPath, entry.name));
}
}
}
} else {
// Handle exact paths like "packages/api"
const exactPath = resolve(cwd, pattern);
if (existsSync(exactPath)) {
const stat = statSync(exactPath);
if (stat.isDirectory()) {
packages.push(exactPath);
}
}
}
}
return packages;
} catch {
return [];
}
}
// ---------------------------------------------------------------------------
// Check runners per package
// ---------------------------------------------------------------------------
/**
* Run all checks for a single package directory.
* Honors explicit configPath and mode filter.
*/
async function runPackageChecks(
cwd: string,
ctx: CliContext,
configPath: string | undefined,
modeFilter: DoctorMode,
packageName?: string,
): Promise<DoctorCheck[]> {
const checks: DoctorCheck[] = [];
// 1. Dependency checks (all modes)
const depResults = runDependencyChecks({
cwd,
nodeVersion: process.version,
});
for (const result of depResults) {
checks.push({ ...result, package: packageName });
}
// 2. Config checks (all modes) — honor explicit configPath
const configResults = await runConfigChecks({ cwd, configPath });
for (const result of configResults) {
checks.push({ ...result, package: packageName });
}
// 3. Route checks (all modes)
const routeResults = await runRouteChecks({ cwd, configPath });
for (const result of routeResults) {
checks.push({ ...result, package: packageName });
}
// 4. Safety checks (mode-scoped) — need loaded config, honor explicit configPath
try {
const loadResult = await loadConfig({ cwd, configPath });
const env = detectEnvironment();
const safetyResults = runSafetyChecks({
cwd,
config: loadResult.config,
env,
modeFilter,
});
for (const result of safetyResults) {
checks.push({ ...result, package: packageName });
}
} catch {
// If config can't be loaded, add a safety check note only if not filtering for observe
if (!modeFilter || modeFilter !== 'observe') {
checks.push({
name: 'safety-checks',
status: 'warn',
message: 'Could not run safety checks (config failed to load).',
mode: 'all',
package: packageName,
});
}
}
// 5. Docs checks (all modes)
const docsResults = runDocsChecks({
cwd,
isCI: ctx.isCI,
});
for (const result of docsResults) {
checks.push({ ...result, package: packageName });
}
// 6. Determinism trust signal
const testSeed = Math.floor(Math.random() * 0x7fffffff);
checks.push({
name: 'determinism',
status: 'pass',
message: `Environment supports deterministic replay (test seed: ${testSeed})`,
detail: `Run with --seed ${testSeed} to reproduce the exact same test sequence`,
mode: 'all',
package: packageName,
});
return checks;
}
// ---------------------------------------------------------------------------
// Output formatting
// ---------------------------------------------------------------------------
/**
* Format check results for human-readable output.
* Each check shows: name, status, message, mode relevance, remediation.
*/
function formatHumanOutput(result: DoctorResult, isMonorepo: boolean, modeFilter?: DoctorMode): string {
const lines: string[] = [];
lines.push('APOPHIS Doctor');
if (modeFilter) {
lines.push(`Mode: ${modeFilter}`);
}
lines.push('');
if (isMonorepo && result.checks.some(c => c.package)) {
// Group by package
const packages = new Map<string | undefined, DoctorCheck[]>();
for (const check of result.checks) {
const pkg = check.package || 'root';
if (!packages.has(pkg)) {
packages.set(pkg, []);
}
packages.get(pkg)!.push(check);
}
for (const [pkg, checks] of packages) {
lines.push(`📦 ${pkg}`);
lines.push('');
for (const check of checks) {
const icon = check.status === 'pass' ? '✓' : check.status === 'warn' ? '⚠' : '✗';
const modeLabel = check.mode === 'all' ? '' : ` [${check.mode}]`;
lines.push(` ${icon} ${check.name}${modeLabel}: ${check.message}`);
if (check.detail) {
lines.push(` ${check.detail}`);
}
if (check.remediation) {
lines.push(`${check.remediation}`);
}
}
lines.push('');
}
} else {
// Flat list
for (const check of result.checks) {
const icon = check.status === 'pass' ? '✓' : check.status === 'warn' ? '⚠' : '✗';
const modeLabel = check.mode === 'all' ? '' : ` [${check.mode}]`;
lines.push(` ${icon} ${check.name}${modeLabel}: ${check.message}`);
if (check.detail) {
lines.push(` ${check.detail}`);
}
if (check.remediation) {
lines.push(`${check.remediation}`);
}
}
lines.push('');
}
// Summary
const { summary } = result;
lines.push(`Summary: ${summary.passed} passed, ${summary.failed} failed, ${summary.warnings} warnings`);
if (summary.failed > 0) {
lines.push('');
lines.push('Run "apophis migrate" to fix legacy config issues.');
lines.push('Run "apophis init" to scaffold missing configuration.');
}
return lines.join('\n');
}
// ---------------------------------------------------------------------------
// Main command handler
// ---------------------------------------------------------------------------
/**
* Main doctor command handler.
*
* Flow:
* 1. Parse mode and strict flags
* 2. Detect if monorepo
* 3. Run checks for root package (honoring explicit configPath)
* 4. If monorepo, run checks for each workspace package
* 5. Aggregate results
* 6. Format output
* 7. Return exit code (warnings fail only if --strict)
*/
export async function doctorCommand(
options: DoctorOptions,
ctx: CliContext,
): Promise<DoctorResult> {
const { config: configPath, cwd, mode: modeFilter, strict } = options;
const workingDir = cwd || ctx.cwd;
try {
// Detect monorepo
const isMonorepo = detectMonorepo(workingDir);
const allChecks: DoctorCheck[] = [];
// Run checks for root — pass explicit configPath so every check uses it
const rootChecks = await runPackageChecks(
workingDir,
ctx,
configPath,
modeFilter,
isMonorepo ? 'root' : undefined,
);
allChecks.push(...rootChecks);
// If monorepo, run checks for each package
if (isMonorepo) {
const packages = findMonorepoPackages(workingDir);
for (const pkgPath of packages) {
const pkgName = pkgPath.split('/').pop() || 'unknown';
const pkgChecks = await runPackageChecks(pkgPath, ctx, configPath, modeFilter, pkgName);
allChecks.push(...pkgChecks);
}
}
// Calculate summary
const passed = allChecks.filter(c => c.status === 'pass').length;
const failed = allChecks.filter(c => c.status === 'fail').length;
const warnings = allChecks.filter(c => c.status === 'warn').length;
// Warnings fail the run only when --strict is passed
const effectiveFailed = failed + (strict ? warnings : 0);
const result: DoctorResult = {
exitCode: effectiveFailed > 0 ? USAGE_ERROR : SUCCESS,
checks: allChecks,
summary: {
total: allChecks.length,
passed,
failed,
warnings,
},
};
// Format message
result.message = formatHumanOutput(result, isMonorepo, modeFilter);
return result;
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return {
exitCode: USAGE_ERROR,
message: `Doctor command failed: ${message}`,
checks: [],
summary: { total: 0, passed: 0, failed: 0, warnings: 0 },
};
}
}
// ---------------------------------------------------------------------------
// CLI adapter
// ---------------------------------------------------------------------------
/**
* Adapter that bridges the CLI framework (cac) to the doctor command handler.
* Parses --mode and --strict from CLI args.
*/
export async function handleDoctor(
args: string[],
ctx: CliContext,
): Promise<number> {
// Parse --mode and --strict from raw args (cac doesn't expose unknown flags nicely)
const modeFlag = args.find(a => a.startsWith('--mode='));
const modeArg = args.find(a => a === '--mode');
const modeIndex = modeArg ? args.indexOf(modeArg) : -1;
let mode: DoctorMode = undefined;
if (modeFlag) {
const value = modeFlag.split('=')[1];
if (value === 'verify' || value === 'observe' || value === 'qualify') {
mode = value;
}
} else if (modeIndex >= 0 && args[modeIndex + 1]) {
const value = args[modeIndex + 1];
if (value === 'verify' || value === 'observe' || value === 'qualify') {
mode = value;
}
}
const strict = args.includes('--strict');
const options: DoctorOptions = {
config: ctx.options.config || undefined,
cwd: ctx.cwd,
format: ctx.options.format as Exclude<DoctorOptions['format'], undefined>,
quiet: ctx.options.quiet,
verbose: ctx.options.verbose,
mode,
strict,
};
const workspaceMode = args.includes('--workspace');
if (workspaceMode) {
const workspaceResult = await runWorkspace(
{
runCommand: async (pkgCtx) => {
const pkgOptions = { ...options, cwd: pkgCtx.cwd };
const pkgResult = await doctorCommand(pkgOptions, pkgCtx);
return {
exitCode: pkgResult.exitCode,
artifact: {
version: 'apophis-artifact/1',
command: 'doctor',
cwd: pkgCtx.cwd,
startedAt: new Date().toISOString(),
durationMs: 0,
summary: {
total: pkgResult.summary.total,
passed: pkgResult.summary.passed,
failed: pkgResult.summary.failed,
},
failures: [],
artifacts: [],
warnings: pkgResult.checks
.filter(c => c.status === 'warn' || c.status === 'fail')
.map(c => `${c.name}: ${c.message}`),
exitReason: pkgResult.exitCode === SUCCESS ? 'success' : 'behavioral_failure',
},
warnings: pkgResult.checks
.filter(c => c.status === 'warn')
.map(c => c.message),
};
},
},
ctx,
);
if (!ctx.options.quiet) {
const format = options.format || ctx.options.format || 'human';
if (format === 'json') {
console.log(formatWorkspaceJson(workspaceResult));
} else if (format === 'ndjson') {
console.log(formatWorkspaceNdjson(workspaceResult));
} else {
console.log(formatWorkspaceHuman(workspaceResult));
}
}
return workspaceResult.exitCode;
}
const result = await doctorCommand(options, ctx);
// Output result based on format
if (!ctx.options.quiet && result.message) {
const format = options.format || ctx.options.format || 'human';
if (format === 'json') {
console.log(renderJson({
exitCode: result.exitCode,
summary: result.summary,
checks: result.checks,
}));
} else if (format === 'ndjson') {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'doctor',
exitCode: result.exitCode,
summary: result.summary,
checks: result.checks,
}) + '\n');
} else {
console.log(result.message);
}
}
return result.exitCode;
}
+644
View File
@@ -0,0 +1,644 @@
/**
* S3: Init command for APOPHIS CLI
* Scaffold config, scripts, and example usage in one pass.
*/
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
import { resolve } from 'node:path';
import type { CliContext } from '../../core/types.js';
import { USAGE_ERROR, SUCCESS } from '../../core/exit-codes.js';
import { getScaffoldForPreset, getPresetNames, type ScaffoldResult } from './scaffolds/index.js';
// ─────────────────────────────────────────────────────────────────────────────
// Types
// ─────────────────────────────────────────────────────────────────────────────
export interface InitOptions {
preset?: string;
force?: boolean;
noninteractive?: boolean;
cwd?: string;
}
export interface InitResult {
exitCode: number;
message: string;
filesWritten: string[];
nextCommand: string;
}
const DEFAULT_INSTALL_PM: Exclude<CliContext['packageManager'], 'unknown'> = 'npm';
function normalizePackageManager(packageManager: CliContext['packageManager'] | undefined): Exclude<CliContext['packageManager'], 'unknown'> {
if (!packageManager || packageManager === 'unknown') {
return DEFAULT_INSTALL_PM;
}
return packageManager;
}
function renderInstallCommand(
packageManager: CliContext['packageManager'] | undefined,
packages: string[],
): string {
const normalized = normalizePackageManager(packageManager);
if (normalized === 'yarn') {
return `yarn add ${packages.join(' ')}`;
}
if (normalized === 'pnpm') {
return `pnpm add ${packages.join(' ')}`;
}
if (normalized === 'bun') {
return `bun add ${packages.join(' ')}`;
}
return `npm install ${packages.join(' ')}`;
}
// ─────────────────────────────────────────────────────────────────────────────
// Fastify detection
// ─────────────────────────────────────────────────────────────────────────────
/**
* Detect if the project is a Fastify app by looking for:
* - fastify imports in JS/TS files
* - Common server file names (server.js, app.js, index.js, etc.)
*/
export async function detectFastifyEntrypoint(cwd: string): Promise<string | null> {
const candidates = [
'app.js',
'app.ts',
'server.js',
'server.ts',
'index.js',
'index.ts',
'src/app.js',
'src/app.ts',
'src/server.js',
'src/server.ts',
'src/index.js',
'src/index.ts',
];
for (const candidate of candidates) {
const fullPath = resolve(cwd, candidate);
if (!existsSync(fullPath)) continue;
const content = readFileSync(fullPath, 'utf-8');
// Look for fastify import patterns
if (
content.includes('fastify') ||
content.includes('Fastify') ||
content.includes('@fastify') ||
content.includes('fastify-plugin')
) {
return candidate;
}
}
return null;
}
/**
* Check if @fastify/swagger is registered in the project.
* We check package.json dependencies and the entrypoint file.
*/
export async function checkSwaggerRegistration(cwd: string, entrypoint: string | null): Promise<{
hasSwaggerDep: boolean;
hasSwaggerImport: boolean;
}> {
const pkgPath = resolve(cwd, 'package.json');
let hasSwaggerDep = false;
if (existsSync(pkgPath)) {
try {
const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
const deps = {
...pkg.dependencies,
...pkg.devDependencies,
};
hasSwaggerDep = '@fastify/swagger' in deps;
} catch {
// Ignore parse errors
}
}
let hasSwaggerImport = false;
if (entrypoint) {
const entryPath = resolve(cwd, entrypoint);
if (existsSync(entryPath)) {
const content = readFileSync(entryPath, 'utf-8');
hasSwaggerImport =
content.includes('@fastify/swagger') ||
content.includes('fastify-swagger');
}
}
return { hasSwaggerDep, hasSwaggerImport };
}
/**
* Detect if the project uses TypeScript.
*/
export function detectTypeScript(cwd: string): boolean {
return (
existsSync(resolve(cwd, 'tsconfig.json')) ||
existsSync(resolve(cwd, 'src/app.ts')) ||
existsSync(resolve(cwd, 'src/server.ts')) ||
existsSync(resolve(cwd, 'src/index.ts'))
);
}
// ─────────────────────────────────────────────────────────────────────────────
// Package.json script merging
// ─────────────────────────────────────────────────────────────────────────────
/**
* Merge apophis scripts into package.json without clobbering existing scripts.
*/
export function mergePackageScripts(pkg: Record<string, unknown>): Record<string, unknown> {
const scripts = (pkg.scripts as Record<string, string>) || {};
const apophisScripts: Record<string, string> = {
'apophis:verify': 'apophis verify --profile quick',
'apophis:doctor': 'apophis doctor',
};
const mergedScripts = { ...scripts };
for (const [key, value] of Object.entries(apophisScripts)) {
// Only add if not already present
if (!(key in mergedScripts)) {
mergedScripts[key] = value;
}
}
return {
...pkg,
scripts: mergedScripts,
};
}
// ─────────────────────────────────────────────────────────────────────────────
// File writing
// ─────────────────────────────────────────────────────────────────────────────
/**
* Write the config file (apophis.config.js or .ts).
*/
export function writeConfigFile(
cwd: string,
scaffold: ScaffoldResult,
isTypeScript: boolean,
force: boolean,
): { path: string; existed: boolean } {
const ext = isTypeScript ? 'ts' : 'js';
const configPath = resolve(cwd, `apophis.config.${ext}`);
const existed = existsSync(configPath);
if (existed && !force) {
return { path: configPath, existed: true };
}
const configContent = generateConfigContent(scaffold.config, isTypeScript);
writeFileSync(configPath, configContent, 'utf-8');
return { path: configPath, existed: false };
}
/**
* Generate config file content as a formatted string.
*/
function generateConfigContent(config: ScaffoldResult['config'], isTypeScript: boolean): string {
const lines: string[] = [];
lines.push('/**');
lines.push(' * APOPHIS configuration');
lines.push(' * Generated by `apophis init`');
lines.push(' */');
lines.push('');
if (isTypeScript) {
lines.push('import type { ApophisConfig } from "apophis-fastify/cli";');
lines.push('');
lines.push('const config: ApophisConfig = ' + stringifyConfig(config) + ';');
lines.push('');
lines.push('export default config;');
} else {
lines.push('export default ' + stringifyConfig(config) + ';');
}
lines.push('');
return lines.join('\n');
}
/**
* Stringify a config object with proper indentation.
*/
function stringifyConfig(obj: unknown, indent = 2): string {
if (obj === null) return 'null';
if (typeof obj === 'string') return JSON.stringify(obj);
if (typeof obj === 'number') return String(obj);
if (typeof obj === 'boolean') return String(obj);
if (Array.isArray(obj)) {
if (obj.length === 0) return '[]';
const items = obj.map(item => stringifyConfig(item, indent + 2)).join(',\n' + ' '.repeat(indent));
return '[\n' + ' '.repeat(indent) + items + '\n' + ' '.repeat(indent - 2) + ']';
}
if (typeof obj === 'object') {
const entries = Object.entries(obj as Record<string, unknown>);
if (entries.length === 0) return '{}';
const items = entries
.map(([key, value]) => {
const keyStr = /^[a-zA-Z_$][a-zA-Z0-9_$]*$/.test(key) ? key : JSON.stringify(key);
return `${keyStr}: ${stringifyConfig(value, indent + 2)}`;
})
.join(',\n' + ' '.repeat(indent));
return '{\n' + ' '.repeat(indent) + items + '\n' + ' '.repeat(indent - 2) + '}';
}
return String(obj);
}
/**
* Write the README guidance file.
*/
export function writeReadmeFile(
cwd: string,
scaffold: ScaffoldResult,
force: boolean,
): { path: string; existed: boolean } {
const readmePath = resolve(cwd, 'APOPHIS.md');
const existed = existsSync(readmePath);
if (existed && !force) {
return { path: readmePath, existed: true };
}
writeFileSync(readmePath, scaffold.readmeContent.trim() + '\n', 'utf-8');
return { path: readmePath, existed: false };
}
/**
* Update package.json with merged scripts.
*/
export function updatePackageJson(cwd: string): { path: string; modified: boolean; error?: string } {
const pkgPath = resolve(cwd, 'package.json');
if (!existsSync(pkgPath)) {
const bootstrapPackage = {
name: 'apophis-app',
version: '0.1.0',
private: true,
type: 'module',
scripts: {
'apophis:doctor': 'apophis doctor',
'apophis:verify': 'apophis verify --profile quick',
},
dependencies: {
fastify: '^5.0.0',
'@fastify/swagger': '^9.0.0',
},
};
try {
writeFileSync(pkgPath, JSON.stringify(bootstrapPackage, null, 2) + '\n', 'utf-8');
return { path: pkgPath, modified: true };
} catch (err) {
return { path: pkgPath, modified: false, error: String(err) };
}
}
try {
const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
const merged = mergePackageScripts(pkg);
// Check if anything changed
const originalScripts = JSON.stringify(pkg.scripts || {});
const mergedScripts = JSON.stringify(merged.scripts || {});
if (originalScripts === mergedScripts) {
return { path: pkgPath, modified: false };
}
writeFileSync(pkgPath, JSON.stringify(merged, null, 2) + '\n', 'utf-8');
return { path: pkgPath, modified: true };
} catch (err) {
return { path: pkgPath, modified: false, error: String(err) };
}
}
export function writeBootstrapAppFile(
cwd: string,
existingEntrypoint: string | null,
): { path: string; created: boolean } {
const appPath = resolve(cwd, 'app.js');
if (existingEntrypoint || existsSync(appPath)) {
return { path: appPath, created: false };
}
const appContent = `/**
* Generated by \`apophis init\`.
* This is a minimal Fastify-like app that is runnable with \`apophis verify\`.
*/
const routes = [
{
method: 'POST',
url: '/users',
schema: {
body: {
type: 'object',
required: ['name'],
properties: {
name: { type: 'string' },
},
},
response: {
201: {
type: 'object',
properties: {
id: { type: 'string' },
name: { type: 'string' },
},
},
},
'x-ensures': [
'response_code(this) == 201',
],
},
},
];
const app = {
routes,
async ready() {},
hasRoute({ method, url }) {
const normalizedMethod = String(method || '').toUpperCase();
return routes.some(route => route.method === normalizedMethod && route.url === url);
},
async inject({ method, url, payload }) {
const normalizedMethod = String(method || '').toUpperCase();
if (normalizedMethod === 'POST' && url === '/users') {
const body = {
id: 'usr-1',
name: payload && typeof payload === 'object' && 'name' in payload
? String(payload.name)
: 'test',
};
return {
statusCode: 201,
headers: { 'content-type': 'application/json' },
body,
json() {
return body;
},
};
}
const body = { message: 'not found' };
return {
statusCode: 404,
headers: { 'content-type': 'application/json' },
body,
json() {
return body;
},
};
},
};
export default app;
`;
writeFileSync(appPath, appContent, 'utf-8');
return { path: appPath, created: true };
}
// ─────────────────────────────────────────────────────────────────────────────
// Interactive prompts (lazy-loaded)
// ─────────────────────────────────────────────────────────────────────────────
interface PromptsModule {
select: (opts: { message: string; options: { value: string; label: string }[] }) => Promise<string>;
confirm: (opts: { message: string }) => Promise<boolean>;
text: (opts: { message: string; placeholder?: string }) => Promise<string>;
}
async function loadPrompts(): Promise<PromptsModule> {
// Lazy-load @clack/prompts only when interactive
const mod = await import('@clack/prompts');
return mod as unknown as PromptsModule;
}
// ─────────────────────────────────────────────────────────────────────────────
// Main init handler
// ─────────────────────────────────────────────────────────────────────────────
export async function initHandler(args: string[], ctx: CliContext): Promise<InitResult> {
const options = parseInitOptions(args, ctx);
const cwd = options.cwd || ctx.cwd;
// Detect project structure
const isTypeScript = detectTypeScript(cwd);
const fastifyEntry = await detectFastifyEntrypoint(cwd);
const swaggerCheck = await checkSwaggerRegistration(cwd, fastifyEntry);
// Determine preset
let preset = options.preset;
if (!preset) {
if (options.noninteractive) {
return {
exitCode: USAGE_ERROR,
message: 'Missing required --preset flag in non-interactive mode. Use one of: ' + getPresetNames().join(', '),
filesWritten: [],
nextCommand: '',
};
}
// Interactive mode: prompt for preset
if (ctx.isTTY && !ctx.isCI) {
try {
const prompts = await loadPrompts();
const presetNames = getPresetNames();
const choice = await prompts.select({
message: 'Choose a preset:',
options: presetNames.map(name => ({
value: name,
label: name,
})),
});
preset = choice;
} catch {
// Fallback if prompts fail
return {
exitCode: USAGE_ERROR,
message: 'Failed to prompt for preset. Use --preset <name> in non-interactive mode.',
filesWritten: [],
nextCommand: '',
};
}
} else {
// Non-TTY, non-CI: default to safe-ci
preset = 'safe-ci';
}
}
// Validate preset
const scaffold = getScaffoldForPreset(preset);
if (!scaffold) {
return {
exitCode: USAGE_ERROR,
message: `Unknown preset "${preset}". Available presets: ${getPresetNames().join(', ')}`,
filesWritten: [],
nextCommand: '',
};
}
// Check for existing config
const configExt = isTypeScript ? 'ts' : 'js';
const configPath = resolve(cwd, `apophis.config.${configExt}`);
const configExisted = existsSync(configPath);
if (configExisted && !options.force) {
return {
exitCode: USAGE_ERROR,
message: `Config file already exists: apophis.config.${configExt}. Use --force to overwrite.`,
filesWritten: [],
nextCommand: '',
};
}
// Write files
const filesWritten: string[] = [];
const forceWrite = options.force ?? false;
const configResult = writeConfigFile(cwd, scaffold, isTypeScript, forceWrite);
if (configResult.existed && !forceWrite) {
return {
exitCode: USAGE_ERROR,
message: `Config file already exists: ${configResult.path}. Use --force to overwrite.`,
filesWritten: [],
nextCommand: '',
};
}
filesWritten.push(configResult.path);
const readmeResult = writeReadmeFile(cwd, scaffold, forceWrite);
if (!readmeResult.existed || forceWrite) {
filesWritten.push(readmeResult.path);
}
const pkgResult = updatePackageJson(cwd);
if (pkgResult.modified) {
filesWritten.push(pkgResult.path);
}
const bootstrapAppResult = writeBootstrapAppFile(cwd, fastifyEntry);
if (bootstrapAppResult.created) {
filesWritten.push(bootstrapAppResult.path);
}
// Build next command
const profileName = scaffold.config.profile || 'quick';
const routeHint = scaffold.config.routes?.[0] || '';
const nextCommand = routeHint
? `apophis verify --profile ${profileName} --routes "${routeHint}"`
: `apophis verify --profile ${profileName}`;
// Build message
const lines: string[] = [];
lines.push(`Initialized APOPHIS with preset "${preset}"`);
lines.push('');
lines.push('Files written:');
for (const file of filesWritten) {
lines.push(` ${file}`);
}
const installPeerDepsCommand = renderInstallCommand(ctx.packageManager, ['fastify', '@fastify/swagger']);
const installSwaggerCommand = renderInstallCommand(ctx.packageManager, ['@fastify/swagger']);
lines.push('');
lines.push('First success path:');
lines.push(` 1. ${installPeerDepsCommand}`);
lines.push(' 2. apophis doctor');
lines.push(` 3. ${nextCommand}`);
lines.push('');
lines.push('If verify says "No behavioral contracts found", add x-ensures to your route schema:');
lines.push(' "x-ensures": [');
lines.push(' "response_code(GET /users/{response_body(this).id}) == 200"');
lines.push(' ]');
lines.push('');
lines.push('See APOPHIS.md and docs/getting-started.md for full examples.');
if (!swaggerCheck.hasSwaggerDep && !bootstrapAppResult.created) {
lines.push('');
lines.push('Warning: @fastify/swagger not found in dependencies.');
lines.push('APOPHIS requires @fastify/swagger to discover routes.');
lines.push('Install it with:');
lines.push(` ${installSwaggerCommand}`);
} else if (!bootstrapAppResult.created && !swaggerCheck.hasSwaggerImport) {
lines.push('');
lines.push('Warning: @fastify/swagger is installed but not imported in your entrypoint.');
lines.push('Register it in your Fastify app:');
lines.push(` await app.register(import("@fastify/swagger"), { openapi: { info: { title: "API", version: "1.0.0" } } });`);
}
if (fastifyEntry) {
lines.push('');
lines.push(`Detected Fastify entrypoint: ${fastifyEntry}`);
}
lines.push('');
lines.push('Next command:');
lines.push(` ${nextCommand}`);
return {
exitCode: SUCCESS,
message: lines.join('\n'),
filesWritten,
nextCommand,
};
}
// ─────────────────────────────────────────────────────────────────────────────
// Option parsing
// ─────────────────────────────────────────────────────────────────────────────
function parseInitOptions(args: string[], ctx: CliContext): InitOptions {
const options: InitOptions = {};
for (let i = 0; i < args.length; i++) {
const arg = args[i];
if (arg === '--preset' || arg === '-p') {
options.preset = args[++i];
} else if (arg === '--force' || arg === '-f') {
options.force = true;
} else if (arg === '--noninteractive') {
options.noninteractive = true;
} else if (arg === '--cwd') {
options.cwd = args[++i];
}
}
// Non-interactive if CI or not TTY
if (ctx.isCI || !ctx.isTTY) {
options.noninteractive = true;
}
return options;
}
// ─────────────────────────────────────────────────────────────────────────────
// CLI adapter
// ─────────────────────────────────────────────────────────────────────────────
export async function handleInit(args: string[], ctx: CliContext): Promise<number> {
const result = await initHandler(args, ctx);
if (result.message) {
console.log(result.message);
}
return result.exitCode;
}
+374
View File
@@ -0,0 +1,374 @@
/**
* S3: Init command scaffold templates
* Each preset returns a config object and file contents for the init command.
*/
import type { ApophisConfig, PresetDefinition, ProfileDefinition, EnvironmentPolicy } from '../../../core/types.js';
export interface ScaffoldResult {
config: ApophisConfig;
readmeContent: string;
}
// ─────────────────────────────────────────────────────────────────────────────
// safe-ci: Minimal CI-safe preset (default)
// ─────────────────────────────────────────────────────────────────────────────
export function safeCiScaffold(): ScaffoldResult {
const preset: PresetDefinition = {
name: 'safe-ci',
depth: 'quick',
timeout: 5000,
parallel: false,
chaos: false,
observe: false,
};
const profile: ProfileDefinition = {
name: 'quick',
mode: 'verify',
preset: 'safe-ci',
routes: ['POST /users'],
};
const envLocal: EnvironmentPolicy = {
name: 'local',
allowVerify: true,
allowObserve: true,
allowQualify: false,
allowChaos: false,
allowBlocking: true,
requireSink: false,
};
const config: ApophisConfig = {
mode: 'verify',
profiles: { quick: profile },
presets: { 'safe-ci': preset },
environments: { local: envLocal },
};
const readmeContent = `
# APOPHIS Setup — safe-ci preset
This project was scaffolded with \`apophis init --preset safe-ci\`.
## Quick Start
1. Ensure you have a Fastify app with @fastify/swagger registered.
2. Add behavioral contracts to your route schemas using \`x-ensures\`.
3. Run: apophis verify --profile quick
## What This Preset Does
- Runs only behavioral contracts (not schema-only routes).
- No chaos, no observe, no stateful testing.
- Safe for CI pipelines.
- Timeout: 5s per route.
## Example Behavioral Contract
Add this inside your route schema to check that a created resource is retrievable:
\`\`\`javascript
"x-ensures": [
"response_code(GET /users/{response_body(this).id}) == 200"
]
\`\`\`
If \`apophis verify\` says "No behavioral contracts found", it means your routes have schemas but no \`x-ensures\` or \`x-requires\` clauses. Add at least one clause per route you want to verify.
## Next Steps
- Add more routes to the \`routes\` array in your profile.
- Try \`apophis init --preset platform-observe\` for production readiness.
- Try \`apophis init --preset protocol-lab\` for multi-step flows.
`;
return { config, readmeContent };
}
// ─────────────────────────────────────────────────────────────────────────────
// platform-observe: Production-ready with observe mode
// ─────────────────────────────────────────────────────────────────────────────
export function platformObserveScaffold(): ScaffoldResult {
const preset: PresetDefinition = {
name: 'platform-observe',
depth: 'standard',
timeout: 10000,
parallel: true,
chaos: false,
observe: true,
};
const profile: ProfileDefinition = {
name: 'staging-observe',
mode: 'observe',
preset: 'platform-observe',
routes: [],
};
const envStaging: EnvironmentPolicy = {
name: 'staging',
allowVerify: true,
allowObserve: true,
allowQualify: true,
allowChaos: false,
allowBlocking: false,
requireSink: true,
};
const envProduction: EnvironmentPolicy = {
name: 'production',
allowVerify: true,
allowObserve: true,
allowQualify: false,
allowChaos: false,
allowBlocking: false,
requireSink: true,
};
const config: ApophisConfig = {
mode: 'observe',
profile: 'staging-observe',
profiles: { 'staging-observe': profile },
presets: { 'platform-observe': preset },
environments: {
staging: envStaging,
production: envProduction,
},
};
const readmeContent = `
# APOPHIS Setup — platform-observe preset
This project was scaffolded with \`apophis init --preset platform-observe\`.
## Quick Start
1. Ensure you have a Fastify app with @fastify/swagger registered.
2. Configure your reporting sink (see environments.staging.requireSink).
3. Run: apophis observe --profile staging-observe
## What This Preset Does
- Enables observe mode for production readiness checks.
- Validates non-blocking semantics and sink configuration.
- Parallel execution for faster feedback.
- Requires sink config in staging/production.
## Safety
- Observe mode is non-blocking by default.
- Production requires explicit policy to enable blocking.
- Chaos is disabled in this preset.
## Next Steps
- Add a sink configuration to your environment policy.
- Run \`apophis doctor\` to validate the full setup.
`;
return { config, readmeContent };
}
// ─────────────────────────────────────────────────────────────────────────────
// llm-safe: Minimal preset for LLM-generated codebases
// ─────────────────────────────────────────────────────────────────────────────
export function llmSafeScaffold(): ScaffoldResult {
const preset: PresetDefinition = {
name: 'llm-safe',
depth: 'quick',
timeout: 3000,
parallel: false,
chaos: false,
observe: false,
};
const profile: ProfileDefinition = {
name: 'llm-check',
mode: 'verify',
preset: 'llm-safe',
routes: [],
};
const envLocal: EnvironmentPolicy = {
name: 'local',
allowVerify: true,
allowObserve: false,
allowQualify: false,
allowChaos: false,
allowBlocking: false,
requireSink: false,
};
const config: ApophisConfig = {
mode: 'verify',
profile: 'llm-check',
profiles: { 'llm-check': profile },
presets: { 'llm-safe': preset },
environments: { local: envLocal },
};
const readmeContent = `
# APOPHIS Setup — llm-safe preset
This project was scaffolded with \`apophis init --preset llm-safe\`.
## Quick Start
1. Ensure you have a Fastify app with @fastify/swagger registered.
2. Add behavioral contracts to your route schemas using \`x-ensures\`.
3. Run: apophis verify --profile llm-check
## What This Preset Does
- Ultra-minimal preset for LLM-generated codebases.
- 3s timeout per route (fast feedback).
- No observe, no qualify, no chaos — verify only.
- Conservative defaults to avoid surprising failures.
## Example Behavioral Contract
Add this inside your route schema to check that a created resource is retrievable:
\`\`\`javascript
"x-ensures": [
"response_code(GET /users/{response_body(this).id}) == 200"
]
\`\`\`
If \`apophis verify\` says "No behavioral contracts found", it means your routes have schemas but no \`x-ensures\` or \`x-requires\` clauses. Add at least one clause per route you want to verify.
## Next Steps
- Add routes to the \`routes\` array once you have behavioral contracts.
- Run \`apophis doctor\` to check for missing dependencies.
`;
return { config, readmeContent };
}
// ─────────────────────────────────────────────────────────────────────────────
// protocol-lab: Multi-step flow and stateful testing
// ─────────────────────────────────────────────────────────────────────────────
export function protocolLabScaffold(): ScaffoldResult {
const preset: PresetDefinition = {
name: 'protocol-lab',
depth: 'deep',
timeout: 15000,
parallel: false,
chaos: true,
observe: false,
};
const profile: ProfileDefinition = {
name: 'oauth-nightly',
mode: 'qualify',
preset: 'protocol-lab',
routes: [],
seed: 42,
};
const envLocal: EnvironmentPolicy = {
name: 'local',
allowVerify: true,
allowObserve: true,
allowQualify: true,
allowChaos: true,
allowBlocking: true,
requireSink: false,
};
const envTest: EnvironmentPolicy = {
name: 'test',
allowVerify: true,
allowObserve: true,
allowQualify: true,
allowChaos: true,
allowBlocking: true,
requireSink: false,
};
const config: ApophisConfig = {
mode: 'qualify',
profile: 'oauth-nightly',
profiles: { 'oauth-nightly': profile },
presets: { 'protocol-lab': preset },
environments: {
local: envLocal,
test: envTest,
},
};
const readmeContent = `
# APOPHIS Setup — protocol-lab preset
This project was scaffolded with \`apophis init --preset protocol-lab\`.
## Quick Start
1. Ensure you have a Fastify app with @fastify/swagger registered.
2. Define multi-step flows in your route schemas.
3. Run: apophis qualify --profile oauth-nightly --seed 42
## What This Preset Does
- Enables qualify mode for stateful and scenario testing.
- Chaos engineering enabled (local/test only).
- Deep depth for thorough exploration.
- 15s timeout per route.
## Safety
- Chaos is blocked in production by default.
- Use \`apophis doctor\` to validate environment safety before qualifying.
## Machine Output in CI
Qualify can produce large output. In CI, use machine-readable formats and filter events:
- \`--format json\` emits a single stable JSON artifact (good for small-to-medium runs).
- \`--format ndjson\` emits one event per line (good for streaming parsers).
- Use \`--quiet\` to suppress human progress text.
- Pipe ndjson to \`jq\` or a custom filter to extract only failures:
\`\`\`bash
apophis qualify --profile oauth-nightly --format ndjson | jq 'select(.type == "route.failed")'
\`\`\`
- For very large runs, consider writing artifacts to a directory and parsing the JSON file instead of stdout:
\`\`\`bash
apophis qualify --profile oauth-nightly --format json --artifact-dir reports/apophis
\`\`\`
## Next Steps
- Define scenario sequences in your config.
- Add route allowlists for chaos if needed.
- Run \`apophis replay --artifact <path>\` to debug failures.
`;
return { config, readmeContent };
}
// ─────────────────────────────────────────────────────────────────────────────
// Preset registry
// ─────────────────────────────────────────────────────────────────────────────
export const PRESETS: Record<string, () => ScaffoldResult> = {
'safe-ci': safeCiScaffold,
'platform-observe': platformObserveScaffold,
'llm-safe': llmSafeScaffold,
'protocol-lab': protocolLabScaffold,
};
export function getPresetNames(): string[] {
return Object.keys(PRESETS);
}
export function getScaffoldForPreset(preset: string): ScaffoldResult | null {
const fn = PRESETS[preset];
return fn ? fn() : null;
}
+610
View File
@@ -0,0 +1,610 @@
/**
* S9: Migrate thread - Config migration command
*
* Responsibilities:
* - Detect legacy config patterns and deprecated API usage
* - Support --check (detect only, don't write)
* - Support --dry-run (show rewrites without writing) - DEFAULT
* - Support --write (perform rewrites)
* - Map legacy fields to new fields with exact replacements
* - Preserve comments/formatting where feasible
* - Handle ambiguous rewrites (stop, require manual choice)
* - Report completed and remaining items separately
* - Exit 0 if nothing to migrate, 2 if issues found, 1 if --write performed
* - Mixed legacy/modern config detection with clear reporting
* - Exact dry-run output with file path, line number, legacy text, replacement text
* - Ambiguous rewrite handling with surrounding context and possible resolutions
* - Safe by default: dry-run is the default mode
*
* Architecture:
* - Dependency injection: all dependencies passed explicitly
* - No optional imports — everything is required or injected
* - Inline comments for documentation
*/
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
import { resolve } from 'node:path';
import type { CliContext } from '../../core/context.js';
import { loadConfig, discoverConfig } from '../../core/config-loader.js';
import { SUCCESS, USAGE_ERROR, BEHAVIORAL_FAILURE } from '../../core/exit-codes.js';
import type { CommandResult } from '../../core/types.js';
import {
rewriteConfigFile,
detectLegacyConfigFields,
detectLegacyFieldsNoEquivalent,
detectMixedLegacyModernFields,
} from './rewriters/config-rewriter.js';
import {
rewriteRouteAnnotations,
detectLegacyRouteAnnotations,
detectAmbiguousRoutePatterns,
} from './rewriters/route-rewriter.js';
import {
rewriteCodePatterns,
detectLegacyCodePatterns,
detectAmbiguousCodePatterns,
} from './rewriters/code-rewriter.js';
import { renderJson } from '../../renderers/json.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface MigrateOptions {
check?: boolean;
dryRun?: boolean;
write?: boolean;
config?: string;
cwd?: string;
format?: 'human' | 'json' | 'ndjson';
quiet?: boolean;
verbose?: boolean;
}
export interface MigrationItem {
type: 'config-field' | 'route-annotation' | 'code-pattern';
file: string;
line?: number;
legacy: string;
replacement: string;
guidance?: string;
ambiguous?: boolean;
}
export interface MigrationResult {
exitCode: number;
items: MigrationItem[];
completed: MigrationItem[];
remaining: MigrationItem[];
message?: string;
filesModified?: string[];
filesWouldBeModified?: string[];
totalRewrites?: number;
manualChoicesRequired?: number;
}
// ---------------------------------------------------------------------------
// File discovery
// ---------------------------------------------------------------------------
/**
* Discover files that may contain legacy patterns.
* Scans the working directory for config files, route files, and code files.
*/
export async function discoverMigrationFiles(
cwd: string,
configPath?: string,
): Promise<{ configFile: string | null; appFiles: string[] }> {
const configFile = configPath
? resolve(cwd, configPath)
: discoverConfig(cwd);
const appFiles: string[] = [];
const candidates = [
'app.js',
'app.ts',
'src/app.js',
'src/app.ts',
'routes.js',
'routes.ts',
'src/routes.js',
'src/routes.ts',
];
for (const candidate of candidates) {
const fullPath = resolve(cwd, candidate);
if (existsSync(fullPath)) {
appFiles.push(fullPath);
}
}
return { configFile, appFiles };
}
// ---------------------------------------------------------------------------
// Pattern detection
// ---------------------------------------------------------------------------
/**
* Detect all legacy patterns in a set of files.
* Includes: legacy fields, route annotations, code patterns,
* ambiguous patterns, fields with no equivalent, and mixed legacy/modern fields.
*/
export async function detectAllLegacyPatterns(
configFile: string | null,
appFiles: string[],
): Promise<MigrationItem[]> {
const items: MigrationItem[] = [];
// Detect config fields
if (configFile && existsSync(configFile)) {
const configContent = readFileSync(configFile, 'utf-8');
items.push(...detectLegacyConfigFields(configContent, configFile));
items.push(...detectLegacyFieldsNoEquivalent(configContent, configFile));
items.push(...detectLegacyRouteAnnotations(configContent, configFile));
items.push(...detectAmbiguousRoutePatterns(configContent, configFile));
items.push(...detectLegacyCodePatterns(configContent, configFile));
items.push(...detectAmbiguousCodePatterns(configContent, configFile));
}
// Detect patterns in app files
for (const appFile of appFiles) {
const content = readFileSync(appFile, 'utf-8');
items.push(...detectLegacyRouteAnnotations(content, appFile));
items.push(...detectAmbiguousRoutePatterns(content, appFile));
items.push(...detectLegacyCodePatterns(content, appFile));
items.push(...detectAmbiguousCodePatterns(content, appFile));
}
return items;
}
// ---------------------------------------------------------------------------
// Migration execution
// ---------------------------------------------------------------------------
/**
* Run the migration process.
*
* Flow:
* 1. Discover config files in the working directory
* 2. Detect legacy patterns in all relevant files
* 3. If --check, report findings and exit
* 4. If --dry-run (default), show exact rewrites without writing
* 5. If --write, perform rewrites
* 6. Report completed and remaining items separately
* 7. Return appropriate exit code
*
* Safety: dry-run is the default mode. No files are modified unless --write is explicitly passed.
*/
export async function migrateCommand(
options: MigrateOptions,
ctx: CliContext,
): Promise<MigrationResult> {
const { check, dryRun, write, config: configPath, cwd } = options;
const workingDir = cwd || ctx.cwd;
// Determine mode: check < dry-run < write
// Default is dry-run (safe by default)
const mode = write ? 'write' : check ? 'check' : 'dry-run';
try {
// 1. Discover files
const { configFile, appFiles } = await discoverMigrationFiles(
workingDir,
configPath,
);
if (!configFile && appFiles.length === 0) {
return {
exitCode: USAGE_ERROR,
items: [],
completed: [],
remaining: [],
message: 'No config or app files found. Run "apophis init" to create one.',
};
}
// 2. Detect legacy patterns
const allItems = await detectAllLegacyPatterns(configFile, appFiles);
// 3. If no legacy patterns found, report success
if (allItems.length === 0) {
return {
exitCode: SUCCESS,
items: [],
completed: [],
remaining: [],
message: 'No legacy patterns detected. Config is up to date.',
};
}
// Separate ambiguous items
const ambiguousItems = allItems.filter((item) => item.ambiguous);
const unambiguousItems = allItems.filter((item) => !item.ambiguous);
// Calculate files that would be modified
const filesWouldBeModified = new Set<string>();
for (const item of allItems) {
filesWouldBeModified.add(item.file);
}
// If ambiguous items exist and we're writing, stop and require manual choice
if (ambiguousItems.length > 0 && mode === 'write') {
return {
exitCode: USAGE_ERROR,
items: allItems,
completed: [],
remaining: ambiguousItems,
message: formatAmbiguousOutput(ambiguousItems),
filesWouldBeModified: Array.from(filesWouldBeModified),
totalRewrites: allItems.length,
manualChoicesRequired: ambiguousItems.length,
};
}
// 4. Check mode: detect only
if (mode === 'check') {
return {
exitCode: BEHAVIORAL_FAILURE,
items: allItems,
completed: [],
remaining: allItems,
message: formatCheckOutput(allItems),
filesWouldBeModified: Array.from(filesWouldBeModified),
totalRewrites: allItems.length,
manualChoicesRequired: ambiguousItems.length,
};
}
// 5. Dry-run mode: show exact rewrites without writing
if (mode === 'dry-run') {
return {
exitCode: BEHAVIORAL_FAILURE,
items: allItems,
completed: [],
remaining: allItems,
message: formatDryRunOutput(allItems),
filesWouldBeModified: Array.from(filesWouldBeModified),
totalRewrites: allItems.length,
manualChoicesRequired: ambiguousItems.length,
};
}
// 6. Write mode: perform rewrites
const filesModified: string[] = [];
const completed: MigrationItem[] = [];
const remaining: MigrationItem[] = [];
// Rewrite config file
if (configFile && existsSync(configFile)) {
const configItems = unambiguousItems.filter(
(item) => item.file === configFile && item.type === 'config-field',
);
if (configItems.length > 0) {
const result = rewriteConfigFile(configFile, configItems);
if (result.modified) {
writeFileSync(configFile, result.content, 'utf-8');
filesModified.push(configFile);
completed.push(...result.itemsRewritten);
remaining.push(...result.itemsRemaining);
} else {
remaining.push(...configItems);
}
}
// Route annotations in config file
const routeItems = unambiguousItems.filter(
(item) => item.file === configFile && item.type === 'route-annotation',
);
if (routeItems.length > 0) {
const result = rewriteRouteAnnotations(configFile, routeItems);
if (result.modified) {
writeFileSync(configFile, result.content, 'utf-8');
if (!filesModified.includes(configFile)) {
filesModified.push(configFile);
}
completed.push(...result.itemsRewritten);
remaining.push(...result.itemsRemaining);
} else {
remaining.push(...routeItems);
}
}
// Code patterns in config file
const codeItems = unambiguousItems.filter(
(item) => item.file === configFile && item.type === 'code-pattern',
);
if (codeItems.length > 0) {
const result = rewriteCodePatterns(configFile, codeItems);
if (result.modified) {
writeFileSync(configFile, result.content, 'utf-8');
if (!filesModified.includes(configFile)) {
filesModified.push(configFile);
}
completed.push(...result.itemsRewritten);
remaining.push(...result.itemsRemaining);
} else {
remaining.push(...codeItems);
}
}
}
// Rewrite app files
for (const appFile of appFiles) {
const fileItems = unambiguousItems.filter((item) => item.file === appFile);
const routeItems = fileItems.filter(
(item) => item.type === 'route-annotation',
);
const codeItems = fileItems.filter(
(item) => item.type === 'code-pattern',
);
let fileModified = false;
let currentContent = readFileSync(appFile, 'utf-8');
if (routeItems.length > 0) {
const result = rewriteRouteAnnotations(appFile, routeItems);
if (result.modified) {
currentContent = result.content;
fileModified = true;
completed.push(...result.itemsRewritten);
remaining.push(...result.itemsRemaining);
} else {
remaining.push(...routeItems);
}
}
if (codeItems.length > 0) {
const result = rewriteCodePatterns(appFile, codeItems);
if (result.modified) {
currentContent = result.content;
fileModified = true;
completed.push(...result.itemsRewritten);
remaining.push(...result.itemsRemaining);
} else {
remaining.push(...codeItems);
}
}
if (fileModified) {
writeFileSync(appFile, currentContent, 'utf-8');
filesModified.push(appFile);
}
}
// Ambiguous items always remain
remaining.push(...ambiguousItems);
return {
exitCode: completed.length > 0 ? BEHAVIORAL_FAILURE : SUCCESS,
items: allItems,
completed,
remaining,
message: formatWriteOutput(completed, remaining),
filesModified,
filesWouldBeModified: Array.from(filesWouldBeModified),
totalRewrites: allItems.length,
manualChoicesRequired: ambiguousItems.length,
};
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return {
exitCode: USAGE_ERROR,
items: [],
completed: [],
remaining: [],
message: `Migration failed: ${message}`,
};
}
}
// ---------------------------------------------------------------------------
// Output formatting
// ---------------------------------------------------------------------------
function formatCheckOutput(items: MigrationItem[]): string {
const lines: string[] = [];
lines.push('Legacy config patterns detected:');
lines.push('');
for (const item of items) {
const location = item.line ? `${item.file}:${item.line}` : item.file;
lines.push(` ${location}`);
lines.push(` Legacy: ${item.legacy}`);
lines.push(` Replace: ${item.replacement}`);
if (item.guidance) {
lines.push(` Guidance: ${item.guidance}`);
}
if (item.ambiguous) {
lines.push(` ⚠ Ambiguous — requires manual choice`);
}
lines.push('');
}
lines.push(`Found ${items.length} item(s) to migrate.`);
lines.push('');
lines.push('Run "apophis migrate --dry-run" to preview rewrites.');
lines.push('Run "apophis migrate --write" to apply rewrites.');
return lines.join('\n');
}
function formatDryRunOutput(items: MigrationItem[]): string {
const lines: string[] = [];
const files = new Set<string>();
const ambiguousCount = items.filter((item) => item.ambiguous).length;
lines.push('Dry run — the following rewrites would be applied:');
lines.push('');
for (const item of items) {
const location = item.line ? `${item.file}:${item.line}` : item.file;
files.add(item.file);
lines.push(` ${location}`);
lines.push(` - ${item.legacy}`);
lines.push(` + ${item.replacement}`);
if (item.guidance) {
lines.push(` # ${item.guidance}`);
}
if (item.ambiguous) {
lines.push(` ⚠ Skipped (ambiguous — requires manual choice)`);
}
lines.push('');
}
lines.push(`Total: ${items.length} item(s) to migrate.`);
lines.push(`Files that would be modified: ${files.size}`);
if (ambiguousCount > 0) {
lines.push(`Items requiring manual choice: ${ambiguousCount}`);
}
lines.push('');
lines.push('Run "apophis migrate --write" to apply these rewrites.');
return lines.join('\n');
}
function formatWriteOutput(
completed: MigrationItem[],
remaining: MigrationItem[],
): string {
const lines: string[] = [];
lines.push('Migration complete:');
lines.push('');
if (completed.length > 0) {
lines.push(` Completed (${completed.length}):`);
for (const item of completed) {
const location = item.line
? `${item.file}:${item.line}`
: item.file;
lines.push(
`${location}${item.legacy}${item.replacement}`,
);
}
lines.push('');
}
if (remaining.length > 0) {
lines.push(` Remaining (${remaining.length}):`);
for (const item of remaining) {
const location = item.line
? `${item.file}:${item.line}`
: item.file;
lines.push(` - ${location}${item.legacy}`);
if (item.ambiguous) {
lines.push(` ⚠ Ambiguous — requires manual choice`);
} else if (item.guidance) {
lines.push(` # ${item.guidance}`);
}
}
lines.push('');
}
if (remaining.length === 0) {
lines.push('All items migrated successfully.');
} else {
lines.push(`Run "apophis migrate --check" to review remaining items.`);
}
return lines.join('\n');
}
function formatAmbiguousOutput(items: MigrationItem[]): string {
const lines: string[] = [];
lines.push('Ambiguous rewrites detected — migration stopped:');
lines.push('');
for (const item of items) {
const location = item.line ? `${item.file}:${item.line}` : item.file;
lines.push(` ${location}`);
lines.push(` ${item.legacy}`);
lines.push(` ⚠ This pattern is ambiguous and requires manual choice.`);
if (item.guidance) {
lines.push(` Consider: ${item.guidance}`);
}
lines.push('');
}
lines.push('Please resolve these items manually, then re-run migrate.');
return lines.join('\n');
}
// ---------------------------------------------------------------------------
// CLI adapter
// ---------------------------------------------------------------------------
/**
* Adapter that bridges the CLI framework (cac) to the migrate command handler.
* This function signature matches what the CLI core expects.
*
* Safety: dry-run is the default mode. No files are modified unless --write is explicitly passed.
*/
export async function handleMigrate(
_args: string[],
ctx: CliContext,
): Promise<number> {
const options: MigrateOptions = {
config: ctx.options.config || undefined,
cwd: ctx.cwd,
format: ctx.options.format as MigrateOptions['format'],
quiet: ctx.options.quiet,
verbose: ctx.options.verbose,
};
// Parse command-specific flags from process.argv
const argv = process.argv.slice(2);
if (argv.includes('--check')) {
options.check = true;
}
if (argv.includes('--dry-run')) {
options.dryRun = true;
}
if (argv.includes('--write')) {
options.write = true;
}
const result = await migrateCommand(options, ctx);
// Output result based on format
if (!ctx.options.quiet && result.message) {
const format = options.format || ctx.options.format || 'human';
if (format === 'json') {
console.log(renderJson({
exitCode: result.exitCode,
items: result.items,
completed: result.completed,
remaining: result.remaining,
filesModified: result.filesModified,
filesWouldBeModified: result.filesWouldBeModified,
totalRewrites: result.totalRewrites,
manualChoicesRequired: result.manualChoicesRequired,
}));
} else if (format === 'ndjson') {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'migrate',
exitCode: result.exitCode,
items: result.items,
completed: result.completed,
remaining: result.remaining,
filesModified: result.filesModified,
filesWouldBeModified: result.filesWouldBeModified,
totalRewrites: result.totalRewrites,
manualChoicesRequired: result.manualChoicesRequired,
}) + '\n');
} else {
console.log(result.message);
}
}
return result.exitCode;
}
@@ -0,0 +1,257 @@
/**
* Code rewriter for APOPHIS migrate command.
*
* Responsibilities:
* - Rewrite legacy JS/TS plugin code patterns
* - contract() → verify({ kind: 'contract' })
* - stateful() → qualify({ kind: 'stateful' })
* - scenario() → qualify({ kind: 'scenario' })
* - Handle ambiguous patterns (stop, require manual choice)
* - Preserve code formatting and comments
* - Show surrounding context for ambiguous patterns
*
* Architecture:
* - Dependency injection: all dependencies passed explicitly
* - No optional imports
* - Inline comments for documentation
*/
import { readFileSync, writeFileSync } from 'node:fs';
import type { MigrationItem } from '../index.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface CodeRewriteResult {
content: string;
modified: boolean;
itemsRewritten: MigrationItem[];
itemsRemaining: MigrationItem[];
}
export interface AmbiguousCodePattern {
pattern: string;
line: number;
context: string[];
possibleResolutions: string[];
}
// ---------------------------------------------------------------------------
// Legacy code pattern mappings
// ---------------------------------------------------------------------------
/**
* Mapping of deprecated code patterns to their modern equivalents.
*
* Some patterns are marked as ambiguous because the semantic intent
* may not be clear from syntax alone (e.g., contract() could mean
* different things in different contexts).
*/
export const LEGACY_CODE_PATTERNS: Record<
string,
{ replacement: string; ambiguous?: boolean }
> = {
'contract()': { replacement: "verify({ kind: 'contract' })", ambiguous: false },
'stateful()': { replacement: "qualify({ kind: 'stateful' })", ambiguous: false },
'scenario()': { replacement: "qualify({ kind: 'scenario' })", ambiguous: false },
};
/**
* Ambiguous code patterns that require manual choice.
* These patterns could mean different things depending on context.
*/
export const AMBIGUOUS_CODE_PATTERNS: Record<
string,
{ possibleResolutions: string[]; guidance: string }
> = {
'oldApi()': {
possibleResolutions: [
"verify({ kind: 'contract' }) — if this is a contract test",
"qualify({ kind: 'stateful' }) — if this is a stateful test",
"Remove the call — if this is dead code",
],
guidance:
'The oldApi() pattern is ambiguous. It could be a contract test, stateful test, or dead code. Review the surrounding context to determine the correct replacement.',
},
'legacyPlugin()': {
possibleResolutions: [
"app.register(newPlugin()) — if migrating to a new plugin",
"Remove the call — if the plugin is no longer needed",
"// TODO: migrate plugin — if manual migration is required",
],
guidance:
'The legacyPlugin() pattern is ambiguous. Determine if the plugin has a modern equivalent or should be removed.',
},
};
// ---------------------------------------------------------------------------
// Core rewriting logic
// ---------------------------------------------------------------------------
/**
* Rewrite legacy code patterns in a JS/TS file.
*
* Strategy:
* 1. Read the raw file content
* 2. For each legacy pattern, replace occurrences
* 3. Skip ambiguous patterns unless explicitly allowed
* 4. Preserve formatting by only replacing the pattern text
* 5. Track which items were rewritten and which remain
*/
export function rewriteCodePatterns(
filePath: string,
items: MigrationItem[],
allowAmbiguous: boolean = false,
): CodeRewriteResult {
const content = readFileSync(filePath, 'utf-8');
let modifiedContent = content;
let modified = false;
const itemsRewritten: MigrationItem[] = [];
const itemsRemaining: MigrationItem[] = [];
for (const item of items) {
if (item.type !== 'code-pattern') {
itemsRemaining.push(item);
continue;
}
// Skip ambiguous items unless explicitly allowed
if (item.ambiguous && !allowAmbiguous) {
itemsRemaining.push(item);
continue;
}
const legacy = item.legacy;
const replacement = item.replacement;
// Match the exact pattern (e.g., contract())
// Need to escape the parentheses in the pattern
// Note: word boundary \b doesn't work after (), so we use a different approach
const escapedLegacy = escapeRegex(legacy);
const regex = new RegExp(
`(^|[^a-zA-Z0-9_])${escapedLegacy}($|[^a-zA-Z0-9_])`,
'g',
);
const newContent = modifiedContent.replace(
regex,
(match, prefix, suffix) => {
return prefix + replacement + suffix;
},
);
if (newContent !== modifiedContent) {
modifiedContent = newContent;
modified = true;
itemsRewritten.push(item);
} else {
itemsRemaining.push(item);
}
}
return {
content: modifiedContent,
modified,
itemsRewritten,
itemsRemaining,
};
}
/**
* Write the rewritten code file to disk.
*/
export function writeRewrittenCode(filePath: string, content: string): void {
writeFileSync(filePath, content, 'utf-8');
}
/**
* Detect legacy code patterns in raw text content.
* Returns migration items for each occurrence.
*/
export function detectLegacyCodePatterns(
content: string,
filePath: string,
): MigrationItem[] {
const items: MigrationItem[] = [];
const lines = content.split('\n');
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line === undefined) continue;
for (const [legacy, mapping] of Object.entries(LEGACY_CODE_PATTERNS)) {
// Match the pattern as a standalone call
// Escape parentheses in the legacy pattern
// Note: word boundary \b doesn't work after (), so we use a different approach
const escapedLegacy = escapeRegex(legacy);
const regex = new RegExp(
`(^|[^a-zA-Z0-9_])${escapedLegacy}($|[^a-zA-Z0-9_])`,
);
if (regex.test(line)) {
items.push({
type: 'code-pattern',
file: filePath,
line: i + 1,
legacy,
replacement: mapping.replacement,
guidance: `Replace '${legacy}' with '${mapping.replacement}'`,
ambiguous: mapping.ambiguous,
});
}
}
}
return items;
}
/**
* Detect ambiguous code patterns that require manual choice.
* Returns ambiguous patterns with surrounding context for human review.
*/
export function detectAmbiguousCodePatterns(
content: string,
filePath: string,
): MigrationItem[] {
const items: MigrationItem[] = [];
const lines = content.split('\n');
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line === undefined) continue;
for (const [pattern, info] of Object.entries(AMBIGUOUS_CODE_PATTERNS)) {
const escapedPattern = escapeRegex(pattern);
const regex = new RegExp(
`(^|[^a-zA-Z0-9_])${escapedPattern}($|[^a-zA-Z0-9_])`,
);
if (regex.test(line)) {
// Capture surrounding context (2 lines before and after)
const contextStart = Math.max(0, i - 2);
const contextEnd = Math.min(lines.length, i + 3);
const context = lines.slice(contextStart, contextEnd);
items.push({
type: 'code-pattern',
file: filePath,
line: i + 1,
legacy: pattern,
replacement: '(ambiguous — see guidance)',
guidance: `${info.guidance}\nPossible resolutions:\n${info.possibleResolutions.map((r) => ` - ${r}`).join('\n')}\n\nContext:\n${context.map((l, idx) => ` ${contextStart + idx + 1}: ${l}`).join('\n')}`,
ambiguous: true,
});
}
}
}
return items;
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function escapeRegex(str: string): string {
return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
@@ -0,0 +1,268 @@
/**
* Config rewriter for APOPHIS migrate command.
*
* Responsibilities:
* - Rewrite config files, replacing legacy fields with modern equivalents
* - Preserve comments and formatting where feasible
* - Handle nested object rewrites
* - Report what was changed and what remains
* - Detect mixed legacy/modern configs and report clearly
* - Emit human guidance for legacy fields with no direct equivalent
*
* Architecture:
* - Dependency injection: all dependencies passed explicitly
* - No optional imports
* - Inline comments for documentation
*/
import { readFileSync, writeFileSync } from 'node:fs';
import type { MigrationItem } from '../index.js';
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function escapeRegex(str: string): string {
return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface ConfigRewriteResult {
content: string;
modified: boolean;
itemsRewritten: MigrationItem[];
itemsRemaining: MigrationItem[];
}
export interface MixedFieldReport {
legacy: string;
modern: string;
line: number;
guidance: string;
}
// ---------------------------------------------------------------------------
// Legacy field mappings
// ---------------------------------------------------------------------------
/**
* Mapping of deprecated config fields to their modern equivalents.
*/
export const LEGACY_CONFIG_MAPPINGS: Record<string, string> = {
// Top-level fields
testMode: 'mode',
// Profile container
testProfiles: 'profiles',
// Profile fields
usesPreset: 'preset',
routeFilter: 'routes',
// Preset container
testPresets: 'presets',
// Preset fields
testDepth: 'depth',
maxDuration: 'timeout',
// Environment container
envPolicies: 'environments',
// Environment fields
canVerify: 'allowVerify',
};
/**
* Legacy fields with no direct equivalent — emit human guidance instead of auto-rewrite.
*/
export const LEGACY_FIELDS_NO_EQUIVALENT: Record<string, { guidance: string; severity: 'warning' | 'error' }> = {
legacyField: {
guidance: 'This field has no modern equivalent. Remove it and review your config manually.',
severity: 'warning',
},
oldApiVersion: {
guidance: 'API versioning is now handled via profiles. Remove this field and set version in each profile.',
severity: 'warning',
},
deprecatedPlugin: {
guidance: 'This plugin is no longer supported. Remove the field and migrate to the new plugin system.',
severity: 'error',
},
};
// ---------------------------------------------------------------------------
// Core rewriting logic
// ---------------------------------------------------------------------------
/**
* Rewrite a config file, replacing legacy field names with modern equivalents.
*
* Strategy:
* 1. Read the raw file content
* 2. For each legacy field mapping, replace occurrences as property keys
* 3. Preserve formatting by only replacing the key name, not surrounding whitespace
* 4. Track which items were rewritten and which remain
*/
export function rewriteConfigFile(
filePath: string,
items: MigrationItem[],
): ConfigRewriteResult {
const content = readFileSync(filePath, 'utf-8');
let modifiedContent = content;
let modified = false;
const itemsRewritten: MigrationItem[] = [];
const itemsRemaining: MigrationItem[] = [];
for (const item of items) {
if (item.type !== 'config-field') {
itemsRemaining.push(item);
continue;
}
// The legacy field name (might be a nested path like "testProfiles.quick")
const legacyKey = item.legacy.split('.').pop() || item.legacy;
const replacement = item.replacement;
// Build a regex that matches the field as a property key
// This handles: key:, "key":, 'key':, key :, etc.
const regex = new RegExp(
`([\\s{,\\[])(['"]?)(${escapeRegex(legacyKey)})\\2\\s*:(?!\\/)`,
'g',
);
const newContent = modifiedContent.replace(regex, (match, prefix, quote, _key) => {
return `${prefix}${quote}${replacement}${quote}:`;
});
if (newContent !== modifiedContent) {
modifiedContent = newContent;
modified = true;
itemsRewritten.push(item);
} else {
itemsRemaining.push(item);
}
}
return {
content: modifiedContent,
modified,
itemsRewritten,
itemsRemaining,
};
}
/**
* Write the rewritten config to disk.
*/
export function writeRewrittenConfig(filePath: string, content: string): void {
writeFileSync(filePath, content, 'utf-8');
}
/**
* Detect legacy config fields in raw text content.
* Returns migration items for each occurrence.
*/
export function detectLegacyConfigFields(
content: string,
filePath: string,
): MigrationItem[] {
const items: MigrationItem[] = [];
const lines = content.split('\n');
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line === undefined) continue;
for (const [legacy, replacement] of Object.entries(LEGACY_CONFIG_MAPPINGS)) {
// Match the field as a property key, avoiding matches inside strings/comments
const regex = new RegExp(`\\b${escapeRegex(legacy)}\\s*:`);
if (regex.test(line)) {
items.push({
type: 'config-field',
file: filePath,
line: i + 1,
legacy,
replacement,
guidance: `Replace '${legacy}' with '${replacement}'`,
});
}
}
}
return items;
}
/**
* Detect legacy fields that have no direct modern equivalent.
* These emit human guidance instead of being auto-rewritten.
*/
export function detectLegacyFieldsNoEquivalent(
content: string,
filePath: string,
): MigrationItem[] {
const items: MigrationItem[] = [];
const lines = content.split('\n');
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line === undefined) continue;
for (const [legacy, info] of Object.entries(LEGACY_FIELDS_NO_EQUIVALENT)) {
const regex = new RegExp(`\\b${escapeRegex(legacy)}\\s*:`);
if (regex.test(line)) {
items.push({
type: 'config-field',
file: filePath,
line: i + 1,
legacy,
replacement: '(removed — see guidance)',
guidance: info.guidance,
});
}
}
}
return items;
}
/**
* Detect mixed legacy and modern config fields.
* When both legacy and modern versions of the same field exist, report each clearly.
*/
export function detectMixedLegacyModernFields(
content: string,
filePath: string,
): MixedFieldReport[] {
const reports: MixedFieldReport[] = [];
const lines = content.split('\n');
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line === undefined) continue;
for (const [legacy, modern] of Object.entries(LEGACY_CONFIG_MAPPINGS)) {
// Check if this line contains the legacy field
const legacyRegex = new RegExp(`\\b${escapeRegex(legacy)}\\s*:`);
if (legacyRegex.test(line)) {
// Check if the modern equivalent also exists somewhere in the file
const modernRegex = new RegExp(`\\b${escapeRegex(modern)}\\s*:`);
if (modernRegex.test(content)) {
reports.push({
legacy,
modern,
line: i + 1,
guidance: `Both '${legacy}' (legacy) and '${modern}' (modern) found. Remove '${legacy}' to avoid conflicts.`,
});
}
}
}
}
return reports;
}
@@ -0,0 +1,216 @@
/**
* Route rewriter for APOPHIS migrate command.
*
* Responsibilities:
* - Rewrite route schema annotations (e.g., x-validate-runtime → runtime)
* - Preserve schema structure and formatting
* - Handle annotations in Fastify route definitions
* - Detect ambiguous annotations and require manual choice
*
* Architecture:
* - Dependency injection: all dependencies passed explicitly
* - No optional imports
* - Inline comments for documentation
*/
import { readFileSync, writeFileSync } from 'node:fs';
import type { MigrationItem } from '../index.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface RouteRewriteResult {
content: string;
modified: boolean;
itemsRewritten: MigrationItem[];
itemsRemaining: MigrationItem[];
}
export interface AmbiguousRoutePattern {
pattern: string;
line: number;
context: string[];
possibleResolutions: string[];
}
// ---------------------------------------------------------------------------
// Legacy annotation mappings
// ---------------------------------------------------------------------------
/**
* Mapping of deprecated route schema annotations to their modern equivalents.
*/
export const LEGACY_ROUTE_ANNOTATIONS: Record<string, string> = {
'x-validate-runtime': 'runtime',
};
/**
* Ambiguous route patterns that require manual choice.
* These patterns could mean different things depending on context.
*/
export const AMBIGUOUS_ROUTE_PATTERNS: Record<string, { possibleResolutions: string[]; guidance: string }> = {
'x-validate': {
possibleResolutions: [
"'runtime' — validate at runtime",
"'build' — validate at build time",
"'both' — validate at both times",
],
guidance: 'The x-validate annotation is ambiguous. Choose the validation timing explicitly.',
},
'x-check': {
possibleResolutions: [
"'runtime' — runtime check",
"'contract' — contract check",
"'schema' — schema-only check",
],
guidance: 'The x-check annotation is ambiguous. Choose the check type explicitly.',
},
};
// ---------------------------------------------------------------------------
// Core rewriting logic
// ---------------------------------------------------------------------------
/**
* Rewrite route annotations in a file.
*
* Strategy:
* 1. Read the raw file content
* 2. For each legacy annotation, replace occurrences in string literals
* 3. Preserve formatting by only replacing the annotation name
* 4. Track which items were rewritten and which remain
*/
export function rewriteRouteAnnotations(
filePath: string,
items: MigrationItem[],
): RouteRewriteResult {
const content = readFileSync(filePath, 'utf-8');
let modifiedContent = content;
let modified = false;
const itemsRewritten: MigrationItem[] = [];
const itemsRemaining: MigrationItem[] = [];
for (const item of items) {
if (item.type !== 'route-annotation') {
itemsRemaining.push(item);
continue;
}
const legacy = item.legacy;
const replacement = item.replacement;
// Match the annotation in string literals (single or double quotes)
// The legacy string might have hyphens, so we need to be careful with word boundaries
const regex = new RegExp(
`(['"])${escapeRegex(legacy)}(['"])`,
'g',
);
const newContent = modifiedContent.replace(regex, `$1${replacement}$2`);
if (newContent !== modifiedContent) {
modifiedContent = newContent;
modified = true;
itemsRewritten.push(item);
} else {
itemsRemaining.push(item);
}
}
return {
content: modifiedContent,
modified,
itemsRewritten,
itemsRemaining,
};
}
/**
* Write the rewritten route file to disk.
*/
export function writeRewrittenRoutes(filePath: string, content: string): void {
writeFileSync(filePath, content, 'utf-8');
}
/**
* Detect legacy route annotations in raw text content.
* Returns migration items for each occurrence.
*/
export function detectLegacyRouteAnnotations(
content: string,
filePath: string,
): MigrationItem[] {
const items: MigrationItem[] = [];
const lines = content.split('\n');
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line === undefined) continue;
for (const [legacy, replacement] of Object.entries(LEGACY_ROUTE_ANNOTATIONS)) {
// Match the annotation in string literals
const regex = new RegExp(`['"]${escapeRegex(legacy)}['"]`);
if (regex.test(line)) {
items.push({
type: 'route-annotation',
file: filePath,
line: i + 1,
legacy,
replacement,
guidance: `Replace '${legacy}' with '${replacement}' in route schema`,
});
}
}
}
return items;
}
/**
* Detect ambiguous route patterns that require manual choice.
* Returns ambiguous patterns with surrounding context for human review.
*/
export function detectAmbiguousRoutePatterns(
content: string,
filePath: string,
): MigrationItem[] {
const items: MigrationItem[] = [];
const lines = content.split('\n');
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line === undefined) continue;
for (const [pattern, info] of Object.entries(AMBIGUOUS_ROUTE_PATTERNS)) {
const regex = new RegExp(`['"]${escapeRegex(pattern)}['"]`);
if (regex.test(line)) {
// Capture surrounding context (2 lines before and after)
const contextStart = Math.max(0, i - 2);
const contextEnd = Math.min(lines.length, i + 3);
const context = lines.slice(contextStart, contextEnd);
items.push({
type: 'route-annotation',
file: filePath,
line: i + 1,
legacy: pattern,
replacement: '(ambiguous — see guidance)',
guidance: `${info.guidance}\nPossible resolutions:\n${info.possibleResolutions.map(r => ` - ${r}`).join('\n')}`,
ambiguous: true,
});
}
}
}
return items;
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function escapeRegex(str: string): string {
return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$\u0026');
}
+328
View File
@@ -0,0 +1,328 @@
/**
* S5: Observe thread - Observe command handler
*
* Responsibilities:
* - Load config and resolve profile
* - Validate observe configuration
* - Check reporting sink setup (logs, metrics, traces)
* - Validate non-blocking semantics
* - Environment safety checks (block blocking behavior in prod by default)
* - Support --check-config (validate only, don't activate)
* - Explain what would be checked and why it is safe
* - Clear output about safety boundaries
* - Exit 0 on valid config, 2 on safety violation
*/
import type { CliContext } from '../../core/context.js';
import { loadConfig } from '../../core/config-loader.js';
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js';
import { SUCCESS, USAGE_ERROR } from '../../core/exit-codes.js';
import { validateObserveConfig } from './validator.js';
import { renderDoctorChecks } from '../../renderers/human.js';
import { renderJson } from '../../renderers/json.js';
import type { OutputContext } from '../../renderers/shared.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface ObserveOptions {
profile?: string;
checkConfig?: boolean;
config?: string;
cwd?: string;
format?: 'human' | 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary';
quiet?: boolean;
verbose?: boolean;
}
export interface ObserveResult {
exitCode: number;
message?: string;
checks?: Array<{
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
}>;
}
// ---------------------------------------------------------------------------
// Command handler
// ---------------------------------------------------------------------------
/**
* Main observe command handler.
*
* Flow:
* 1. Load and resolve config
* 2. Run policy engine checks
* 3. Validate observe-specific configuration
* 4. If --check-config, stop after validation
* 5. Otherwise, report what would be activated and why it is safe
* 6. Return appropriate exit code
*/
export async function observeCommand(
options: ObserveOptions,
ctx: CliContext,
): Promise<ObserveResult> {
const { profile, checkConfig, config: configPath, cwd } = options;
const workingDir = cwd || ctx.cwd;
// Detect environment from context
const env = detectEnvironment();
try {
// 1. Load config
const loadResult = await loadConfig({
cwd: workingDir,
configPath,
profileName: profile,
env,
});
if (!loadResult.configPath) {
return {
exitCode: USAGE_ERROR,
message: 'No config found. Run "apophis init" to create one.',
};
}
const config = loadResult.config;
// 2. Run policy engine checks
const policyEngine = new PolicyEngine({
config,
env,
mode: 'observe',
profileName: profile || undefined,
presetName: loadResult.presetName || undefined,
});
const policyResult = policyEngine.check();
if (!policyResult.allowed) {
const message = [
'Policy check failed:',
...policyResult.errors.map(e => `${e}`),
].join('\n');
return {
exitCode: USAGE_ERROR,
message,
};
}
// 3. Validate observe-specific configuration
const validationResult = validateObserveConfig(config, profile || undefined, env);
if (!validationResult.valid) {
const message = formatValidationOutput(validationResult, { checkConfig, env, profile });
return {
exitCode: USAGE_ERROR,
message,
checks: validationResult.checks,
};
}
// 4. If --check-config, stop after validation with success
if (checkConfig) {
const message = formatValidationOutput(validationResult, {
checkConfig: true,
env,
profile,
});
return {
exitCode: SUCCESS,
message,
checks: validationResult.checks,
};
}
// 5. Report what would be activated and why it is safe
const activationMessage = formatActivationOutput(validationResult, {
env,
profile,
configPath: loadResult.configPath,
});
return {
exitCode: SUCCESS,
message: activationMessage,
checks: validationResult.checks,
};
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return {
exitCode: USAGE_ERROR,
message: `Failed to run observe command: ${message}`,
checks: [],
};
}
}
// ---------------------------------------------------------------------------
// Output formatting
// ---------------------------------------------------------------------------
interface FormatOptions {
checkConfig?: boolean;
env: string;
profile?: string;
configPath?: string;
}
/**
* Format validation results for human-readable output.
*/
function formatValidationOutput(
result: import('./validator.js').ObserveValidationResult,
options: FormatOptions,
): string {
const lines: string[] = [];
const mode = options.checkConfig ? 'Config validation' : 'Observe validation';
lines.push(`${mode} for environment "${options.env}"`);
if (options.profile) {
lines.push(`Profile: ${options.profile}`);
}
lines.push('');
// Print each check
for (const check of result.checks) {
const icon = check.status === 'pass' ? '✓' : check.status === 'warn' ? '⚠' : '✗';
lines.push(` ${icon} ${check.name}: ${check.message}`);
if (check.detail) {
lines.push(` ${check.detail}`);
}
}
lines.push('');
// Summary
if (result.errors.length > 0) {
lines.push(`Failed with ${result.errors.length} error(s).`);
lines.push('');
lines.push('Safety boundaries:');
lines.push(' - Observe mode is non-blocking by default');
lines.push(' - Blocking behavior is prohibited in production');
lines.push(' - Qualify-only features (chaos, stateful, etc.) are not allowed');
lines.push(' - Sampling rate must be between 0.0 and 1.0');
lines.push(' - Sinks must be configured when required by environment policy');
} else if (result.warnings.length > 0) {
lines.push(`Passed with ${result.warnings.length} warning(s).`);
} else {
lines.push('All checks passed.');
}
return lines.join('\n');
}
/**
* Format activation output explaining what would be checked and why it is safe.
*/
function formatActivationOutput(
result: import('./validator.js').ObserveValidationResult,
options: FormatOptions,
): string {
const lines: string[] = [];
lines.push(`Observe mode ready for environment "${options.env}"`);
if (options.profile) {
lines.push(`Profile: ${options.profile}`);
}
if (options.configPath) {
lines.push(`Config: ${options.configPath}`);
}
lines.push('');
// Print checks
for (const check of result.checks) {
const icon = check.status === 'pass' ? '✓' : check.status === 'warn' ? '⚠' : '✗';
lines.push(` ${icon} ${check.name}: ${check.message}`);
}
lines.push('');
lines.push('What would be checked:');
lines.push(' - Request/response contracts are evaluated asynchronously');
lines.push(' - Violations are logged to configured sinks without blocking');
lines.push(' - Sampling controls the fraction of requests observed');
lines.push(' - Metrics and traces provide runtime visibility into contract health');
lines.push('');
lines.push('Why this is safe:');
lines.push(' - Non-blocking semantics guarantee observation does not affect latency');
lines.push(' - No chaos injection or stateful sequences are activated in observe mode');
lines.push(' - Production environments require explicit non-blocking configuration');
lines.push(' - All qualify-only features are blocked by validation');
if (result.warnings.length > 0) {
lines.push('');
lines.push('Warnings:');
for (const warning of result.warnings) {
lines.push(`${warning}`);
}
}
lines.push('');
lines.push('To activate observation, run without --check-config.');
return lines.join('\n');
}
// ---------------------------------------------------------------------------
// CLI adapter
// ---------------------------------------------------------------------------
/**
* Adapter that bridges the CLI framework (cac) to the observe command handler.
* This function signature matches what the CLI core expects.
*/
export async function handleObserve(
_args: string[],
ctx: CliContext,
): Promise<number> {
const options: ObserveOptions = {
profile: ctx.options.profile || undefined,
checkConfig: false,
config: ctx.options.config || undefined,
cwd: ctx.cwd,
format: ctx.options.format as ObserveOptions['format'],
quiet: ctx.options.quiet,
verbose: ctx.options.verbose,
};
// Parse command-specific flags from process.argv
// cac passes these as parsed options, but we need to extract --check-config
// Since cac doesn't expose parsed command-specific flags in the options object,
// we scan process.argv directly for observe-specific flags
const argv = process.argv.slice(2);
if (argv.includes('--check-config')) {
options.checkConfig = true;
}
const result = await observeCommand(options, ctx);
// Output result based on format
if (!ctx.options.quiet && result.message) {
const format = options.format || ctx.options.format || 'human';
if (format === 'json') {
console.log(renderJson({
exitCode: result.exitCode,
checks: result.checks,
message: result.message,
}));
} else if (format === 'ndjson') {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'observe',
exitCode: result.exitCode,
checks: result.checks,
message: result.message,
}) + '\n');
} else {
console.log(result.message);
}
}
return result.exitCode;
}
+539
View File
@@ -0,0 +1,539 @@
/**
* S5: Observe thread - Observe config validation logic
*
* Validates observe-specific configuration including:
* - Sink configuration checks (logs, metrics, traces)
* - Sampling rate validation
* - Feature restriction checks (no qualify-only features in observe)
* - Non-blocking semantics validation
*/
import type { Config, ProfileDefinition, PresetDefinition } from '../../core/config-loader.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface ObserveValidationResult {
valid: boolean;
errors: string[];
warnings: string[];
checks: ObserveCheck[];
}
export interface ObserveCheck {
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
}
export interface SinkConfig {
logs?: boolean;
metrics?: boolean;
traces?: boolean;
endpoint?: string;
}
export interface ObserveProfileConfig {
sampling?: number;
blocking?: boolean;
sinks?: SinkConfig;
features?: string[];
}
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
/** Features that are only valid in qualify mode */
const QUALIFY_ONLY_FEATURES = new Set([
'chaos',
'stateful',
'scenario',
'outbound-mocks',
'protocol-flow',
]);
/** Valid sampling rate bounds */
const SAMPLING_MIN = 0.0;
const SAMPLING_MAX = 1.0;
// ---------------------------------------------------------------------------
// Validation functions
// ---------------------------------------------------------------------------
/**
* Validate observe configuration for a given profile and environment.
*/
export function validateObserveConfig(
config: Config,
profileName: string | undefined,
env: string,
): ObserveValidationResult {
const errors: string[] = [];
const warnings: string[] = [];
const checks: ObserveCheck[] = [];
// Resolve the effective profile config (preset + profile overrides)
const profileConfig = resolveObserveProfileConfig(config, profileName);
// 1. Check profile exists and is observe mode
const profileCheck = validateProfileMode(config, profileName);
checks.push(profileCheck);
if (profileCheck.status === 'fail') {
errors.push(profileCheck.message);
}
// 2. Check for qualify-only features (uses resolved profile config)
const featureCheck = validateFeatures(profileConfig.features, profileName);
checks.push(featureCheck);
if (featureCheck.status === 'fail') {
errors.push(featureCheck.message);
}
// 3. Validate sampling rate (uses resolved profile config)
const samplingCheck = validateSamplingRate(profileConfig.sampling);
checks.push(samplingCheck);
if (samplingCheck.status === 'fail') {
errors.push(samplingCheck.message);
}
// 4. Validate sink configuration (uses resolved profile config)
const sinkCheck = validateSinkConfig(profileConfig.sinks, env, config);
checks.push(sinkCheck);
if (sinkCheck.status === 'fail') {
errors.push(sinkCheck.message);
} else if (sinkCheck.status === 'warn') {
warnings.push(sinkCheck.message);
}
// 5. Validate non-blocking semantics (uses resolved profile config)
const blockingCheck = validateBlockingSemantics(profileConfig.blocking, env, config);
checks.push(blockingCheck);
if (blockingCheck.status === 'fail') {
errors.push(blockingCheck.message);
}
// 6. Environment policy check: must explicitly allow observe
const envPolicyCheck = validateEnvironmentPolicy(config, env);
checks.push(envPolicyCheck);
if (envPolicyCheck.status === 'fail') {
errors.push(envPolicyCheck.message);
}
// 7. Environment safety check
const envCheck = validateEnvironmentSafety(env, profileConfig);
checks.push(envCheck);
if (envCheck.status === 'warn') {
warnings.push(envCheck.message);
}
// 8. Profile must be configured for observe mode
const profileObserveCheck = validateProfileObserveMode(config, profileName);
checks.push(profileObserveCheck);
if (profileObserveCheck.status === 'fail') {
errors.push(profileObserveCheck.message);
}
return {
valid: errors.length === 0,
errors,
warnings,
checks,
};
}
/**
* Resolve the observe-specific configuration from profile and preset.
* Preset values are applied first, then profile overrides.
*/
function resolveObserveProfileConfig(
config: Config,
profileName: string | undefined,
): ObserveProfileConfig {
const result: ObserveProfileConfig = {};
if (!profileName || !config.profiles) {
return result;
}
const profile = config.profiles[profileName];
if (!profile) {
return result;
}
// Apply preset first if referenced
if (profile.preset && config.presets) {
const preset = config.presets[profile.preset];
if (preset) {
Object.assign(result, presetToObserveConfig(preset));
}
}
// Apply profile overrides
Object.assign(result, profileToObserveConfig(profile));
return result;
}
/**
* Convert preset definition to observe config.
*/
function presetToObserveConfig(preset: PresetDefinition): ObserveProfileConfig {
return {
features: preset.features,
sampling: (preset as Record<string, unknown>).sampling as number | undefined,
blocking: (preset as Record<string, unknown>).blocking as boolean | undefined,
sinks: (preset as Record<string, unknown>).sinks as SinkConfig | undefined,
};
}
/**
* Convert profile definition to observe config.
*/
function profileToObserveConfig(profile: ProfileDefinition): ObserveProfileConfig {
return {
features: profile.features,
sampling: (profile as Record<string, unknown>).sampling as number | undefined,
blocking: (profile as Record<string, unknown>).blocking as boolean | undefined,
sinks: (profile as Record<string, unknown>).sinks as SinkConfig | undefined,
};
}
/**
* Validate that the profile exists.
* Note: mode validation is handled by validateProfileObserveMode.
*/
function validateProfileMode(
config: Config,
profileName: string | undefined,
): ObserveCheck {
if (!profileName) {
return {
name: 'profile-mode',
status: 'pass',
message: 'No profile specified, using default observe configuration',
};
}
if (!config.profiles || !config.profiles[profileName]) {
const available = config.profiles ? Object.keys(config.profiles).join(', ') : 'none';
return {
name: 'profile-mode',
status: 'fail',
message: `Profile "${profileName}" not found. Available profiles: ${available}`,
};
}
return {
name: 'profile-mode',
status: 'pass',
message: `Profile "${profileName}" exists`,
};
}
/**
* Validate that the profile is explicitly configured for observe mode.
*/
function validateProfileObserveMode(
config: Config,
profileName: string | undefined,
): ObserveCheck {
if (!profileName) {
return {
name: 'profile-observe-mode',
status: 'pass',
message: 'No profile specified, mode will be determined by top-level config',
};
}
if (!config.profiles || !config.profiles[profileName]) {
return {
name: 'profile-observe-mode',
status: 'pass',
message: `Profile "${profileName}" not found — will be validated by profile-mode check`,
};
}
const profile = config.profiles[profileName];
const profileMode = profile.mode;
if (profileMode && profileMode !== 'observe') {
return {
name: 'profile-observe-mode',
status: 'fail',
message: `Profile "${profileName}" is configured for "${profileMode}" mode but observe command requires "observe" mode`,
detail: 'Change the profile mode to "observe" or use the appropriate command ' +
`for "${profileMode}" mode (e.g., apophis ${profileMode}).`,
};
}
return {
name: 'profile-observe-mode',
status: 'pass',
message: `Profile "${profileName}" is configured for observe mode`,
};
}
/**
* Validate that no qualify-only features are used in observe mode.
*/
function validateFeatures(
features: string[] | undefined,
profileName: string | undefined,
): ObserveCheck {
if (!features || features.length === 0) {
return {
name: 'feature-restrictions',
status: 'pass',
message: 'No features configured',
};
}
const invalidFeatures = features.filter(f => QUALIFY_ONLY_FEATURES.has(f));
if (invalidFeatures.length > 0) {
const profileRef = profileName ? `Profile "${profileName}"` : 'Configuration';
return {
name: 'feature-restrictions',
status: 'fail',
message: `${profileRef} references qualify-only features that cannot be used in observe mode: ${invalidFeatures.join(', ')}`,
detail: `Remove these features from the profile or preset. Qualify-only features: ${Array.from(QUALIFY_ONLY_FEATURES).join(', ')}`,
};
}
return {
name: 'feature-restrictions',
status: 'pass',
message: `All features are valid for observe mode: ${features.join(', ')}`,
};
}
/**
* Validate sampling rate is within valid bounds [0.0, 1.0].
*/
export function validateSamplingRate(sampling: number | undefined): ObserveCheck {
if (sampling === undefined || sampling === null) {
return {
name: 'sampling-rate',
status: 'pass',
message: 'No sampling rate configured, using default (1.0)',
};
}
if (typeof sampling !== 'number' || Number.isNaN(sampling)) {
return {
name: 'sampling-rate',
status: 'fail',
message: `Sampling rate must be a number, got ${typeof sampling}`,
detail: `Valid range: ${SAMPLING_MIN} to ${SAMPLING_MAX} (inclusive)`,
};
}
if (sampling < SAMPLING_MIN || sampling > SAMPLING_MAX) {
return {
name: 'sampling-rate',
status: 'fail',
message: `Sampling rate ${sampling} is out of bounds`,
detail: `Set sampling to a value between ${SAMPLING_MIN} and ${SAMPLING_MAX} (inclusive). ` +
`A rate of 0.0 disables observation, 1.0 observes all requests.`,
};
}
return {
name: 'sampling-rate',
status: 'pass',
message: `Sampling rate ${sampling} is valid`,
};
}
/**
* Validate sink configuration for the environment.
*/
function validateSinkConfig(
sinks: SinkConfig | undefined,
env: string,
config: Config,
): ObserveCheck {
// Check if environment requires sinks
const envPolicy = config.environments?.[env];
const requireSink = envPolicy?.requireSink ?? false;
if (!sinks || Object.keys(sinks).length === 0) {
if (requireSink) {
return {
name: 'sink-config',
status: 'fail',
message: `Environment "${env}" requires sink configuration but none is provided`,
detail: 'Add sinks to your profile (e.g., sinks: { logs: true }) ' +
'or set requireSink: false in the environment policy.',
};
}
return {
name: 'sink-config',
status: 'warn',
message: 'No sinks configured. Observation data will not be persisted.',
detail: 'Configure at least one sink (logs, metrics, or traces) ' +
'to capture observation data for analysis.',
};
}
const activeSinks = [];
if (sinks.logs) activeSinks.push('logs');
if (sinks.metrics) activeSinks.push('metrics');
if (sinks.traces) activeSinks.push('traces');
if (activeSinks.length === 0) {
return {
name: 'sink-config',
status: 'warn',
message: 'Sinks are configured but none are enabled. Observation data will not be persisted.',
detail: 'Set at least one of logs, metrics, or traces to true in your sink configuration.',
};
}
return {
name: 'sink-config',
status: 'pass',
message: `Active sinks: ${activeSinks.join(', ')}`,
};
}
/**
* Validate non-blocking semantics for the environment.
* Blocking is NEVER allowed in production unless explicitly enabled by a break-glass policy.
*/
function validateBlockingSemantics(
blocking: boolean | undefined,
env: string,
config: Config,
): ObserveCheck {
const isProd = env === 'production' || env === 'prod';
if (blocking === true && isProd) {
// Check for break-glass policy override
const envPolicy = config.environments?.[env];
const allowBlocking = envPolicy?.allowBlocking ?? false;
if (!allowBlocking) {
return {
name: 'blocking-semantics',
status: 'fail',
message: `Blocking behavior is not allowed in production environment "${env}"`,
detail: 'Set blocking: false in your profile, use a non-production environment, ' +
'or set allowBlocking: true in the environment policy for break-glass scenarios.',
};
}
return {
name: 'blocking-semantics',
status: 'pass',
message: `Blocking behavior is enabled in production "${env}" via break-glass policy`,
detail: 'WARNING: blocking observation can severely impact request latency. ' +
'This should only be used during active incident response.',
};
}
if (blocking === true) {
return {
name: 'blocking-semantics',
status: 'pass',
message: `Blocking behavior is enabled in non-production environment "${env}"`,
detail: 'Warning: blocking observation can increase request latency. ' +
'Only enable in environments where latency impact is acceptable.',
};
}
// blocking is false or undefined (default to non-blocking)
return {
name: 'blocking-semantics',
status: 'pass',
message: `Non-blocking semantics confirmed for environment "${env}"`,
detail: 'Observation will run asynchronously without blocking request handling.',
};
}
/**
* Validate environment policy explicitly allows observe mode.
*/
function validateEnvironmentPolicy(
config: Config,
env: string,
): ObserveCheck {
const envPolicy = config.environments?.[env];
if (!envPolicy) {
// No explicit policy for this environment — warn but don't fail
return {
name: 'environment-policy',
status: 'pass',
message: `No environment policy defined for "${env}"`,
detail: 'Observe mode is allowed by default when no policy is configured.',
};
}
const allowObserve = envPolicy.allowObserve;
if (allowObserve === false) {
return {
name: 'environment-policy',
status: 'fail',
message: `Environment policy for "${env}" explicitly blocks observe mode`,
detail: 'Set allowObserve: true in the environment policy to enable observe mode, ' +
'or run in an environment where observe is allowed.',
};
}
return {
name: 'environment-policy',
status: 'pass',
message: `Environment "${env}" explicitly allows observe mode`,
};
}
/**
* Validate environment-specific safety constraints.
*/
function validateEnvironmentSafety(
env: string,
profileConfig: ObserveProfileConfig,
): ObserveCheck {
const isProd = env === 'production' || env === 'prod';
if (isProd) {
const warnings = [];
if (profileConfig.sampling === undefined) {
warnings.push('sampling rate not configured (will use default 1.0)');
}
if (!profileConfig.sinks) {
warnings.push('no sinks configured');
}
if (warnings.length > 0) {
return {
name: 'environment-safety',
status: 'warn',
message: `Production environment "${env}" observe configuration has warnings: ${warnings.join(', ')}`,
detail: 'In production, configure explicit sampling rate and sinks ' +
'to control observation overhead and ensure data capture.',
};
}
}
return {
name: 'environment-safety',
status: 'pass',
message: `Environment "${env}" safety checks passed`,
};
}
// ---------------------------------------------------------------------------
// Exports for testing
// ---------------------------------------------------------------------------
export {
QUALIFY_ONLY_FEATURES,
SAMPLING_MIN,
SAMPLING_MAX,
};
+148
View File
@@ -0,0 +1,148 @@
/**
* S6: Qualify thread - Chaos execution handler
*
* Responsibilities:
* - Run a single route with chaos injection and collect traces
* - Generate deterministic chaos events for CLI qualify mode
* - Uses chaos-v3 pure functions for deterministic adversity
*
* Architecture:
* - Pure execution function that accepts injected dependencies
* - No optional imports — everything is passed via parameters
*/
import { applyChaosToExecution, createChaosEventArbitrary, formatChaosEvents } from '../../../quality/chaos-v3.js'
import { SeededRng } from '../../../infrastructure/seeded-rng.js'
import type {
RouteContract,
EvalContext,
ChaosConfig,
} from '../../../types.js'
import type { QualifyRunnerDeps, ChaosRunResult } from './runner.js'
/**
* Run a single route with chaos injection and collect traces.
* Uses chaos-v3 pure functions for deterministic adversity.
*/
export async function runChaosOnRoute(
deps: QualifyRunnerDeps,
route: RouteContract,
chaosConfig: ChaosConfig,
): Promise<{ ctx: EvalContext; chaosResult: ChaosRunResult }> {
const started = Date.now()
// Generate chaos events using seeded RNG via fast-check
// For CLI qualify, we use a deterministic subset
const rng = new SeededRng(deps.seed)
const contractNames: string[] = []
// Build a minimal request for the route
const request = {
method: route.method,
url: route.path,
headers: {},
query: undefined as Record<string, string> | undefined,
body: undefined as unknown,
}
// Execute the request
const { executeHttp } = await import('../../../infrastructure/http-executor.js')
const ctx = await executeHttp(deps.fastify, route, request, undefined, deps.timeout)
// Generate and apply chaos events
const chaosArb = createChaosEventArbitrary(chaosConfig, contractNames)
// For deterministic CLI runs, we generate a fixed small set of events
// In practice, fast-check would be used in property tests; here we simulate
const events = generateDeterministicChaosEvents(chaosConfig, deps.seed)
const application = applyChaosToExecution(ctx, events)
const chaosResult: ChaosRunResult = {
applied: application.applied,
events: application.events
.filter(e => e.type !== 'none')
.map(e => formatChaosEvents([e])),
route: `${route.method} ${route.path}`,
durationMs: Date.now() - started,
}
return { ctx: application.ctx, chaosResult }
}
/**
* Generate a deterministic set of chaos events for CLI qualify mode.
* Uses seeded RNG for reproducibility.
*/
export function generateDeterministicChaosEvents(config: ChaosConfig, seed: number): import('../../../quality/chaos-v3.js').ChaosEvent[] {
const rng = new SeededRng(seed)
const events: import('../../../quality/chaos-v3.js').ChaosEvent[] = []
// Only inject chaos if probability threshold is met
if (config.probability <= 0 || rng.next() > config.probability) {
return events
}
// Pick one chaos type deterministically
const types: Array<'delay' | 'error' | 'dropout' | 'corruption'> = []
if (config.delay) types.push('delay')
if (config.error) types.push('error')
if (config.dropout) types.push('dropout')
if (config.corruption) types.push('corruption')
if (types.length === 0) return events
const chosen = types[Math.floor(rng.next() * types.length)]
if (!chosen) return events
switch (chosen) {
case 'delay': {
if (config.delay) {
const minMs = config.delay.minMs
const maxMs = config.delay.maxMs
const delayMs = minMs + Math.floor(rng.next() * (maxMs - minMs + 1))
events.push({
type: 'inbound-delay',
target: 'inbound',
delayMs,
})
}
break
}
case 'error': {
if (config.error) {
events.push({
type: 'inbound-error',
target: 'inbound',
statusCode: config.error.statusCode,
body: config.error.body,
})
}
break
}
case 'dropout': {
if (config.dropout) {
events.push({
type: 'inbound-dropout',
target: 'inbound',
statusCode: config.dropout.statusCode ?? 504,
})
}
break
}
case 'corruption': {
if (config.corruption) {
const strategies = ['truncate', 'malformed', 'field-corrupt'] as const
const strategy = strategies[Math.floor(rng.next() * strategies.length)]
events.push({
type: 'inbound-corruption',
target: 'inbound',
corruptionStrategy: strategy,
corruptionField: strategy === 'field-corrupt' ? 'id' : undefined,
})
}
break
}
}
return events
}
+868
View File
@@ -0,0 +1,868 @@
/**
* S6: Qualify thread - Qualify command handler
*
* Responsibilities:
* - Load config and resolve profile
* - Block prod runs by default (policy engine)
* - Run scenario/stateful/chaos based on profile
* - Generate seed if omitted, always print it
* - Rich artifact emission with step traces
* - Handle cleanup failures separately
* - Exit 0 on pass, 1 on qualification failure, 2 on safety violation
*
* Architecture:
* - Dependency injection: all dependencies passed explicitly
* - No optional imports — everything is required or injected
* - Inline comments for documentation
*/
import type { CliContext } from '../../core/context.js'
import { loadConfig } from '../../core/config-loader.js'
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
import { resolveGenerationProfileOverride, GenerationProfileResolutionError } from '../../core/generation-profile.js'
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
import type { CommandResult, Artifact, FailureRecord } from '../../core/types.js'
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
import {
runQualify,
resolveProfileGates,
type QualifyRunResult,
type StepTrace,
type CleanupFailure,
} from './runner.js'
import { SeededRng } from '../../../infrastructure/seeded-rng.js'
import type { ScenarioConfig, TestConfig, RouteContract, ChaosConfig } from '../../../types.js'
import { renderHumanArtifact } from '../../renderers/human.js'
import { renderJson, renderJsonArtifact, renderJsonSummaryArtifact } from '../../renderers/json.js'
import { renderNdjsonArtifact, renderNdjsonSummaryArtifact } from '../../renderers/ndjson.js'
import type { OutputContext } from '../../renderers/shared.js'
import { resolve } from 'node:path'
import { pathToFileURL } from 'node:url'
const ROUTE_IDENTITY_PATTERN = /^[A-Z]+\s+\/\S*$/
function normalizeRouteIdentity(route: string): string {
const normalized = route.trim().replace(/\s+/g, ' ')
const [method, ...pathParts] = normalized.split(' ')
if (!method || pathParts.length === 0) {
return normalized
}
return `${method.toUpperCase()} ${pathParts.join(' ')}`
}
function isReplayCompatibleRoute(route: string): boolean {
return ROUTE_IDENTITY_PATTERN.test(route)
}
function coerceDepth(value: unknown): TestConfig['depth'] {
if (value === 'quick' || value === 'standard' || value === 'thorough') {
return value
}
return 'standard'
}
function coerceTimeout(value: unknown): number | undefined {
return typeof value === 'number' ? value : undefined
}
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface QualifyOptions {
profile?: string
generationProfile?: string
seed?: number
config?: string
cwd?: string
format?: 'human' | 'json' | 'ndjson'
quiet?: boolean
verbose?: boolean
artifactDir?: string
}
interface FastifyAppLike {
ready?: () => Promise<void>
close?: () => Promise<void>
}
// ---------------------------------------------------------------------------
// Seed generation
// ---------------------------------------------------------------------------
/**
* Generate a deterministic seed if none provided.
* Uses current time + process pid + counter for uniqueness.
*/
let seedCounter = 0
export function generateSeed(): number {
seedCounter++
return Date.now() + (process.pid || 0) + seedCounter
}
// ---------------------------------------------------------------------------
// Route discovery helper
// ---------------------------------------------------------------------------
/**
* Discover routes from the Fastify app for chaos execution.
* Injected fastify instance must have routes registered.
*/
async function discoverAppRoutes(fastify: unknown): Promise<RouteContract[]> {
// Cast to access routes
const app = fastify as { routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> }> }
if (!app.routes) return []
return app.routes.map(r => ({
path: r.url,
method: r.method as RouteContract['method'],
category: 'observer',
requires: [],
ensures: [],
invariants: [],
regexPatterns: {},
validateRuntime: false,
schema: r.schema,
}))
}
// ---------------------------------------------------------------------------
// Scenario builder from profile
// ---------------------------------------------------------------------------
/**
* Build scenario configs from profile routes for protocol-lab fixture.
* Creates an OAuth-like multi-step scenario.
*/
function buildScenarioConfigs(routes: string[], seed: number): ScenarioConfig[] {
// For the protocol-lab fixture, build the OAuth scenario
const hasOAuth = routes.some(r => r.includes('/oauth/authorize'))
if (!hasOAuth) return []
const rng = new SeededRng(seed)
const clientId = `client-${Math.floor(rng.next() * 10000)}`
return [{
name: 'oauth-flow',
steps: [
{
name: 'authorize',
request: {
method: 'POST',
url: '/oauth/authorize',
body: {
client_id: clientId,
redirect_uri: 'http://localhost/callback',
scope: 'read',
},
},
expect: ['status:200', 'response_body(this).code != null'],
capture: { code: 'response_body(this).code' },
},
{
name: 'token',
request: {
method: 'POST',
url: '/oauth/token',
body: {
code: '$authorize.code',
client_id: clientId,
client_secret: 'secret',
redirect_uri: 'http://localhost/callback',
},
},
expect: ['status:200', 'response_body(this).access_token != null'],
capture: { accessToken: 'response_body(this).access_token' },
},
{
name: 'user',
request: {
method: 'GET',
url: '/api/user',
headers: {
authorization: 'Bearer $token.accessToken',
},
},
expect: ['status:200', 'response_body(this).id != null'],
},
],
}]
}
// ---------------------------------------------------------------------------
// Artifact builder
// ---------------------------------------------------------------------------
/**
* Build a rich artifact document from qualify results.
* Includes step traces, cleanup failures, and replay info.
*/
export function buildArtifact(
runResult: QualifyRunResult,
options: {
cwd: string
configPath?: string
profile?: string
preset?: string
env: string
seed: number
},
): Artifact {
const failures: FailureRecord[] = []
const warnings: string[] = []
const replayCompatibleExecutedRoutes = (runResult.executedRoutes || [])
.map(normalizeRouteIdentity)
.filter(isReplayCompatibleRoute)
// Collect scenario failures
for (const scenario of runResult.scenarioResults) {
if (!scenario.ok) {
for (let stepIdx = 0; stepIdx < scenario.steps.length; stepIdx++) {
const step = scenario.steps[stepIdx]!
if (!step.ok && step.diagnostics) {
// Use actual HTTP route from step trace for stable replay identity
const trace = runResult.stepTraces.find(
t => t.name === step.name && t.status === 'failed'
)
const route = normalizeRouteIdentity(trace?.route || `${scenario.name} / ${step.name}`)
if (!isReplayCompatibleRoute(route)) {
warnings.push(`Scenario step "${scenario.name}/${step.name}" did not resolve to METHOD /path route identity.`)
}
failures.push({
route,
contract: step.diagnostics.formula || 'scenario-step',
expected: step.diagnostics.expected || 'success',
observed: step.diagnostics.error || 'failure',
seed: runResult.seed,
replayCommand: `apophis replay --artifact <artifact-path-unavailable>`,
category: step.diagnostics.error ? classifyError(step.diagnostics.error) : ErrorTaxonomy.RUNTIME,
diff: step.diagnostics.diff ?? undefined,
actual: step.diagnostics.actual ?? undefined,
})
}
}
}
}
// Collect stateful failures
if (runResult.statefulResult) {
let fallbackRouteIdx = 0
for (const test of runResult.statefulResult.tests) {
if (!test.ok) {
let route = normalizeRouteIdentity(test.name)
if (!isReplayCompatibleRoute(route)) {
route = replayCompatibleExecutedRoutes[fallbackRouteIdx] || route
fallbackRouteIdx++
}
if (!isReplayCompatibleRoute(route)) {
warnings.push(`Stateful failure "${test.name}" did not resolve to METHOD /path route identity.`)
}
failures.push({
route,
contract: test.diagnostics?.formula || 'stateful-test',
expected: test.diagnostics?.expected || 'success',
observed: test.diagnostics?.error || 'failure',
seed: runResult.seed,
replayCommand: `apophis replay --artifact <artifact-path-unavailable>`,
category: test.diagnostics?.error ? classifyError(test.diagnostics.error) : ErrorTaxonomy.RUNTIME,
diff: test.diagnostics?.diff ?? undefined,
actual: test.diagnostics?.actual ?? undefined,
})
}
}
}
const totalTests =
runResult.scenarioResults.reduce((sum, s) => sum + s.steps.length, 0) +
(runResult.statefulResult?.tests.length ?? 0)
const passedTests =
runResult.scenarioResults.reduce((sum, s) => sum + s.summary.passed, 0) +
(runResult.statefulResult?.summary.passed ?? 0)
if (runResult.cleanupFailures.length > 0) {
warnings.push(
`Cleanup failures: ${runResult.cleanupFailures.map(c => `${c.resource}: ${c.error}`).join(', ')}`
)
}
// Build cleanup outcomes from cleanup failures
const cleanupOutcomes = runResult.cleanupFailures.map(cf => ({
resource: cf.resource,
cleaned: false,
error: cf.error,
}))
// Build execution summary from runner result
const executionSummary = runResult.executionSummary
// Build profile gates from the result context
// We need to pass gates through or infer from results
const profileGates = {
scenario: runResult.scenarioResults.length > 0 || executionSummary.scenariosRun > 0,
stateful: (runResult.statefulResult?.tests.length ?? 0) > 0 || executionSummary.statefulTestsRun > 0,
chaos: (runResult.chaosResult !== undefined) || executionSummary.chaosRunsRun > 0,
}
// Deterministic parameters for audit
const deterministicParams = {
seed: runResult.seed,
profileGates,
}
return {
version: 'apophis-artifact/1',
command: 'qualify',
mode: 'qualify',
cwd: options.cwd,
configPath: options.configPath,
profile: options.profile,
preset: options.preset,
env: options.env,
seed: options.seed,
startedAt: new Date(Date.now() - runResult.durationMs).toISOString(),
durationMs: runResult.durationMs,
summary: {
total: totalTests,
passed: passedTests,
failed: failures.length,
},
executionSummary,
executedRoutes: (runResult.executedRoutes || []).map(normalizeRouteIdentity),
skippedRoutes: (runResult.skippedRoutes || []).map(sr => ({
route: sr.route,
executed: false,
reason: sr.reason,
})),
stepTraces: runResult.stepTraces,
cleanupOutcomes,
profileGates,
deterministicParams,
failures,
artifacts: [],
warnings,
exitReason: runResult.passed ? 'success' : 'behavioral_failure',
}
}
function attachReplayCommands(artifact: Artifact, artifactPath: string): void {
for (const failure of artifact.failures) {
failure.replayCommand = `apophis replay --artifact ${artifactPath}`
}
}
async function emitArtifact(
artifact: Artifact,
options: {
command: 'qualify'
cwd: string
preferredDir?: string
force: boolean
},
): Promise<string | undefined> {
if (!options.force && !options.preferredDir) {
return undefined
}
const defaultDir = resolve(options.cwd, 'reports', 'apophis')
const candidateDirs = [options.preferredDir, defaultDir].filter(Boolean) as string[]
const attempted = new Set<string>()
for (const dir of candidateDirs) {
if (attempted.has(dir)) continue
attempted.add(dir)
try {
const { mkdirSync, writeFileSync } = await import('node:fs')
const artifactPath = resolve(dir, `${options.command}-${new Date().toISOString().replace(/[:.]/g, '-')}.json`)
mkdirSync(dir, { recursive: true })
attachReplayCommands(artifact, artifactPath)
writeFileSync(artifactPath, JSON.stringify(artifact, null, 2))
if (!artifact.artifacts.includes(artifactPath)) {
artifact.artifacts.push(artifactPath)
}
return artifactPath
} catch {
// Try fallback directory if available.
}
}
artifact.warnings.push('Failed to write artifact to disk')
return undefined
}
// ---------------------------------------------------------------------------
// Output formatting
// ---------------------------------------------------------------------------
function formatHumanOutput(
result: QualifyRunResult,
options: { profile?: string; seed: number; env: string },
): string {
const lines: string[] = []
lines.push(`Qualify run for environment "${options.env}"`)
if (options.profile) {
lines.push(`Profile: ${options.profile}`)
}
lines.push(`Seed: ${options.seed}`)
lines.push('')
// Scenario results
for (const scenario of result.scenarioResults) {
lines.push(`Scenario: ${scenario.name}`)
for (const step of scenario.steps) {
const icon = step.ok ? '✓' : '✗'
lines.push(` ${icon} ${step.name} (${step.statusCode ?? 'no-status'})`)
if (!step.ok && step.diagnostics) {
lines.push(` Expected: ${step.diagnostics.expected || 'success'}`)
lines.push(` Observed: ${step.diagnostics.error || 'failure'}`)
if (step.diagnostics.actual) {
lines.push(` Actual: ${step.diagnostics.actual}`)
}
if (step.diagnostics.diff) {
lines.push(` Diff:`)
for (const line of String(step.diagnostics.diff).split('\n')) {
lines.push(` ${line}`)
}
}
}
}
lines.push('')
}
// Stateful results
if (result.statefulResult) {
lines.push(`Stateful: ${result.statefulResult.summary.passed} passed, ${result.statefulResult.summary.failed} failed`)
lines.push('')
}
// Chaos results
if (result.chaosResult) {
lines.push(`Chaos: ${result.chaosResult.applied ? 'applied' : 'none'}`)
if (result.chaosResult.events.length > 0) {
for (const event of result.chaosResult.events) {
lines.push(` ${event}`)
}
}
lines.push('')
}
// Step traces
if (result.stepTraces.length > 0) {
lines.push('Step traces:')
for (const trace of result.stepTraces.slice(0, 20)) {
const icon = trace.status === 'passed' ? '✓' : trace.status === 'skipped' ? '⊘' : '✗'
lines.push(` ${icon} ${trace.name} (${trace.durationMs}ms)`)
}
if (result.stepTraces.length > 20) {
lines.push(` ... and ${result.stepTraces.length - 20} more`)
}
lines.push('')
}
// Cleanup failures
if (result.cleanupFailures.length > 0) {
lines.push('Cleanup failures (reported separately):')
for (const cf of result.cleanupFailures) {
lines.push(`${cf.resource}: ${cf.error}`)
}
lines.push('')
}
// Per-profile gate execution counts
lines.push('Profile gate execution counts:')
lines.push(` Scenario: ${result.executionSummary.scenariosRun} run`)
lines.push(` Stateful: ${result.executionSummary.statefulTestsRun} tests run`)
lines.push(` Chaos: ${result.executionSummary.chaosRunsRun} runs run`)
lines.push('')
// Executed routes
if (result.executedRoutes.length > 0) {
lines.push(`Executed routes (${result.executedRoutes.length}):`)
for (const route of result.executedRoutes) {
lines.push(` ${route}`)
}
lines.push('')
}
// Skipped routes
if (result.skippedRoutes.length > 0) {
lines.push(`Skipped routes (${result.skippedRoutes.length}):`)
for (const sr of result.skippedRoutes) {
lines.push(` ${sr.route}: ${sr.reason}`)
}
lines.push('')
}
// Summary
if (result.passed) {
lines.push('All qualifications passed.')
} else {
lines.push('Qualification failed.')
lines.push(`Replay: apophis replay --artifact <artifact-path>`)
}
return lines.join('\n')
}
// ---------------------------------------------------------------------------
// Main command handler
// ---------------------------------------------------------------------------
/**
* Main qualify command handler.
*
* Flow:
* 1. Load and resolve config
* 2. Run policy engine checks (block prod by default)
* 3. Generate seed if omitted, always print it
* 4. Resolve profile gates (scenario/stateful/chaos)
* 5. Build scenario configs from profile routes
* 6. Run execution modes
* 7. Build rich artifact with step traces
* 8. Handle cleanup failures separately
* 9. Return appropriate exit code
*/
export async function qualifyCommand(
options: QualifyOptions,
ctx: CliContext,
): Promise<CommandResult> {
const {
profile,
generationProfile,
seed: explicitSeed,
config: configPath,
cwd,
artifactDir,
} = options
const workingDir = cwd || ctx.cwd
const format = options.format || ctx.options.format || 'human'
// Detect environment
const env = detectEnvironment()
try {
// 1. Load config
const loadResult = await loadConfig({
cwd: workingDir,
configPath,
profileName: profile,
env,
})
if (!loadResult.configPath) {
return {
exitCode: USAGE_ERROR,
message: 'No config found. Run "apophis init" to create one.',
}
}
const config = loadResult.config
const resolvedGenerationProfile = resolveGenerationProfileOverride(generationProfile, config)
// 2. Run policy engine checks
const policyEngine = new PolicyEngine({
config,
env,
mode: 'qualify',
profileName: profile || undefined,
presetName: loadResult.presetName || undefined,
})
const policyResult = policyEngine.check()
if (!policyResult.allowed) {
const message = [
'Policy check failed:',
...policyResult.errors.map(e => `${e}`),
].join('\n')
return {
exitCode: USAGE_ERROR,
message,
}
}
// 3. Generate seed if omitted
const seed = explicitSeed ?? generateSeed()
if (!ctx.options.quiet && format === 'human') {
console.log(`Seed: ${seed}`)
}
// 4. Resolve profile gates
const profileDef = profile ? config.profiles?.[profile] : undefined
const gates = resolveProfileGates(profileDef?.features)
// 5. Build scenario configs from profile routes
const routes = profileDef?.routes ?? []
const scenarios = buildScenarioConfigs(routes, seed)
// 6. Build stateful config
const presetName = profileDef?.preset
const preset = presetName ? config.presets?.[presetName] : undefined
const presetDepth = coerceDepth((preset as { depth?: unknown } | undefined)?.depth)
const presetTimeout = coerceTimeout((preset as { timeout?: unknown } | undefined)?.timeout)
const statefulConfig: TestConfig | undefined = gates.stateful
? {
depth: presetDepth,
generationProfile: resolvedGenerationProfile,
seed,
timeout: presetTimeout,
routes: profileDef?.routes,
}
: undefined
// 7. Build chaos config
const chaosConfig: ChaosConfig | undefined = gates.chaos && preset?.chaos
? {
probability: 0.5,
delay: { probability: 0.3, minMs: 100, maxMs: 500 },
error: { probability: 0.2, statusCode: 503 },
dropout: { probability: 0.2, statusCode: 504 },
corruption: { probability: 0.1 },
}
: undefined
// 8. Load the Fastify app for execution
// Try to import the app from the fixture
let fastify: FastifyAppLike | undefined
try {
const appPath = resolve(workingDir, 'app.js')
const appUrl = pathToFileURL(appPath)
appUrl.searchParams.set('apophisRun', String(Date.now()))
const appModule = await import(appUrl.href)
fastify = (appModule.default || appModule) as FastifyAppLike
if (fastify && typeof fastify.ready === 'function') {
await fastify.ready()
}
} catch (err) {
// App not available — return a result indicating no app to test
if (process.env.APOPHIS_DEBUG === '1') {
console.error('Failed to load app:', err)
}
return {
exitCode: USAGE_ERROR,
message: 'No Fastify app found. Ensure app.js exports a Fastify instance.',
}
}
try {
// 9. Discover routes for chaos
const appRoutes = await discoverAppRoutes(fastify)
// 10. Run qualify execution
const deps = {
fastify: fastify as any,
seed,
timeout: presetTimeout,
}
const runResult = await runQualify(deps, gates, scenarios, statefulConfig, chaosConfig, appRoutes)
// 11. Build artifact first so we can reference it for guardrails
const artifact = buildArtifact(runResult, {
cwd: workingDir,
configPath: loadResult.configPath,
profile: profile || undefined,
preset: presetName,
env,
seed,
})
// 12. Signal quality guardrails — fail if zero checks executed
const execSummary = runResult.executionSummary
const warnings: string[] = [...artifact.warnings]
if (execSummary.totalExecuted === 0) {
await emitArtifact(artifact, {
command: 'qualify',
cwd: workingDir,
preferredDir: artifactDir || config.artifactDir,
force: true,
})
return {
exitCode: BEHAVIORAL_FAILURE,
message: 'Qualify failed: zero checks executed. No scenarios, stateful tests, or chaos runs were performed. Verify profile gates and app configuration.',
artifact,
warnings: artifact.warnings,
}
}
// Warn if execution counts are suspiciously low
if (gates.scenario && execSummary.scenariosRun === 0) {
warnings.push('WARNING: scenario gate enabled but zero scenarios executed. Check route configuration.')
}
if (gates.stateful && execSummary.statefulTestsRun === 0) {
warnings.push('WARNING: stateful gate enabled but zero stateful tests executed. Check app routes and schema.')
}
if (gates.chaos && execSummary.chaosRunsRun === 0) {
warnings.push('WARNING: chaos gate enabled but zero chaos runs executed. Check chaos config and route availability.')
}
// 12. Write artifact if configured or on failure
const shouldEmitArtifact = Boolean(artifactDir || config.artifactDir || !runResult.passed)
await emitArtifact(artifact, {
command: 'qualify',
cwd: workingDir,
preferredDir: artifactDir || config.artifactDir,
force: shouldEmitArtifact,
})
// 13. Format output based on format option
const outputCtx: OutputContext = {
isTTY: ctx.isTTY,
isCI: ctx.isCI,
colorMode: ctx.options.color,
}
let message = ''
if (!ctx.options.quiet) {
if (format === 'json') {
message = renderJsonArtifact(artifact)
} else if (format === 'json-summary') {
message = renderJsonSummaryArtifact(artifact)
} else if (format === 'ndjson') {
// For ndjson, we don't return a message string; events are streamed
message = ''
} else if (format === 'ndjson-summary') {
// Concise ndjson: only summary events
message = ''
} else {
// human format
message = renderHumanArtifact(artifact, outputCtx)
}
}
return {
exitCode: runResult.passed ? SUCCESS : BEHAVIORAL_FAILURE,
artifact,
message,
warnings: artifact.warnings,
}
} finally {
if (fastify && typeof fastify.close === 'function') {
try {
await fastify.close()
} catch (closeErr) {
if (process.env.APOPHIS_DEBUG === '1') {
console.error('Failed to close Fastify app after qualify run:', closeErr)
}
}
}
}
} catch (error) {
if (error instanceof GenerationProfileResolutionError) {
return {
exitCode: USAGE_ERROR,
message: error.message,
}
}
const message = error instanceof Error ? error.message : String(error)
return {
exitCode: INTERNAL_ERROR,
message: `Internal error in qualify command: ${message}`,
}
}
}
// ---------------------------------------------------------------------------
// CLI adapter
// ---------------------------------------------------------------------------
/**
* Adapter that bridges the CLI framework (cac) to the qualify command handler.
* This function signature matches what the CLI core expects.
*/
export async function handleQualify(
args: string[],
ctx: CliContext,
): Promise<number> {
const options: QualifyOptions = {
profile: ctx.options.profile || undefined,
generationProfile: ctx.options.generationProfile,
seed: undefined,
config: ctx.options.config || undefined,
cwd: ctx.cwd,
format: ctx.options.format as QualifyOptions['format'],
quiet: ctx.options.quiet,
verbose: ctx.options.verbose,
artifactDir: ctx.options.artifactDir || undefined,
}
const seedIdx = args.indexOf('--seed')
if (seedIdx !== -1 && args[seedIdx + 1]) {
const parsed = parseInt(args[seedIdx + 1]!, 10)
if (!isNaN(parsed)) {
options.seed = parsed
}
}
const generationProfileIdx = args.indexOf('--generation-profile')
if (generationProfileIdx !== -1 && args[generationProfileIdx + 1]) {
options.generationProfile = args[generationProfileIdx + 1]
}
const result = await qualifyCommand(options, ctx)
const format = options.format || ctx.options.format || 'human'
const machineMode = format === 'json' || format === 'ndjson' || format === 'json-summary' || format === 'ndjson-summary'
if (!ctx.options.quiet) {
if (format === 'json') {
if (result.artifact) {
console.log(renderJsonArtifact(result.artifact))
} else {
console.log(renderJson({
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}))
}
} else if (format === 'json-summary') {
if (result.artifact) {
console.log(renderJsonSummaryArtifact(result.artifact))
} else {
console.log(renderJson({
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}))
}
} else if (format === 'ndjson') {
if (result.artifact) {
renderNdjsonArtifact(result.artifact)
} else {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'qualify',
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}) + '\n')
}
} else if (format === 'ndjson-summary') {
if (result.artifact) {
renderNdjsonSummaryArtifact(result.artifact)
} else {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'qualify',
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}) + '\n')
}
} else if (result.message) {
console.log(result.message)
}
}
// Print warnings in human mode only
if (!machineMode && result.warnings && result.warnings.length > 0 && !ctx.options.quiet) {
for (const warning of result.warnings) {
console.warn(`Warning: ${warning}`)
}
}
return result.exitCode
}
+255
View File
@@ -0,0 +1,255 @@
/**
* S6: Qualify thread - Runner for scenario, stateful, and chaos execution
*
* Responsibilities:
* - Scenario execution (multi-step flows with capture/rebind)
* - Stateful execution (model-based property testing)
* - Chaos execution (adversity injection via chaos-v3)
* - Profile gating logic (determine which execution modes to run)
* - Step trace collection for rich artifacts
* - Cleanup failure tracking (reported separately)
*
* Architecture:
* - Pure execution functions that accept injected dependencies
* - No optional imports — everything is passed via constructor/parameters
* - Step traces collected as arrays and returned in result
*/
import { runScenarioWithTraces } from './scenario-handler.js'
import { runStatefulWithTraces } from './stateful-handler.js'
import { runChaosOnRoute } from './chaos-handler.js'
import { SeededRng } from '../../../infrastructure/seeded-rng.js'
import type {
ScenarioConfig,
ScenarioResult,
TestConfig,
TestSuite,
RouteContract,
ChaosConfig,
FastifyInjectInstance,
} from '../../../types.js'
import type { ExtensionRegistry } from '../../../extension/types.js'
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface StepTrace {
step: number
name: string
route: string
durationMs: number
status: 'passed' | 'failed' | 'skipped'
error?: string
}
export interface QualifyRunResult {
passed: boolean
scenarioResults: ScenarioResult[]
statefulResult?: TestSuite
chaosResult?: ChaosRunResult
stepTraces: StepTrace[]
cleanupFailures: CleanupFailure[]
durationMs: number
seed: number
executionSummary: {
totalPlanned: number
totalExecuted: number
totalPassed: number
totalFailed: number
scenariosRun: number
statefulTestsRun: number
chaosRunsRun: number
totalSteps: number
}
executedRoutes: string[]
skippedRoutes: { route: string; reason: string }[]
}
export interface ChaosRunResult {
applied: boolean
events: string[]
route: string
durationMs: number
}
export interface CleanupFailure {
resource: string
error: string
}
export interface QualifyRunnerDeps {
fastify: FastifyInjectInstance
extensionRegistry?: ExtensionRegistry
seed: number
timeout?: number
}
// ---------------------------------------------------------------------------
// Profile gating logic
// ---------------------------------------------------------------------------
export interface ProfileGates {
scenario: boolean
stateful: boolean
chaos: boolean
}
/**
* Determine which execution modes to enable based on profile features.
* Default: all enabled if no features specified.
*/
export function resolveProfileGates(features?: string[]): ProfileGates {
if (!features || features.length === 0) {
return { scenario: true, stateful: true, chaos: true }
}
return {
scenario: features.includes('scenario') || features.includes('protocol-flow'),
stateful: features.includes('stateful'),
chaos: features.includes('chaos'),
}
}
// ---------------------------------------------------------------------------
// Main qualify runner
// ---------------------------------------------------------------------------
/**
* Run all qualify execution modes based on profile gates.
* Collects step traces, handles cleanup failures separately.
*/
export async function runQualify(
deps: QualifyRunnerDeps,
gates: ProfileGates,
scenarios: ScenarioConfig[],
statefulConfig?: TestConfig,
chaosConfig?: ChaosConfig,
routes?: RouteContract[],
): Promise<QualifyRunResult> {
const started = Date.now()
const scenarioResults: ScenarioResult[] = []
const allTraces: StepTrace[] = []
const cleanupFailures: CleanupFailure[] = []
let statefulResult: TestSuite | undefined
let chaosResult: ChaosRunResult | undefined
// Run scenarios
if (gates.scenario) {
for (const scenarioConfig of scenarios) {
const { result, traces } = await runScenarioWithTraces(deps, scenarioConfig)
scenarioResults.push(result)
allTraces.push(...traces)
}
}
// Run stateful tests
if (gates.stateful && statefulConfig) {
const { result, traces } = await runStatefulWithTraces(deps, statefulConfig)
statefulResult = result
allTraces.push(...traces)
}
// Run chaos on routes
if (gates.chaos && chaosConfig && routes && routes.length > 0) {
// Pick one route deterministically for CLI chaos demo
const rng = new SeededRng(deps.seed)
const route = routes[Math.floor(rng.next() * routes.length)]
if (route) {
const { chaosResult: cr } = await runChaosOnRoute(deps, route, chaosConfig)
chaosResult = cr
}
}
// Simulate cleanup tracking
// In real usage, cleanupManager would be injected and tracked
// For now, cleanup failures are empty unless injected by caller
const durationMs = Date.now() - started
// Determine overall pass/fail
const scenarioPassed = scenarioResults.every(r => r.ok)
const statefulPassed = !statefulResult || statefulResult.summary.failed === 0
const chaosPassed = !chaosResult || chaosResult.applied // chaos "passes" if it applied
// Count execution metrics
const scenariosRun = scenarioResults.length
const statefulTestsRun = statefulResult?.tests.length ?? 0
const chaosRunsRun = chaosResult ? 1 : 0
const totalSteps = allTraces.length
const totalExecuted = scenariosRun + statefulTestsRun + chaosRunsRun
const totalPassed = scenarioResults.reduce((sum, r) => sum + r.summary.passed, 0) +
(statefulResult?.summary.passed ?? 0) +
(chaosResult?.applied ? 1 : 0)
const totalFailed = scenarioResults.reduce((sum, r) => sum + r.summary.failed, 0) +
(statefulResult?.summary.failed ?? 0)
// Track executed and skipped routes for transparency
const executedRoutes: string[] = []
const skippedRoutes: { route: string; reason: string }[] = []
// Track scenario routes
for (const scenario of scenarioResults) {
for (const step of scenario.steps) {
const trace = allTraces.find(t => t.name === step.name)
if (trace) {
executedRoutes.push(trace.route)
}
}
}
// Track stateful test routes
if (statefulResult) {
for (const test of statefulResult.tests) {
executedRoutes.push(test.name)
}
}
// Track chaos route
if (chaosResult) {
executedRoutes.push(chaosResult.route)
}
// Track skipped routes from profile filters
if (routes) {
const executedSet = new Set(executedRoutes)
for (const route of routes) {
const routeStr = `${route.method} ${route.path}`
if (!executedSet.has(routeStr)) {
let reason = 'Not selected for execution'
if (!gates.scenario && !gates.stateful && !gates.chaos) {
reason = 'All profile gates disabled'
} else if (gates.scenario && !scenarios.some(s => s.steps.some(st => st.request.url === route.path))) {
reason = 'No scenario covers this route'
} else if (gates.stateful && !statefulConfig) {
reason = 'Stateful config missing or invalid'
} else if (gates.chaos && !chaosConfig) {
reason = 'Chaos config missing or invalid'
}
skippedRoutes.push({ route: routeStr, reason })
}
}
}
return {
passed: scenarioPassed && statefulPassed && chaosPassed,
scenarioResults,
statefulResult,
chaosResult,
stepTraces: allTraces,
cleanupFailures,
durationMs,
seed: deps.seed,
executionSummary: {
totalPlanned: scenarios.length + (statefulConfig ? 1 : 0) + (chaosConfig && routes && routes.length > 0 ? 1 : 0),
totalExecuted,
totalPassed,
totalFailed,
scenariosRun,
statefulTestsRun,
chaosRunsRun,
totalSteps,
},
executedRoutes: [...new Set(executedRoutes)],
skippedRoutes,
}
}
@@ -0,0 +1,55 @@
/**
* S6: Qualify thread - Scenario execution handler
*
* Responsibilities:
* - Run scenario configs and collect step traces
* - Wrap the scenario-runner with trace collection
*
* Architecture:
* - Pure execution function that accepts injected dependencies
* - No optional imports — everything is passed via parameters
*/
import { runScenario } from '../../../test/scenario-runner.js'
import type {
ScenarioConfig,
ScenarioResult,
} from '../../../types.js'
import type { QualifyRunnerDeps, StepTrace } from './runner.js'
/**
* Run a scenario config and collect step traces.
* Returns the scenario result plus per-step traces.
*/
export async function runScenarioWithTraces(
deps: QualifyRunnerDeps,
config: ScenarioConfig,
): Promise<{ result: ScenarioResult; traces: StepTrace[] }> {
const scopeHeaders: Record<string, string> = {}
const result = await runScenario(deps.fastify, config, scopeHeaders, deps.extensionRegistry)
const traces: StepTrace[] = result.steps.map((step, idx) => {
const trace: StepTrace = {
step: idx + 1,
name: step.name,
route: `${config.steps[idx]?.request.method ?? 'UNKNOWN'} ${config.steps[idx]?.request.url ?? 'UNKNOWN'}`,
durationMs: 0, // scenario-runner doesn't track per-step timing; use total
status: step.ok ? 'passed' : 'failed',
}
if (!step.ok && step.diagnostics) {
trace.error = typeof step.diagnostics.error === 'string'
? step.diagnostics.error
: JSON.stringify(step.diagnostics.error)
}
return trace
})
// Distribute total time across steps roughly
const perStepMs = result.summary.timeMs / Math.max(result.steps.length, 1)
for (const trace of traces) {
trace.durationMs = perStepMs
}
return { result, traces }
}
@@ -0,0 +1,55 @@
/**
* S6: Qualify thread - Stateful execution handler
*
* Responsibilities:
* - Run stateful tests with the given config
* - Wrap the existing stateful runner with trace collection
*
* Architecture:
* - Pure execution function that accepts injected dependencies
* - No optional imports — everything is passed via parameters
*/
import { runStatefulTests } from '../../../test/stateful-runner.js'
import type {
TestConfig,
TestSuite,
} from '../../../types.js'
import type { QualifyRunnerDeps, StepTrace } from './runner.js'
/**
* Run stateful tests with the given config.
* Wraps the existing stateful runner.
*/
export async function runStatefulWithTraces(
deps: QualifyRunnerDeps,
config: TestConfig,
): Promise<{ result: TestSuite; traces: StepTrace[] }> {
const started = Date.now()
const result = await runStatefulTests(
deps.fastify,
config,
undefined, // cleanupManager — injected if needed by caller
undefined, // scopeRegistry
deps.extensionRegistry,
undefined, // pluginContractRegistry
undefined, // outboundContractRegistry
)
const traces: StepTrace[] = result.tests.map((test, idx) => ({
step: idx + 1,
name: test.name,
route: test.name, // stateful tests name includes route
durationMs: 0,
status: test.ok ? 'passed' : test.directive ? 'skipped' : 'failed',
error: test.diagnostics?.error,
}))
const perStepMs = (Date.now() - started) / Math.max(traces.length, 1)
for (const trace of traces) {
trace.durationMs = perStepMs
}
return { result, traces }
}
+569
View File
@@ -0,0 +1,569 @@
/**
* S7: Replay thread - Replay command handler
*
* Responsibilities:
* - Load artifact from --artifact path
* - Validate artifact schema version
* - Check CLI version compatibility
* - Re-run the failing route/contract with the same seed
* - Handle source code changes since artifact (warn but attempt)
* - Handle missing/corrupted artifacts
* - Handle route no longer existing
* - Fast startup (must feel instant)
* - Exit 0 if replay reproduces same failure, 1 if different, 2 on error
*
* Architecture:
* - Dependency injection: all dependencies passed explicitly
* - No optional imports — everything is required or injected
* - Inline comments for documentation
* - Reuses verify runner for actual replay execution
*/
import type { CliContext } from '../../core/context.js'
import { loadConfig } from '../../core/config-loader.js'
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
import type { CommandResult, Artifact, FailureRecord } from '../../core/types.js'
import { runVerify } from '../verify/runner.js'
import { loadArtifact, type ArtifactLoadResult } from './loader.js'
import { renderJson } from '../../renderers/json.js'
import type { OutputContext } from '../../renderers/shared.js'
import { executeHttp } from '../../../infrastructure/http-executor.js'
import { parse } from '../../../formula/parser.js'
import { evaluateAsync } from '../../../formula/evaluator.js'
import { createOperationResolver } from '../../../formula/runtime.js'
import type { EvalContext, RouteContract } from '../../../types.js'
import { resolve } from 'node:path'
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface ReplayOptions {
artifact: string
config?: string
cwd?: string
format?: 'human' | 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary'
quiet?: boolean
verbose?: boolean
route?: string
}
export interface ReplayResult {
exitCode: number
message?: string
warnings?: string[]
reproduced: boolean
originalFailure?: FailureRecord
newFailure?: FailureRecord
}
// ---------------------------------------------------------------------------
// Human output formatting
// ---------------------------------------------------------------------------
/**
* Format human-readable output for replay results.
*/
function formatHumanOutput(result: ReplayResult, artifact: Artifact): string {
const lines: string[] = []
const sourceDriftDetected = (result.warnings || []).some(w =>
w.includes('Source code has changed since artifact was created') ||
w.includes('modified since artifact was created') ||
w.includes('Artifact cwd no longer exists')
)
if (result.reproduced) {
lines.push('Replay reproduced the original failure.')
lines.push('')
lines.push('Original failure')
lines.push(` Route: ${result.originalFailure?.route}`)
lines.push(` Contract: ${result.originalFailure?.contract}`)
lines.push(` Expected: ${result.originalFailure?.expected}`)
lines.push(` Observed: ${result.originalFailure?.observed}`)
lines.push(` Seed: ${artifact.seed}`)
} else if (result.newFailure) {
lines.push('Replay produced a different result.')
lines.push('')
lines.push('Original failure')
lines.push(` Route: ${result.originalFailure?.route}`)
lines.push(` Contract: ${result.originalFailure?.contract}`)
lines.push('')
lines.push('New result')
lines.push(` Route: ${result.newFailure.route}`)
lines.push(` Contract: ${result.newFailure.contract}`)
lines.push(` Expected: ${result.newFailure.expected}`)
lines.push(` Observed: ${result.newFailure.observed}`)
lines.push(` Seed: ${artifact.seed}`)
} else {
lines.push('Replay passed — failure no longer reproduces.')
lines.push('')
lines.push('Original failure')
lines.push(` Route: ${result.originalFailure?.route}`)
lines.push(` Contract: ${result.originalFailure?.contract}`)
lines.push(` Seed: ${artifact.seed}`)
}
// Add trust labeling and stabilization guidance when replay does not exactly match.
if (!result.reproduced) {
lines.push('')
lines.push('Replay confidence')
if (sourceDriftDetected) {
lines.push(' Degraded: source drift detected since artifact creation; exact reproduction is not guaranteed.')
} else {
lines.push(' Degraded: same-seed replay diverged without source drift; likely runtime/data nondeterminism.')
}
lines.push('')
lines.push('Stabilization guidance:')
lines.push(' 1. Ensure the app database/state is reset to a known baseline')
lines.push(' 2. Run with --seed for explicit control')
lines.push(' 3. Freeze time/randomness in app code and isolate external dependencies')
lines.push(' 4. Disable chaos/stateful gates in profile if not needed for this failure')
}
if (result.warnings && result.warnings.length > 0) {
lines.push('')
lines.push('Warnings')
for (const warning of result.warnings) {
lines.push(`${warning}`)
}
}
return lines.join('\n')
}
// ---------------------------------------------------------------------------
// Direct contract execution (bypasses route discovery)
// ---------------------------------------------------------------------------
/**
* Execute a contract directly against a Fastify instance without route discovery.
* Used by replay when the app doesn't have APOPHIS plugin pre-registered.
*/
async function executeContractDirect(
fastify: any,
route: string,
contract: string,
seed: number,
): Promise<{ success: boolean; observed?: string }> {
// Parse route into method and path
const parts = route.split(' ')
const method = parts[0] || 'GET'
const path = parts.slice(1).join(' ')
// Check if route exists using hasRoute
const hasRoute = typeof fastify.hasRoute === 'function' &&
fastify.hasRoute({ url: path, method })
if (!hasRoute) {
return { success: false, observed: `Route "${route}" no longer exists` }
}
// Build a minimal route contract
const routeContract: RouteContract = {
method: method as RouteContract['method'],
path,
category: 'observer',
schema: {},
requires: [],
ensures: [contract],
invariants: [],
regexPatterns: {},
validateRuntime: false,
}
// Build request
const headers: Record<string, string> = {}
// Execute request
try {
const ctx = await executeHttp(fastify, routeContract, {
method,
url: path,
headers,
query: {},
})
// Build eval context
const evalCtx: EvalContext = {
...ctx,
operationResolver: createOperationResolver(fastify, headers, ctx),
}
// Parse and evaluate contract
const parsed = parse(contract)
const result = await evaluateAsync(parsed.ast, evalCtx)
if (!result.success || !result.value) {
return {
success: false,
observed: result.success ? String(result.value) : result.error,
}
}
return { success: true }
} catch (error) {
return {
success: false,
observed: error instanceof Error ? error.message : String(error),
}
}
}
// ---------------------------------------------------------------------------
// Replay execution
// ---------------------------------------------------------------------------
/**
* Run the replay by re-executing verify with the same seed and route filter.
*
* Flow:
* 1. Load the Fastify app from artifact.cwd
* 2. Run verify with the artifact's seed and route filter
* 3. Compare results to the original failure
* 4. Return whether the failure was reproduced
*/
async function executeReplay(
artifact: Artifact,
failure: FailureRecord,
artifactPath: string,
ctx: CliContext,
options?: { sourceChanged?: boolean },
): Promise<ReplayResult> {
const workingDir = artifact.cwd
const warnings: string[] = []
// Load the Fastify app
let fastify: unknown
try {
const { loadApp } = await import('../../core/app-loader.js')
const loaded = await loadApp(workingDir)
fastify = loaded.fastify
if (fastify && typeof (fastify as any).ready === 'function') {
// Only register APOPHIS plugin if not already registered
// The fixture apps already register it, so re-registering throws
const hasApophis = (fastify as any).apophis !== undefined
const canRegister = typeof (fastify as any).register === 'function'
if (!hasApophis && canRegister) {
const { apophisPlugin } = await import('../../../plugin/index.js')
if (typeof apophisPlugin === 'function') {
await (fastify as any).register(apophisPlugin, { runtime: 'off' })
}
}
await (fastify as any).ready()
}
} catch (err) {
const errorMessage = err instanceof Error ? err.message : String(err)
return {
exitCode: USAGE_ERROR,
message: `Cannot load Fastify app from ${workingDir}/app.js: ${errorMessage}`,
warnings,
reproduced: false,
originalFailure: failure,
}
}
// Try to run verify first (works if app has APOPHIS plugin)
let runResult = await runVerify({
fastify: fastify as any,
seed: artifact.seed || 42,
routeFilters: [failure.route],
})
// If no routes matched, or route found but no contracts (plugin not registered before routes),
// try direct contract execution
if (runResult.noRoutesMatched || runResult.noContractsFound) {
const directResult = await executeContractDirect(
fastify as any,
failure.route,
failure.contract,
artifact.seed || 42,
)
if (!directResult.success) {
// Check if it's a route-not-found error
if (directResult.observed?.includes('no longer exists')) {
return {
exitCode: USAGE_ERROR,
message: `Route "${failure.route}" no longer exists in the application.\n` +
`The source code has drifted since the artifact was created.`,
warnings: [...warnings, `Route "${failure.route}" no longer exists`],
reproduced: false,
originalFailure: failure,
}
}
// Same failure reproduced via direct execution
return {
exitCode: BEHAVIORAL_FAILURE,
message: formatHumanOutput({
exitCode: BEHAVIORAL_FAILURE,
reproduced: true,
originalFailure: failure,
warnings,
}, artifact),
warnings,
reproduced: true,
originalFailure: failure,
}
}
// Direct execution passed — failure no longer reproduces
return {
exitCode: SUCCESS,
message: formatHumanOutput({
exitCode: SUCCESS,
reproduced: false,
originalFailure: failure,
warnings,
}, artifact),
warnings,
reproduced: false,
originalFailure: failure,
}
}
// Check if the same failure was reproduced
const reproducedFailure = runResult.failures.find(f =>
f.route === failure.route && f.contract === failure.contract
)
if (reproducedFailure) {
// Same failure reproduced
return {
exitCode: BEHAVIORAL_FAILURE,
message: formatHumanOutput({
exitCode: BEHAVIORAL_FAILURE,
reproduced: true,
originalFailure: failure,
warnings,
}, artifact),
warnings,
reproduced: true,
originalFailure: failure,
}
}
// Check if there are different failures
if (runResult.failures.length > 0) {
const newFailure = runResult.failures[0]
if (!newFailure) {
return {
exitCode: SUCCESS,
message: formatHumanOutput({
exitCode: SUCCESS,
reproduced: false,
originalFailure: failure,
warnings,
}, artifact),
warnings,
reproduced: false,
originalFailure: failure,
}
}
return {
exitCode: BEHAVIORAL_FAILURE,
message: formatHumanOutput({
exitCode: BEHAVIORAL_FAILURE,
reproduced: false,
originalFailure: failure,
newFailure: {
route: newFailure.route,
contract: newFailure.contract,
expected: newFailure.expected,
observed: newFailure.observed,
seed: artifact.seed || 42,
replayCommand: `apophis replay --artifact ${artifactPath}`,
},
warnings,
}, artifact),
warnings,
reproduced: false,
originalFailure: failure,
newFailure: {
route: newFailure.route,
contract: newFailure.contract,
expected: newFailure.expected,
observed: newFailure.observed,
seed: artifact.seed || 42,
replayCommand: `apophis replay --artifact ${artifactPath}`,
},
}
}
// No failures — the bug was fixed
if (!options?.sourceChanged) {
warnings.push('Replay diverged with same seed and no source drift detected. Likely runtime/data nondeterminism.')
}
return {
exitCode: SUCCESS,
message: formatHumanOutput({
exitCode: SUCCESS,
reproduced: false,
originalFailure: failure,
warnings,
}, artifact),
warnings,
reproduced: false,
originalFailure: failure,
}
}
// ---------------------------------------------------------------------------
// Main command handler
// ---------------------------------------------------------------------------
/**
* Main replay command handler.
*
* Flow:
* 1. Load and validate artifact
* 2. Check CLI version compatibility
* 3. Detect source code changes (warn but continue)
* 4. Load Fastify app and re-run verify with same seed
* 5. Compare results to original failure
* 6. Return appropriate exit code
*
* Exit codes:
* - 0: Replay passed (failure no longer reproduces)
* - 1: Same failure reproduced OR different failure found
* - 2: Error (missing artifact, corrupted, route no longer exists, etc.)
*/
export async function replayCommand(
options: ReplayOptions,
ctx: CliContext,
): Promise<CommandResult> {
const { artifact: artifactPath, config: configPath, cwd } = options
const workingDir = cwd || ctx.cwd
const resolvedArtifactPath = resolve(workingDir, artifactPath)
try {
// 1. Load and validate artifact
const loadResult = loadArtifact({
artifactPath,
cwd: workingDir,
routeFilter: options.route,
})
if (!loadResult.success) {
return {
exitCode: USAGE_ERROR,
message: loadResult.message,
warnings: loadResult.warnings,
}
}
const artifact = loadResult.artifact!
const failure = loadResult.failure!
const warnings = [...loadResult.warnings]
// 2. Execute replay
const replayResult = await executeReplay(artifact, failure, resolvedArtifactPath, ctx, {
sourceChanged: loadResult.sourceChanged,
})
// Merge warnings
if (replayResult.warnings) {
warnings.push(...replayResult.warnings)
}
return {
exitCode: replayResult.exitCode as import('../../core/types.js').ExitCode,
message: replayResult.message,
warnings: warnings.length > 0 ? warnings : undefined,
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error)
return {
exitCode: INTERNAL_ERROR,
message: `Internal error in replay command: ${message}`,
}
}
}
// ---------------------------------------------------------------------------
// CLI adapter
// ---------------------------------------------------------------------------
/**
* Adapter that bridges the CLI framework (cac) to the replay command handler.
* This function signature matches what the CLI core expects.
*/
export async function handleReplay(
args: string[],
ctx: CliContext,
): Promise<number> {
const options: ReplayOptions = {
artifact: '',
config: ctx.options.config || undefined,
cwd: ctx.cwd,
format: ctx.options.format as ReplayOptions['format'],
quiet: ctx.options.quiet,
verbose: ctx.options.verbose,
}
// Parse command-specific flags from args (passed by CLI dispatcher)
const artifactIdx = args.indexOf('--artifact')
if (artifactIdx !== -1 && args[artifactIdx + 1]) {
options.artifact = args[artifactIdx + 1]!
}
const routeIdx = args.indexOf('--route')
if (routeIdx !== -1 && args[routeIdx + 1]) {
options.route = args[routeIdx + 1]!
}
if (!options.artifact) {
const format = options.format || ctx.options.format || 'human'
if (format === 'json') {
console.log(renderJson({
exitCode: USAGE_ERROR,
error: 'Error: --artifact is required',
}))
} else if (format === 'ndjson') {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'replay',
exitCode: USAGE_ERROR,
error: 'Error: --artifact is required',
}) + '\n')
} else {
console.error('Error: --artifact is required')
}
return USAGE_ERROR
}
const result = await replayCommand(options, ctx)
// Output result based on format
if (!ctx.options.quiet && result.message) {
const format = options.format || ctx.options.format || 'human'
if (format === 'json') {
console.log(renderJson({
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}))
} else if (format === 'ndjson') {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'replay',
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}) + '\n')
} else {
console.log(result.message)
}
}
// Print warnings in human mode only
const format = options.format || ctx.options.format || 'human'
if (format === 'human' && result.warnings && result.warnings.length > 0 && !ctx.options.quiet) {
for (const warning of result.warnings) {
console.warn(`Warning: ${warning}`)
}
}
return result.exitCode
}
+424
View File
@@ -0,0 +1,424 @@
/**
* S7: Replay thread - Artifact loader and validation
*
* Responsibilities:
* - Load artifact from filesystem
* - Validate artifact schema version
* - Check CLI version compatibility
* - Detect source code changes since artifact
* - Provide degraded replay guidance
*
* Architecture:
* - Pure functions with dependency injection
* - No optional imports — everything is required or injected
* - Inline comments for documentation
*/
import { readFileSync, existsSync, statSync } from 'node:fs';
import { resolve, dirname } from 'node:path';
import type { Artifact, FailureRecord } from '../../core/types.js';
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
/** Supported artifact schema version */
const SUPPORTED_ARTIFACT_VERSION = 'apophis-artifact/1';
/** Current CLI version for compatibility checks */
const CLI_VERSION = '2.0.0';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
/**
* Result of loading and validating an artifact.
*/
export interface ArtifactLoadResult {
/** Whether the load was successful */
success: boolean;
/** The loaded artifact (if successful) */
artifact?: Artifact;
/** The failure record to replay (if successful and artifact has failures) */
failure?: FailureRecord;
/** Human-readable message about the result */
message: string;
/** Warnings about degraded replay conditions */
warnings: string[];
/** Whether the artifact is compatible with this CLI version */
compatible: boolean;
/** Whether source code has changed since the artifact was created */
sourceChanged: boolean;
}
/**
* Options for loading an artifact.
*/
export interface LoadArtifactOptions {
/** Absolute or relative path to the artifact file */
artifactPath: string;
/** Current working directory for resolving relative paths */
cwd: string;
/** CLI version to check compatibility against (injected) */
cliVersion?: string;
/** Optional route filter to select a specific failure */
routeFilter?: string;
}
// ---------------------------------------------------------------------------
// Artifact loading
// ---------------------------------------------------------------------------
/**
* Load an artifact file from disk.
* Returns the parsed artifact or throws with a clear message.
*/
export function loadArtifactFile(artifactPath: string, cwd: string): Artifact {
const resolvedPath = resolve(cwd, artifactPath);
if (!existsSync(resolvedPath)) {
throw new ArtifactLoadError(
`Artifact not found: ${resolvedPath}`,
'missing',
resolvedPath,
);
}
let content: string;
try {
content = readFileSync(resolvedPath, 'utf-8');
} catch (err) {
throw new ArtifactLoadError(
`Cannot read artifact at ${resolvedPath}: ${err instanceof Error ? err.message : String(err)}`,
'unreadable',
resolvedPath,
);
}
let parsed: unknown;
try {
parsed = JSON.parse(content);
} catch (err) {
throw new ArtifactLoadError(
`Artifact is corrupted (invalid JSON) at ${resolvedPath}: ${err instanceof Error ? err.message : String(err)}`,
'corrupted',
resolvedPath,
);
}
if (!parsed || typeof parsed !== 'object') {
throw new ArtifactLoadError(
`Artifact is corrupted (not an object) at ${resolvedPath}`,
'corrupted',
resolvedPath,
);
}
return parsed as Artifact;
}
// ---------------------------------------------------------------------------
// Schema validation
// ---------------------------------------------------------------------------
/**
* Validate that an artifact matches the expected schema.
* Checks version, required fields, and basic structure.
*/
export function validateArtifactSchema(artifact: unknown): { valid: boolean; errors: string[] } {
const errors: string[] = [];
if (!artifact || typeof artifact !== 'object') {
errors.push('Artifact must be an object');
return { valid: false, errors };
}
const obj = artifact as Record<string, unknown>;
// Check version
if (!obj.version || typeof obj.version !== 'string') {
errors.push('Missing or invalid "version" field');
} else if (obj.version !== SUPPORTED_ARTIFACT_VERSION) {
errors.push(
`Unsupported artifact version: "${obj.version}". ` +
`Expected: "${SUPPORTED_ARTIFACT_VERSION}"`,
);
}
// Check required fields
const requiredFields = ['command', 'cwd', 'startedAt', 'durationMs', 'summary'];
for (const field of requiredFields) {
if (!(field in obj)) {
errors.push(`Missing required field: "${field}"`);
}
}
// Check summary structure
if (obj.summary && typeof obj.summary === 'object') {
const summary = obj.summary as Record<string, unknown>;
const summaryFields = ['total', 'passed', 'failed'];
for (const field of summaryFields) {
if (typeof summary[field] !== 'number') {
errors.push(`Summary field "${field}" must be a number`);
}
}
}
// Check failures array
if (obj.failures !== undefined && !Array.isArray(obj.failures)) {
errors.push('Field "failures" must be an array');
}
return { valid: errors.length === 0, errors };
}
// ---------------------------------------------------------------------------
// CLI version compatibility
// ---------------------------------------------------------------------------
/**
* Check if the CLI version is compatible with the artifact.
* Artifacts from newer CLI versions may not be replayable.
*/
export function checkCliCompatibility(
artifact: Artifact,
cliVersion: string = CLI_VERSION,
): { compatible: boolean; message?: string } {
// For now, we only support exact version match
// In the future, this could support semver ranges
const artifactCliVersion = (artifact as unknown as Record<string, unknown>).cliVersion as string | undefined;
if (!artifactCliVersion) {
// No CLI version in artifact — assume compatible but warn
return {
compatible: true,
message: 'Artifact does not specify CLI version. Replay may behave differently.',
};
}
if (artifactCliVersion === cliVersion) {
return { compatible: true };
}
// Parse major versions
const artifactMajor = artifactCliVersion.split('.')[0];
const cliMajor = cliVersion.split('.')[0];
if (artifactMajor !== cliMajor) {
return {
compatible: false,
message:
`CLI version mismatch: artifact was created with v${artifactCliVersion}, ` +
`but current CLI is v${cliVersion}. Major version differences may prevent replay.`,
};
}
// Same major, different minor/patch — warn but allow
return {
compatible: true,
message:
`CLI version mismatch: artifact was created with v${artifactCliVersion}, ` +
`current CLI is v${cliVersion}. Replay should work but may differ slightly.`,
};
}
// ---------------------------------------------------------------------------
// Source code change detection
// ---------------------------------------------------------------------------
/**
* Detect if source code has changed since the artifact was created.
* Uses artifact mtime vs source file mtimes as a heuristic.
*/
export function detectSourceChanges(
artifact: Artifact,
artifactPath: string,
): { changed: boolean; details: string[] } {
const details: string[] = [];
try {
const artifactStat = statSync(artifactPath);
const artifactMtime = artifactStat.mtime;
// Check if cwd exists and get its stats
const cwd = artifact.cwd;
if (!existsSync(cwd)) {
return {
changed: true,
details: ['Artifact cwd no longer exists: ' + cwd],
};
}
// Try to find the app.js file in the cwd
const appPath = resolve(cwd, 'app.js');
if (existsSync(appPath)) {
const appStat = statSync(appPath);
if (appStat.mtime > artifactMtime) {
details.push('app.js has been modified since artifact was created');
}
}
// Check config file if referenced
if (artifact.configPath) {
const configPath = resolve(cwd, artifact.configPath);
if (existsSync(configPath)) {
const configStat = statSync(configPath);
if (configStat.mtime > artifactMtime) {
details.push('Config file has been modified since artifact was created');
}
}
}
} catch {
// If we can't stat files, assume no changes (fail open)
}
return {
changed: details.length > 0,
details,
};
}
// ---------------------------------------------------------------------------
// Route existence check
// ---------------------------------------------------------------------------
/**
* Check if the route from a failure record still exists in the current app.
* This is a heuristic — the actual check happens during replay execution.
*/
export function checkRouteExists(
failure: FailureRecord,
availableRoutes: string[],
): boolean {
return availableRoutes.includes(failure.route);
}
// ---------------------------------------------------------------------------
// Main loader
// ---------------------------------------------------------------------------
/**
* Load and validate an artifact for replay.
*
* Flow:
* 1. Load artifact file from disk
* 2. Validate schema
* 3. Check CLI version compatibility
* 4. Detect source code changes
* 5. Extract failure to replay
* 6. Return result with warnings
*/
export function loadArtifact(options: LoadArtifactOptions): ArtifactLoadResult {
const { artifactPath, cwd, cliVersion = CLI_VERSION, routeFilter } = options;
const warnings: string[] = [];
// 1. Load artifact file
let artifact: Artifact;
try {
artifact = loadArtifactFile(artifactPath, cwd);
} catch (err) {
if (err instanceof ArtifactLoadError) {
return {
success: false,
message: err.message,
warnings: [],
compatible: false,
sourceChanged: false,
};
}
throw err;
}
// 2. Validate schema
const validation = validateArtifactSchema(artifact);
if (!validation.valid) {
return {
success: false,
message: 'Artifact validation failed:\n' + validation.errors.map(e => ' ✗ ' + e).join('\n'),
warnings: [],
compatible: false,
sourceChanged: false,
};
}
// 3. Check CLI version compatibility
const compatibility = checkCliCompatibility(artifact, cliVersion);
if (!compatibility.compatible) {
return {
success: false,
message: compatibility.message!,
warnings: [],
compatible: false,
sourceChanged: false,
};
}
if (compatibility.message) {
warnings.push(compatibility.message);
}
// 4. Detect source code changes
const resolvedPath = resolve(cwd, artifactPath);
const sourceChanges = detectSourceChanges(artifact, resolvedPath);
if (sourceChanges.changed) {
warnings.push(...sourceChanges.details);
warnings.push('Source code has changed since artifact was created. Replay confidence is degraded and results may differ.');
warnings.push('Stabilize replay by checking out the same revision or rebuilding the fixture state used by the original run.');
}
// 5. Extract failure to replay
// If routeFilter is provided, find matching failure; otherwise use first failure
let failure: FailureRecord | undefined;
if (routeFilter) {
failure = artifact.failures.find(f => f.route === routeFilter);
if (!failure) {
return {
success: false,
message: `No failure found for route "${routeFilter}". Available routes: ${artifact.failures.map(f => f.route).join(', ')}`,
warnings,
compatible: compatibility.compatible,
sourceChanged: sourceChanges.changed,
};
}
} else {
failure = artifact.failures[0];
}
if (!failure) {
return {
success: false,
message: 'Artifact contains no failures to replay.',
warnings,
compatible: compatibility.compatible,
sourceChanged: sourceChanges.changed,
};
}
return {
success: true,
artifact,
failure,
message: `Loaded artifact: ${artifact.command} run with seed ${artifact.seed} (${artifact.summary.failed} failure(s))`,
warnings,
compatible: compatibility.compatible,
sourceChanged: sourceChanges.changed,
};
}
// ---------------------------------------------------------------------------
// Error type
// ---------------------------------------------------------------------------
/**
* Error type for artifact loading failures.
*/
export class ArtifactLoadError extends Error {
constructor(
message: string,
public readonly code: 'missing' | 'unreadable' | 'corrupted' | 'incompatible',
public readonly path: string,
) {
super(message);
this.name = 'ArtifactLoadError';
}
}
+803
View File
@@ -0,0 +1,803 @@
/**
* S4: Verify thread - Deterministic contract verification command
*
* Responsibilities:
* - Load config and resolve profile
* - Discover routes from Fastify app
* - Filter routes by --routes flag (supports wildcards/patterns)
* - Run deterministic contract verification
* - Generate seed if omitted, always print it
* - Produce canonical failure output matching golden snapshot
* - Emit artifact JSON
* - Print replay command
* - Support --changed for git-based filtering
* - Exit 0 on pass, 1 on behavioral failure, 2 on config error
*
* Architecture:
* - Dependency injection: all dependencies passed explicitly
* - No optional imports — everything is required or injected
* - Inline comments for documentation
*/
import type { CliContext } from '../../core/context.js'
import { loadConfig, findWorkspacePackages } from '../../core/config-loader.js'
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
import { resolveGenerationProfileOverride, GenerationProfileResolutionError } from '../../core/generation-profile.js'
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
import type { CommandResult, Artifact, FailureRecord, RouteResult, WorkspaceRun, WorkspaceResult } from '../../core/types.js'
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
import { runVerify, type VerifyRunResult } from './runner.js'
import { renderCanonicalFailure, renderHumanArtifact } from '../../renderers/human.js'
import { renderJson, renderJsonArtifact, renderJsonSummaryArtifact } from '../../renderers/json.js'
import { renderNdjsonArtifact, renderNdjsonSummaryArtifact } from '../../renderers/ndjson.js'
import type { OutputContext } from '../../renderers/shared.js'
import { resolve, basename } from 'node:path'
const ROUTE_IDENTITY_PATTERN = /^[A-Z]+\s+\/\S*$/
function normalizeRouteIdentity(route: string): string {
const normalized = route.trim().replace(/\s+/g, ' ')
const [method, ...pathParts] = normalized.split(' ')
if (!method || pathParts.length === 0) {
return normalized
}
return `${method.toUpperCase()} ${pathParts.join(' ')}`
}
function isReplayCompatibleRoute(route: string): boolean {
return ROUTE_IDENTITY_PATTERN.test(route)
}
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface VerifyOptions {
profile?: string
generationProfile?: string
routes?: string
seed?: number
changed?: boolean
config?: string
cwd?: string
format?: 'human' | 'json' | 'ndjson'
quiet?: boolean
verbose?: boolean
artifactDir?: string
}
// ---------------------------------------------------------------------------
// Seed generation
// ---------------------------------------------------------------------------
/**
* Generate a deterministic seed if none provided.
* Uses current time + process pid for uniqueness.
*/
export function generateSeed(): number {
return Date.now() + (process.pid || 0)
}
// ---------------------------------------------------------------------------
// Route filter parsing
// ---------------------------------------------------------------------------
/**
* Parse --routes flag into filter patterns.
* Supports comma-separated patterns with wildcards.
*/
function parseRouteFilters(routesFlag: string | undefined): string[] | undefined {
if (!routesFlag) return undefined
return routesFlag.split(',').map(r => r.trim()).filter(Boolean)
}
// ---------------------------------------------------------------------------
// Artifact builder
// ---------------------------------------------------------------------------
/**
* Build artifact document from verify results.
*/
function buildArtifact(
runResult: VerifyRunResult,
options: {
cwd: string
configPath?: string
profile?: string
preset?: string
env: string
seed: number
routeFilters?: string[]
},
): Artifact {
const warnings: string[] = []
const failures: FailureRecord[] = runResult.failures.map(f => {
const route = normalizeRouteIdentity(f.route)
if (!isReplayCompatibleRoute(route)) {
warnings.push(`Failure route "${f.route}" is not in METHOD /path format; replay matching may be less precise.`)
}
return {
route,
contract: f.contract,
expected: f.expected,
observed: f.observed,
seed: options.seed,
replayCommand: `apophis replay --artifact ${f.artifactPath || '<artifact-path-unavailable>'}`,
category: f.observed ? classifyError(f.observed) : ErrorTaxonomy.RUNTIME,
}
})
if (runResult.noContractsFound) {
warnings.push('No behavioral contracts found. Schema-only routes are not enough for verify. Add x-ensures or x-requires to route schemas. See docs/getting-started.md for examples.')
}
if (runResult.noRoutesMatched) {
warnings.push(`No routes matched the filter. Available routes: ${runResult.availableRoutes?.join(', ') || 'none'}`)
}
if (runResult.notGitRepo) {
warnings.push('--changed requires a git repository. Current directory is not inside a git repo.')
}
if (runResult.noRelevantChanges) {
warnings.push('No relevant changes detected. Git shows no modified files that match any route.')
}
if (runResult.failures.length > 0) {
const profileFlag = options.profile ? ` --profile ${options.profile}` : ''
const routesFlag = options.routeFilters && options.routeFilters.length > 0
? ` --routes "${options.routeFilters.join(',')}"`
: ''
warnings.push(`Deterministic rerun: apophis verify --seed ${options.seed}${profileFlag}${routesFlag}`)
warnings.push('If rerun output differs with same seed, stabilize app state/data and isolate time/external dependencies.')
}
return {
version: 'apophis-artifact/1',
cliVersion: '2.0.0',
command: 'verify',
mode: 'verify',
cwd: options.cwd,
configPath: options.configPath,
profile: options.profile,
preset: options.preset,
env: options.env,
seed: options.seed,
startedAt: new Date(Date.now() - runResult.durationMs).toISOString(),
durationMs: runResult.durationMs,
summary: {
total: runResult.total,
passed: runResult.passedCount,
failed: runResult.failed,
},
deterministicParams: {
seed: options.seed,
routeFilters: options.routeFilters ?? [],
},
failures,
artifacts: runResult.artifactPaths,
warnings,
exitReason: runResult.passed ? 'success' : 'behavioral_failure',
}
}
function attachReplayCommands(artifact: Artifact, artifactPath: string): void {
for (const failure of artifact.failures) {
failure.replayCommand = `apophis replay --artifact ${artifactPath}`
}
}
async function emitArtifact(
artifact: Artifact,
options: {
command: 'verify'
cwd: string
preferredDir?: string
force: boolean
},
): Promise<string | undefined> {
if (!options.force && !options.preferredDir) {
return undefined
}
const defaultDir = resolve(options.cwd, 'reports', 'apophis')
const candidateDirs = [options.preferredDir, defaultDir].filter(Boolean) as string[]
const attempted = new Set<string>()
for (const dir of candidateDirs) {
if (attempted.has(dir)) continue
attempted.add(dir)
try {
const { mkdirSync, writeFileSync } = await import('node:fs')
const artifactPath = resolve(dir, `${options.command}-${new Date().toISOString().replace(/[:.]/g, '-')}.json`)
mkdirSync(dir, { recursive: true })
attachReplayCommands(artifact, artifactPath)
writeFileSync(artifactPath, JSON.stringify(artifact, null, 2))
if (!artifact.artifacts.includes(artifactPath)) {
artifact.artifacts.push(artifactPath)
}
return artifactPath
} catch {
// Try fallback directory if available.
}
}
artifact.warnings.push('Failed to write artifact to disk')
return undefined
}
// ---------------------------------------------------------------------------
// Human output formatting
// ---------------------------------------------------------------------------
/**
* Format canonical failure output matching golden snapshot.
*/
function formatHumanFailure(failure: FailureRecord, profile?: string): string {
const lines: string[] = []
lines.push('Contract violation')
lines.push(failure.route)
lines.push(`Profile: ${profile || 'default'}`)
lines.push(`Seed: ${failure.seed}`)
lines.push('')
lines.push('Expected')
lines.push(` ${failure.contract}`)
lines.push('')
lines.push('Observed')
lines.push(` ${failure.observed}`)
lines.push('')
lines.push('Why this matters')
lines.push(` The resource created by ${failure.route.split(' ')[1]} is not retrievable.`)
lines.push('')
lines.push('Replay')
lines.push(` ${failure.replayCommand}`)
lines.push('')
lines.push('Next')
lines.push(` Check the create/read consistency for ${failure.route} and GET ${failure.route.split(' ')[1]}/{id}.`)
return lines.join('\n')
}
/**
* Format human-readable output for verify results.
*/
function formatHumanOutput(
runResult: VerifyRunResult,
options: { profile?: string; seed: number; env: string; routeFilters?: string[] },
): string {
const lines: string[] = []
if (runResult.notGitRepo) {
lines.push(`--changed requires a git repository.`)
lines.push(`Current directory is not inside a git repo.`)
lines.push('')
lines.push('Next:')
lines.push(` Initialize git with \`git init\`, or run verify without --changed.`)
lines.push('')
return lines.join('\n')
}
if (runResult.noRelevantChanges) {
lines.push(`No relevant changes detected.`)
lines.push(`Git shows no modified files that match any route.`)
lines.push('')
return lines.join('\n')
}
if (runResult.noRoutesMatched) {
lines.push(`No routes matched the filter.`)
lines.push(`Filters applied: ${options.routeFilters?.join(', ') || 'none'}`)
lines.push(`Available routes:`)
for (const r of runResult.availableRoutes || []) {
lines.push(` ${r}`)
}
lines.push('')
lines.push('Next:')
lines.push(` Adjust --routes filter or add routes to your app.`)
lines.push('')
return lines.join('\n')
}
if (runResult.noContractsFound) {
lines.push('No behavioral contracts found.')
lines.push('')
lines.push('APOPHIS discovered routes, but none have behavioral contracts.')
lines.push('Schema-only routes (with response schemas) are not enough.')
lines.push('You must add x-ensures or x-requires clauses that check behavior.')
lines.push('')
lines.push('Example — add this to your route schema:')
lines.push(' "x-ensures": [')
lines.push(' "response_code(GET /users/{response_body(this).id}) == 200"')
lines.push(' ]')
lines.push('')
lines.push('Next steps:')
lines.push(' 1. Open your route file (e.g., app.js or src/routes/users.js)')
lines.push(' 2. Find the route you want to test')
lines.push(' 3. Add an "x-ensures" array inside the schema object')
lines.push(' 4. Run: apophis verify --profile quick --routes "POST /users"')
lines.push('')
lines.push('For more examples, see docs/getting-started.md')
lines.push('')
return lines.join('\n')
}
// Print failures using canonical format
for (const failure of runResult.failures) {
const failureRecord: FailureRecord = {
route: failure.route,
contract: failure.contract,
expected: failure.expected,
observed: failure.observed,
seed: options.seed,
replayCommand: `apophis replay --artifact ${failure.artifactPath || 'reports/apophis/failure-*.json'}`,
}
lines.push(formatHumanFailure(failureRecord, options.profile))
lines.push('')
}
// Summary
if (runResult.passed) {
lines.push(`All ${runResult.total} contract(s) passed.`)
} else {
lines.push(`Failed: ${runResult.failed} of ${runResult.total} contract(s) failed.`)
}
lines.push(`Seed: ${options.seed}`)
// Replay command on failure
if (!runResult.passed && runResult.failures.length > 0) {
lines.push('')
lines.push('Replay')
lines.push(` apophis replay --artifact <path-to-artifact>`)
lines.push('')
lines.push('Determinism')
lines.push(` This run used seed ${options.seed}.`)
lines.push(` Same seed + same app state = same results.`)
lines.push(` If results differ on re-run, the app has nondeterministic behavior.`)
lines.push(` Stabilize: reset app state, mock external services, avoid time-dependent logic.`)
}
return lines.join('\n')
}
// ---------------------------------------------------------------------------
// Main command handler
// ---------------------------------------------------------------------------
/**
* Main verify command handler.
*
* Flow:
* 1. Load and resolve config
* 2. Run policy engine checks
* 3. Generate seed if omitted, always print it
* 4. Parse route filters
* 5. Load Fastify app and discover routes
* 6. Run deterministic contract verification
* 7. Build artifact
* 8. Format output
* 9. Write artifact if artifactDir specified
* 10. Return appropriate exit code
*/
export async function verifyCommand(
options: VerifyOptions,
ctx: CliContext,
): Promise<CommandResult> {
const {
profile,
generationProfile,
routes: routesFlag,
seed: explicitSeed,
changed,
config: configPath,
cwd,
artifactDir,
} = options
const workingDir = cwd || ctx.cwd
const format = options.format || ctx.options.format || 'human'
// Detect environment
const env = detectEnvironment()
try {
// 1. Load config
const loadResult = await loadConfig({
cwd: workingDir,
configPath,
profileName: profile,
env,
})
if (!loadResult.configPath) {
return {
exitCode: USAGE_ERROR,
message: 'No config found. Run "apophis init" to create one.',
}
}
const config = loadResult.config
const resolvedGenerationProfile = resolveGenerationProfileOverride(generationProfile, config)
// 2a. Resolve profile — if explicitly requested but missing, list available ones
if (profile && !config.profiles?.[profile]) {
const available = Object.keys(config.profiles ?? {}).join(', ') || 'none'
return {
exitCode: USAGE_ERROR,
message: `Unknown profile "${profile}". Available profiles: ${available}.\n\nNext:\n Run \`apophis init\` to scaffold a new profile, or use one of the profiles listed above.`,
}
}
// 2. Run policy engine checks
const policyEngine = new PolicyEngine({
config,
env,
mode: 'verify',
profileName: profile || undefined,
presetName: loadResult.presetName || undefined,
})
const policyResult = policyEngine.check()
if (!policyResult.allowed) {
const message = [
'Policy check failed:',
...policyResult.errors.map(e => `${e}`),
].join('\n')
return {
exitCode: USAGE_ERROR,
message,
}
}
// 3. Generate seed if omitted
const seed = explicitSeed ?? generateSeed()
if (!ctx.options.quiet && format === 'human') {
console.log(`Seed: ${seed}`)
}
// 4. Parse route filters
const routeFilters = parseRouteFilters(routesFlag)
// 5. Load the Fastify app
let fastify: unknown
try {
const { loadApp } = await import('../../core/app-loader.js')
const loaded = await loadApp(workingDir)
fastify = loaded.fastify
if (fastify && typeof (fastify as any).ready === 'function') {
await (fastify as any).ready()
}
} catch (err) {
const errorMessage = err instanceof Error ? err.message : String(err)
return {
exitCode: USAGE_ERROR,
message: `No Fastify app found. Ensure app.js exports a Fastify instance.\n\nError: ${errorMessage}\n\nNext:\n Run \`apophis init\` to scaffold a working app.js and config.`,
}
}
// 6. Run verify execution
const runResult = await runVerify({
fastify: fastify as any,
seed,
generationProfile: resolvedGenerationProfile,
timeout: typeof config.presets?.[loadResult.presetName || '']?.timeout === 'number'
? (config.presets[loadResult.presetName || ''] as { timeout?: number }).timeout
: undefined,
routeFilters,
changed,
profileRoutes: config.profiles?.[profile || '']?.routes,
})
// 7. Build artifact
const artifact = buildArtifact(runResult, {
cwd: workingDir,
configPath: loadResult.configPath,
profile: profile || undefined,
preset: loadResult.presetName || undefined,
env,
seed,
routeFilters,
})
// 8. Write artifact if configured or on failure
const shouldEmitArtifact = Boolean(artifactDir || config.artifactDir || !runResult.passed)
await emitArtifact(artifact, {
command: 'verify',
cwd: workingDir,
preferredDir: artifactDir || config.artifactDir,
force: shouldEmitArtifact,
})
// 9. Format output based on format option
const outputCtx: OutputContext = {
isTTY: ctx.isTTY,
isCI: ctx.isCI,
colorMode: ctx.options.color,
}
let message: string
if (format === 'json') {
message = renderJsonArtifact(artifact)
} else if (format === 'json-summary') {
message = renderJsonSummaryArtifact(artifact)
} else if (format === 'ndjson') {
// For ndjson, we don't return a message string; events are streamed
message = ''
} else if (format === 'ndjson-summary') {
// Concise ndjson: only summary events
message = ''
} else {
// human format
message = renderHumanArtifact(artifact, outputCtx)
}
// Determine exit code
let exitCode: number = SUCCESS
if (runResult.noRoutesMatched || runResult.noContractsFound || runResult.notGitRepo) {
exitCode = USAGE_ERROR
} else if (!runResult.passed) {
exitCode = BEHAVIORAL_FAILURE
}
return {
exitCode: exitCode as import('../../core/types.js').ExitCode,
artifact,
message,
warnings: artifact.warnings,
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error)
// Config validation errors are usage errors, not internal errors
if (error instanceof Error && error.name === 'ConfigValidationError') {
return {
exitCode: USAGE_ERROR,
message: `Config validation failed: ${message}`,
}
}
if (error instanceof GenerationProfileResolutionError) {
return {
exitCode: USAGE_ERROR,
message,
}
}
return {
exitCode: INTERNAL_ERROR,
message: `Internal error in verify command: ${message}`,
}
}
}
// ---------------------------------------------------------------------------
// CLI adapter
// ---------------------------------------------------------------------------
/**
* Adapter that bridges the CLI framework (cac) to the verify command handler.
* This function signature matches what the CLI core expects.
*/
export async function handleVerify(
args: string[],
ctx: CliContext,
): Promise<number> {
const options: VerifyOptions = {
profile: ctx.options.profile || undefined,
generationProfile: ctx.options.generationProfile,
routes: undefined,
seed: undefined,
changed: false,
config: ctx.options.config || undefined,
cwd: ctx.cwd,
format: ctx.options.format as VerifyOptions['format'],
quiet: ctx.options.quiet,
verbose: ctx.options.verbose,
artifactDir: ctx.options.artifactDir || undefined,
}
// Parse command-specific flags from args (passed by CLI dispatcher)
const routesIdx = args.indexOf('--routes')
if (routesIdx !== -1 && args[routesIdx + 1]) {
options.routes = args[routesIdx + 1]
}
const seedIdx = args.indexOf('--seed')
if (seedIdx !== -1 && args[seedIdx + 1]) {
const parsed = parseInt(args[seedIdx + 1]!, 10)
if (!isNaN(parsed)) {
options.seed = parsed
}
}
options.seed = options.seed as number | undefined
if (args.includes('--changed')) {
options.changed = true
}
const generationProfileIdx = args.indexOf('--generation-profile')
if (generationProfileIdx !== -1 && args[generationProfileIdx + 1]) {
options.generationProfile = args[generationProfileIdx + 1]
}
const workspaceMode = args.includes('--workspace')
if (workspaceMode) {
const packages = findWorkspacePackages(ctx.cwd)
if (packages.length === 0) {
if (!ctx.options.quiet) {
console.error('No workspace packages found. Ensure workspaces are defined in root package.json or pnpm-workspace.yaml.')
}
return USAGE_ERROR
}
const runs: WorkspaceRun[] = []
let overallExitCode = SUCCESS
const allWarnings: string[] = []
for (const pkgPath of packages) {
const pkgName = basename(pkgPath)
const pkgOptions = { ...options, cwd: pkgPath }
const pkgCtx: CliContext = { ...ctx, cwd: pkgPath }
const pkgResult = await verifyCommand(pkgOptions, pkgCtx)
if (pkgResult.artifact) {
pkgResult.artifact.package = pkgName
runs.push({ package: pkgName, cwd: pkgPath, artifact: pkgResult.artifact })
}
if (pkgResult.exitCode !== SUCCESS) {
overallExitCode = pkgResult.exitCode
}
if (pkgResult.warnings) {
allWarnings.push(...pkgResult.warnings.map(w => `[${pkgName}] ${w}`))
}
}
const workspaceResult: WorkspaceResult = {
exitCode: overallExitCode as import('../../core/types.js').ExitCode,
runs,
warnings: allWarnings,
}
const format = options.format || ctx.options.format || 'human'
if (!ctx.options.quiet) {
if (format === 'json') {
console.log(renderJson({
exitCode: workspaceResult.exitCode,
runs: workspaceResult.runs.map(r => ({
package: r.package,
cwd: r.cwd,
artifact: r.artifact,
})),
warnings: workspaceResult.warnings,
}))
} else if (format === 'json-summary') {
console.log(renderJson({
exitCode: workspaceResult.exitCode,
runs: workspaceResult.runs.map(r => ({
package: r.package,
cwd: r.cwd,
summary: r.artifact.summary,
exitReason: r.artifact.exitReason,
})),
warnings: workspaceResult.warnings,
}))
} else if (format === 'ndjson') {
for (const run of workspaceResult.runs) {
process.stdout.write(JSON.stringify({
type: 'workspace.run.completed',
package: run.package,
cwd: run.cwd,
summary: run.artifact.summary,
exitReason: run.artifact.exitReason,
}) + '\n')
}
process.stdout.write(JSON.stringify({
type: 'workspace.completed',
exitCode: workspaceResult.exitCode,
packages: workspaceResult.runs.length,
}) + '\n')
} else if (format === 'ndjson-summary') {
for (const run of workspaceResult.runs) {
process.stdout.write(JSON.stringify({
type: 'workspace.run.completed',
package: run.package,
cwd: run.cwd,
summary: run.artifact.summary,
exitReason: run.artifact.exitReason,
}) + '\n')
}
process.stdout.write(JSON.stringify({
type: 'workspace.completed',
exitCode: workspaceResult.exitCode,
packages: workspaceResult.runs.length,
}) + '\n')
} else {
// Human format
const lines: string[] = []
lines.push('Workspace verify results')
lines.push('')
for (const run of workspaceResult.runs) {
const a = run.artifact
const status = a.exitReason === 'success' ? '✓' : '✗'
lines.push(` ${status} ${run.package}: ${a.summary.passed}/${a.summary.total} passed`)
if (a.summary.failed > 0) {
lines.push(` ${a.summary.failed} failed`)
}
}
lines.push('')
lines.push(`Overall: ${workspaceResult.exitCode === SUCCESS ? 'passed' : 'failed'}`)
console.log(lines.join('\n'))
}
}
if (format !== 'json' && format !== 'ndjson' && format !== 'json-summary' && format !== 'ndjson-summary' && allWarnings.length > 0 && !ctx.options.quiet) {
for (const warning of allWarnings) {
console.warn(`Warning: ${warning}`)
}
}
return workspaceResult.exitCode
}
const result = await verifyCommand(options, ctx)
const format = options.format || ctx.options.format || 'human'
const machineMode = format === 'json' || format === 'ndjson' || format === 'json-summary' || format === 'ndjson-summary'
if (!ctx.options.quiet) {
if (format === 'json') {
if (result.artifact) {
console.log(renderJsonArtifact(result.artifact))
} else {
console.log(renderJson({
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}))
}
} else if (format === 'json-summary') {
if (result.artifact) {
console.log(renderJsonSummaryArtifact(result.artifact))
} else {
console.log(renderJson({
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}))
}
} else if (format === 'ndjson') {
if (result.artifact) {
renderNdjsonArtifact(result.artifact)
} else {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'verify',
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}) + '\n')
}
} else if (format === 'ndjson-summary') {
if (result.artifact) {
renderNdjsonSummaryArtifact(result.artifact)
} else {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'verify',
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}) + '\n')
}
} else if (result.message) {
console.log(result.message)
}
}
// Print warnings in human mode only
if (!machineMode && result.warnings && result.warnings.length > 0 && !ctx.options.quiet) {
for (const warning of result.warnings) {
console.warn(`Warning: ${warning}`)
}
}
return result.exitCode
}
+490
View File
@@ -0,0 +1,490 @@
/**
* S4: Verify thread - Runner for deterministic contract verification
*
* Responsibilities:
* - Route discovery from Fastify app
* - Route filtering by patterns and git changes
* - Contract execution using existing plugin/evaluator code
* - Deterministic execution with seed
* - Result aggregation
*
* Architecture:
* - Pure execution functions that accept injected dependencies
* - Reuses existing APOPHIS plugin and formula code
* - No reimplementation of parser/evaluator
*/
import { discoverRoutes } from '../../../domain/discovery.js'
import { extractContract } from '../../../domain/contract.js'
import { executeHttp } from '../../../infrastructure/http-executor.js'
import { parse } from '../../../formula/parser.js'
import { evaluateAsync } from '../../../formula/evaluator.js'
import { createOperationResolver } from '../../../formula/runtime.js'
import type { EvalContext, RouteContract, FastifyInjectInstance } from '../../../types.js'
import type { RouteResult } from '../../core/types.js'
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface VerifyFailure {
route: string
contract: string
expected: string
observed: string
artifactPath?: string
}
export interface VerifyRunResult {
passed: boolean
total: number
passedCount: number
failed: number
failures: VerifyFailure[]
durationMs: number
noRoutesMatched: boolean
noContractsFound: boolean
notGitRepo?: boolean
noRelevantChanges?: boolean
availableRoutes?: string[]
artifactPaths: string[]
}
export interface VerifyRunnerDeps {
fastify: FastifyInjectInstance
seed: number
generationProfile?: 'quick' | 'standard' | 'thorough'
timeout?: number
routeFilters?: string[]
changed?: boolean
profileRoutes?: string[]
}
// ---------------------------------------------------------------------------
// Route discovery
// ---------------------------------------------------------------------------
/**
* Discover routes from a Fastify instance.
* Uses the existing discovery module.
*/
export async function discoverAppRoutes(fastify: FastifyInjectInstance): Promise<RouteContract[]> {
return discoverRoutes(fastify)
}
/**
* Check if specific routes exist in a Fastify instance using hasRoute.
* Used when the APOPHIS plugin wasn't registered before routes.
*/
export async function discoverSpecificRoutes(
fastify: FastifyInjectInstance,
routePatterns: string[],
): Promise<RouteContract[]> {
if (typeof fastify.hasRoute !== 'function') {
return []
}
const routes: RouteContract[] = []
const seen = new Set<string>()
for (const pattern of routePatterns) {
// Parse pattern like "GET /users" or "POST /api/*"
const parts = pattern.split(' ')
const method = parts[0] || 'GET'
const path = parts.slice(1).join(' ')
// For exact routes (no wildcards), check if route exists
if (!pattern.includes('*') && !pattern.includes('?')) {
try {
if (fastify.hasRoute({ url: path, method })) {
const key = `${method} ${path}`
if (!seen.has(key)) {
seen.add(key)
routes.push({
method: method as RouteContract['method'],
path,
category: 'observer',
schema: {},
requires: [],
ensures: [],
invariants: [],
regexPatterns: {},
validateRuntime: false,
})
}
}
} catch {
// Route doesn't exist
}
}
}
return routes
}
// ---------------------------------------------------------------------------
// Route filtering
// ---------------------------------------------------------------------------
/**
* Check if a route matches a filter pattern.
* Supports wildcards: * matches any characters.
*/
function matchRoutePattern(route: string, pattern: string): boolean {
// Convert pattern to regex
const regexPattern = pattern
.replace(/\*/g, '.*')
.replace(/\?/g, '.')
const regex = new RegExp(`^${regexPattern}$`, 'i')
return regex.test(route)
}
/**
* Filter routes by patterns.
*/
function filterRoutesByPatterns(routes: RouteContract[], patterns: string[]): RouteContract[] {
return routes.filter(route => {
const routeStr = `${route.method} ${route.path}`
return patterns.some(pattern => matchRoutePattern(routeStr, pattern))
})
}
/**
* Check if cwd is inside a git repository.
*/
async function isGitRepo(cwd: string): Promise<boolean> {
try {
const { execSync } = await import('node:child_process')
execSync('git rev-parse --git-dir', { cwd, encoding: 'utf-8', stdio: 'pipe' })
return true
} catch {
return false
}
}
/**
* Get git-modified files for --changed filtering.
*/
async function getGitChangedFiles(cwd: string): Promise<string[]> {
try {
const { execSync } = await import('node:child_process')
const output = execSync('git diff --name-only HEAD', { cwd, encoding: 'utf-8' })
return output.split('\n').filter(Boolean)
} catch {
return []
}
}
/**
* Filter routes to only those modified in git.
*/
async function filterChangedRoutes(
routes: RouteContract[],
cwd: string,
): Promise<RouteContract[]> {
const changedFiles = await getGitChangedFiles(cwd)
// Map route paths to potential file paths (heuristic)
return routes.filter(route => {
const routePath = route.path
// Check if any changed file might contain this route
return changedFiles.some(file => {
// Simple heuristic: check if route path segments appear in file path
const segments = routePath.split('/').filter(Boolean)
return segments.some(segment => file.includes(segment))
})
})
}
// ---------------------------------------------------------------------------
// Contract execution
// ---------------------------------------------------------------------------
/**
* Build a request for a route.
*/
function buildRouteRequest(route: RouteContract): {
method: string
url: string
body?: unknown
headers: Record<string, string>
} {
const headers: Record<string, string> = {
'content-type': 'application/json',
}
// Build body from schema if available
let body: unknown = undefined
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
if (bodySchema && route.method === 'POST') {
body = buildExampleBody(bodySchema)
}
return {
method: route.method,
url: route.path,
body,
headers,
}
}
/**
* Build an example body from JSON Schema.
*/
function buildExampleBody(schema: Record<string, unknown>): unknown {
if (schema.type === 'object' && schema.properties) {
const obj: Record<string, unknown> = {}
const properties = schema.properties as Record<string, Record<string, unknown>>
for (const [key, propSchema] of Object.entries(properties)) {
obj[key] = buildExampleValue(propSchema)
}
return obj
}
return undefined
}
/**
* Build an example value from a property schema.
*/
function buildExampleValue(schema: Record<string, unknown>): unknown {
if (schema.type === 'string') {
if (schema.enum && Array.isArray(schema.enum) && schema.enum.length > 0) {
return schema.enum[0]
}
return 'test'
}
if (schema.type === 'number' || schema.type === 'integer') {
return 1
}
if (schema.type === 'boolean') {
return true
}
if (schema.type === 'array') {
return []
}
if (schema.type === 'object' && schema.properties) {
return buildExampleBody(schema)
}
return undefined
}
/**
* Execute a single contract for a route.
* Returns the evaluation context and any failure.
*/
async function executeContract(
fastify: FastifyInjectInstance,
route: RouteContract,
contract: string,
timeout?: number,
variant?: { name: string; headers?: Record<string, string> },
): Promise<{ ctx: EvalContext; failure?: VerifyFailure }> {
const request = buildRouteRequest(route)
// Merge variant headers if provided
const headers = variant?.headers
? { ...request.headers, ...variant.headers }
: request.headers
// Execute the primary request
const ctx = await executeHttp(fastify, route, {
method: request.method,
url: request.url,
body: request.body,
headers,
query: {},
}, undefined, timeout)
// Build eval context with operation resolver for cross-operation calls
const evalCtx: EvalContext = {
...ctx,
operationResolver: createOperationResolver(fastify, headers, ctx),
}
// Parse and evaluate the contract
try {
const parsed = parse(contract)
const result = await evaluateAsync(parsed.ast, evalCtx)
if (!result.success || !result.value) {
return {
ctx: evalCtx,
failure: {
route: variant && variant.name !== 'default'
? `[variant:${variant.name}] ${route.method} ${route.path}`
: `${route.method} ${route.path}`,
contract,
expected: 'true',
observed: result.success ? String(result.value) : result.error,
},
}
}
return { ctx: evalCtx }
} catch (error) {
return {
ctx: evalCtx,
failure: {
route: variant && variant.name !== 'default'
? `[variant:${variant.name}] ${route.method} ${route.path}`
: `${route.method} ${route.path}`,
contract,
expected: 'true',
observed: error instanceof Error ? error.message : String(error),
},
}
}
}
// ---------------------------------------------------------------------------
// Main verify runner
// ---------------------------------------------------------------------------
/**
* Run deterministic contract verification.
*
* Flow:
* 1. Discover routes from Fastify app
* 2. Apply route filters (patterns, changed, profile routes)
* 3. Check for behavioral contracts
* 4. Execute each contract deterministically
* 5. Aggregate results
*/
export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult> {
const started = Date.now()
const { fastify, routeFilters, changed, profileRoutes } = deps
// 1. Discover routes
let allRoutes = await discoverAppRoutes(fastify)
// If no routes discovered (plugin not registered before routes),
// try to discover specific routes from filters
if (allRoutes.length === 0 && (routeFilters?.length || profileRoutes?.length)) {
const patternsToCheck = [
...(routeFilters || []),
...(profileRoutes || []),
]
allRoutes = await discoverSpecificRoutes(fastify, patternsToCheck)
}
const availableRoutes = allRoutes.map(r => `${r.method} ${r.path}`)
// 2. Apply filters
let routes = allRoutes
// Apply profile routes filter first
if (profileRoutes && profileRoutes.length > 0) {
routes = filterRoutesByPatterns(routes, profileRoutes)
}
// Apply --routes flag filter
if (routeFilters && routeFilters.length > 0) {
routes = filterRoutesByPatterns(routes, routeFilters)
}
// Apply --changed filter
if (changed) {
const cwd = process.cwd()
const inGit = await isGitRepo(cwd)
if (!inGit) {
return {
passed: false,
total: 0,
passedCount: 0,
failed: 0,
failures: [],
durationMs: Date.now() - started,
noRoutesMatched: false,
noContractsFound: false,
availableRoutes,
artifactPaths: [],
notGitRepo: true,
}
}
routes = await filterChangedRoutes(routes, cwd)
}
// Check if any routes matched
if (routes.length === 0) {
return {
passed: false,
total: 0,
passedCount: 0,
failed: 0,
failures: [],
durationMs: Date.now() - started,
noRoutesMatched: true,
noContractsFound: false,
availableRoutes,
artifactPaths: [],
}
}
// 3. Check for behavioral contracts
const routesWithContracts = routes.filter(route =>
route.ensures.length > 0 || route.requires.length > 0
)
if (routesWithContracts.length === 0) {
return {
passed: false,
total: 0,
passedCount: 0,
failed: 0,
failures: [],
durationMs: Date.now() - started,
noRoutesMatched: false,
noContractsFound: true,
availableRoutes,
artifactPaths: [],
}
}
// 4. Execute contracts (with variant expansion)
const failures: VerifyFailure[] = []
let total = 0
let passedCount = 0
for (const route of routesWithContracts) {
const contracts = [...route.requires, ...route.ensures]
const variants = route.variants && route.variants.length > 0
? route.variants
: [{ name: 'default' }]
for (const variant of variants) {
for (const contract of contracts) {
total++
const result = await executeContract(fastify, route, contract, deps.timeout, variant)
if (result.failure) {
failures.push(result.failure)
} else {
passedCount++
}
}
}
}
const durationMs = Date.now() - started
// Sort failures deterministically by route then contract for stable output
const sortedFailures = failures.sort((a, b) => {
const routeCmp = a.route.localeCompare(b.route)
if (routeCmp !== 0) return routeCmp
return a.contract.localeCompare(b.contract)
})
return {
passed: failures.length === 0,
total,
passedCount,
failed: failures.length,
failures: sortedFailures,
durationMs,
noRoutesMatched: false,
noContractsFound: false,
availableRoutes,
artifactPaths: [],
}
}
+101
View File
@@ -0,0 +1,101 @@
/**
* App loader utility for CLI commands.
* Handles various app export patterns and module systems.
*/
import { resolve } from 'node:path'
import { pathToFileURL } from 'node:url'
export interface LoadedApp {
fastify: unknown
source: 'default' | 'named' | 'commonjs'
}
/**
* Load a Fastify app from app.js in the given directory.
* Supports:
* - ESM default export: export default fastifyInstance
* - ESM named export: export const createApp = () => fastifyInstance
* - CommonJS: module.exports = fastifyInstance
* - CommonJS named: exports.createApp = () => fastifyInstance
*/
export async function loadApp(cwd: string): Promise<LoadedApp> {
const appPath = resolve(cwd, 'app.js')
const appUrl = pathToFileURL(appPath).href + '?t=' + Date.now()
let appModule: Record<string, unknown>
try {
appModule = await import(appUrl) as Record<string, unknown>
} catch (err) {
throw new AppLoadError(
`Cannot load app.js: ${err instanceof Error ? err.message : String(err)}`,
'import_failed',
)
}
// Try default export first
if (appModule.default && isFastifyInstance(appModule.default)) {
return { fastify: appModule.default, source: 'default' }
}
// Try named exports that look like Fastify instances or factory functions
for (const [key, value] of Object.entries(appModule)) {
if (key === 'default') continue
if (isFastifyInstance(value)) {
return { fastify: value, source: 'named' }
}
// Try calling factory functions
if (typeof value === 'function' && !isClass(value)) {
try {
const result = await value()
if (isFastifyInstance(result)) {
return { fastify: result, source: 'named' }
}
} catch {
// Factory function failed, try next
}
}
}
// If module itself is a Fastify instance (CommonJS)
if (isFastifyInstance(appModule)) {
return { fastify: appModule, source: 'commonjs' }
}
throw new AppLoadError(
'No Fastify instance found in app.js. Ensure app.js exports a Fastify instance or a factory function.',
'no_fastify',
)
}
/**
* Check if a value looks like a Fastify instance.
*/
function isFastifyInstance(value: unknown): boolean {
return value !== null &&
typeof value === 'object' &&
typeof (value as Record<string, unknown>).ready === 'function'
}
/**
* Check if a function is a class constructor.
*/
function isClass(fn: unknown): boolean {
return typeof fn === 'function' &&
fn.toString().startsWith('class ')
}
/**
* Error type for app loading failures.
*/
export class AppLoadError extends Error {
constructor(
message: string,
public readonly code: 'import_failed' | 'no_fastify',
) {
super(message)
this.name = 'AppLoadError'
}
}
+330
View File
@@ -0,0 +1,330 @@
/**
* Tests for config-loader.ts
*/
import { test } from 'node:test';
import assert from 'node:assert';
import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
discoverConfig,
loadPackageJsonConfig,
loadConfigFile,
validateConfigAgainstSchema,
resolveProfile,
applyEnvironmentOverrides,
detectMonorepo,
loadConfig,
ConfigValidationError,
CONFIG_SCHEMA,
type Config,
} from './config-loader.js';
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function createTempDir(): string {
const dir = join(tmpdir(), `apophis-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
mkdirSync(dir, { recursive: true });
return dir;
}
function cleanup(dir: string): void {
rmSync(dir, { recursive: true, force: true });
}
// ---------------------------------------------------------------------------
// discoverConfig
// ---------------------------------------------------------------------------
test('discoverConfig finds apophis.config.js', () => {
const dir = createTempDir();
writeFileSync(join(dir, 'apophis.config.js'), 'module.exports = {}');
const result = discoverConfig(dir);
assert.strictEqual(result, join(dir, 'apophis.config.js'));
cleanup(dir);
});
test('discoverConfig finds apophis.config.ts', () => {
const dir = createTempDir();
writeFileSync(join(dir, 'apophis.config.ts'), 'export default {}');
const result = discoverConfig(dir);
assert.strictEqual(result, join(dir, 'apophis.config.ts'));
cleanup(dir);
});
test('discoverConfig finds apophis.config.json', () => {
const dir = createTempDir();
writeFileSync(join(dir, 'apophis.config.json'), '{}');
const result = discoverConfig(dir);
assert.strictEqual(result, join(dir, 'apophis.config.json'));
cleanup(dir);
});
test('discoverConfig returns null when no config found', () => {
const dir = createTempDir();
const result = discoverConfig(dir);
assert.strictEqual(result, null);
cleanup(dir);
});
// ---------------------------------------------------------------------------
// loadPackageJsonConfig
// ---------------------------------------------------------------------------
test('loadPackageJsonConfig finds apophis field', () => {
const dir = createTempDir();
writeFileSync(
join(dir, 'package.json'),
JSON.stringify({ name: 'test', apophis: { mode: 'verify' } }),
);
const result = loadPackageJsonConfig(dir);
assert.deepStrictEqual(result.config, { mode: 'verify' });
assert.strictEqual(result.path, join(dir, 'package.json'));
cleanup(dir);
});
test('loadPackageJsonConfig returns null when no apophis field', () => {
const dir = createTempDir();
writeFileSync(join(dir, 'package.json'), JSON.stringify({ name: 'test' }));
const result = loadPackageJsonConfig(dir);
assert.strictEqual(result.config, null);
cleanup(dir);
});
// ---------------------------------------------------------------------------
// loadConfigFile
// ---------------------------------------------------------------------------
test('loadConfigFile loads JSON config', async () => {
const dir = createTempDir();
const path = join(dir, 'config.json');
writeFileSync(path, JSON.stringify({ mode: 'verify', seed: 42 }));
const config = await loadConfigFile(path);
assert.deepStrictEqual(config, { mode: 'verify', seed: 42 });
cleanup(dir);
});
// ---------------------------------------------------------------------------
// validateConfigAgainstSchema
// ---------------------------------------------------------------------------
test('validateConfigAgainstSchema passes for valid keys', () => {
const config = {
mode: 'verify',
seed: 42,
routes: ['GET /users'],
};
assert.doesNotThrow(() => validateConfigAgainstSchema(config, CONFIG_SCHEMA));
});
test('validateConfigAgainstSchema fails for unknown top-level key', () => {
const config = {
mode: 'verify',
unknownKey: true,
};
assert.throws(
() => validateConfigAgainstSchema(config, CONFIG_SCHEMA),
(err: unknown) => {
assert.ok(err instanceof ConfigValidationError);
assert.strictEqual((err as ConfigValidationError).path, 'unknownKey');
assert.strictEqual((err as ConfigValidationError).key, 'unknownKey');
return true;
},
);
});
test('validateConfigAgainstSchema fails for unknown nested key', () => {
const config = {
environments: {
local: {
allowedModes: ['verify'],
badKey: true,
},
},
};
assert.throws(
() => validateConfigAgainstSchema(config, CONFIG_SCHEMA),
(err: unknown) => {
assert.ok(err instanceof ConfigValidationError);
assert.ok((err as ConfigValidationError).path.startsWith('environments'));
return true;
},
);
});
// ---------------------------------------------------------------------------
// resolveProfile
// ---------------------------------------------------------------------------
test('resolveProfile returns original config when no profile specified', () => {
const config: Config = { mode: 'verify', seed: 42 };
const result = resolveProfile(config, undefined);
assert.deepStrictEqual(result.config, config);
assert.strictEqual(result.profileName, null);
assert.strictEqual(result.presetName, null);
});
test('resolveProfile applies preset defaults then profile overrides', () => {
const config: Config = {
presets: {
safe: { mode: 'verify', seed: 1 },
},
profiles: {
quick: {
preset: 'safe',
seed: 99,
},
},
};
const result = resolveProfile(config, 'quick');
assert.strictEqual(result.config.mode, 'verify');
assert.strictEqual(result.config.seed, 99);
assert.strictEqual(result.profileName, 'quick');
assert.strictEqual(result.presetName, 'safe');
});
test('resolveProfile throws for unknown profile', () => {
const config = { profiles: {} };
assert.throws(() => resolveProfile(config, 'missing'), /Unknown profile/);
});
// ---------------------------------------------------------------------------
// applyEnvironmentOverrides
// ---------------------------------------------------------------------------
test('applyEnvironmentOverrides returns original config when no env', () => {
const config: Config = { mode: 'verify' };
const result = applyEnvironmentOverrides(config, undefined);
assert.deepStrictEqual(result, config);
});
test('applyEnvironmentOverrides applies env policy', () => {
const config: Config = {
mode: 'verify',
environments: {
staging: { blockQualify: true },
},
};
const result = applyEnvironmentOverrides(config, 'staging');
assert.deepStrictEqual(result.environments?.staging, { blockQualify: true });
});
// ---------------------------------------------------------------------------
// detectMonorepo
// ---------------------------------------------------------------------------
test('detectMonorepo returns true for workspaces', () => {
const dir = createTempDir();
writeFileSync(
join(dir, 'package.json'),
JSON.stringify({ name: 'root', workspaces: ['packages/*'] }),
);
const result = detectMonorepo(dir);
assert.strictEqual(result, true);
cleanup(dir);
});
test('detectMonorepo returns true for pnpm-workspace.yaml', () => {
const dir = createTempDir();
writeFileSync(join(dir, 'package.json'), JSON.stringify({ name: 'root' }));
writeFileSync(join(dir, 'pnpm-workspace.yaml'), 'packages:\n - packages/*');
const result = detectMonorepo(dir);
assert.strictEqual(result, true);
cleanup(dir);
});
test('detectMonorepo returns false for single package', () => {
const dir = createTempDir();
writeFileSync(join(dir, 'package.json'), JSON.stringify({ name: 'single' }));
const result = detectMonorepo(dir);
assert.strictEqual(result, false);
cleanup(dir);
});
// ---------------------------------------------------------------------------
// loadConfig (integration)
// ---------------------------------------------------------------------------
test('loadConfig loads JS config file', async () => {
const dir = createTempDir();
writeFileSync(
join(dir, 'apophis.config.js'),
'export default { mode: "verify", seed: 42 }',
);
const result = await loadConfig({ cwd: dir });
assert.strictEqual(result.config.mode, 'verify');
assert.strictEqual(result.config.seed, 42);
assert.strictEqual(result.configPath, join(dir, 'apophis.config.js'));
cleanup(dir);
});
test('loadConfig loads from package.json field', async () => {
const dir = createTempDir();
writeFileSync(
join(dir, 'package.json'),
JSON.stringify({ name: 'test', apophis: { mode: 'observe' } }),
);
const result = await loadConfig({ cwd: dir });
assert.strictEqual(result.config.mode, 'observe');
assert.strictEqual(result.configPath, join(dir, 'package.json'));
cleanup(dir);
});
test('loadConfig rejects unknown keys', async () => {
const dir = createTempDir();
writeFileSync(
join(dir, 'apophis.config.json'),
JSON.stringify({ mode: 'verify', badKey: true }),
);
await assert.rejects(
loadConfig({ cwd: dir }),
(err: unknown) => {
assert.ok(err instanceof ConfigValidationError);
assert.strictEqual((err as ConfigValidationError).path, 'badKey');
return true;
},
);
cleanup(dir);
});
test('loadConfig resolves profile', async () => {
const dir = createTempDir();
writeFileSync(
join(dir, 'apophis.config.json'),
JSON.stringify({
presets: { safe: { mode: 'verify' } },
profiles: { quick: { preset: 'safe', seed: 99 } },
}),
);
const result = await loadConfig({ cwd: dir, profileName: 'quick' });
assert.strictEqual(result.config.mode, 'verify');
assert.strictEqual(result.config.seed, 99);
assert.strictEqual(result.profileName, 'quick');
assert.strictEqual(result.presetName, 'safe');
cleanup(dir);
});
test('loadConfig returns empty config when nothing found', async () => {
const dir = createTempDir();
const result = await loadConfig({ cwd: dir });
assert.deepStrictEqual(result.config, {});
assert.strictEqual(result.configPath, null);
assert.strictEqual(result.isMonorepo, false);
cleanup(dir);
});
test('loadConfig uses explicit --config path', async () => {
const dir = createTempDir();
const subdir = join(dir, 'sub');
mkdirSync(subdir);
writeFileSync(
join(subdir, 'custom.config.js'),
'export default { mode: "qualify" }',
);
const result = await loadConfig({ cwd: dir, configPath: 'sub/custom.config.js' });
assert.strictEqual(result.config.mode, 'qualify');
cleanup(dir);
});
+901
View File
@@ -0,0 +1,901 @@
/**
* Config loader for APOPHIS CLI.
*
* Responsibilities:
* - Config file discovery (.js, .ts, .json, or "apophis" field in package.json)
* - Config loading with tsx for .ts files
* - Profile resolution from config.profiles
* - Preset resolution and application
* - Environment-specific overrides
* - Unknown-key hard failure with exact path
* - Monorepo boundary detection
*/
import { readFileSync, existsSync, readdirSync, statSync } from 'node:fs';
import { resolve, dirname } from 'node:path';
import { pathToFileURL } from 'node:url';
import { resolvePacks } from '../../protocol-packs/index.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface Config {
mode?: 'verify' | 'observe' | 'qualify';
profile?: string;
preset?: string;
routes?: string[];
seed?: number;
artifactDir?: string;
environments?: Record<string, EnvironmentPolicy>;
profiles?: Record<string, ProfileDefinition>;
presets?: Record<string, PresetDefinition>;
generationProfiles?: Record<string, 'quick' | 'standard' | 'thorough' | { base: 'quick' | 'standard' | 'thorough' }>;
[key: string]: unknown;
}
export interface EnvironmentPolicy {
allowedModes?: ('verify' | 'observe' | 'qualify')[];
blockQualify?: boolean;
allowChaosOnProtected?: boolean;
[key: string]: unknown;
}
export interface ProfileDefinition {
preset?: string;
routes?: string[];
seed?: number;
features?: string[];
[key: string]: unknown;
}
export interface PresetDefinition {
mode?: 'verify' | 'observe' | 'qualify';
routes?: string[];
seed?: number;
features?: string[];
[key: string]: unknown;
}
export interface LoadConfigOptions {
cwd: string;
configPath?: string;
profileName?: string;
env?: string;
}
export interface LoadConfigResult {
config: Config;
configPath: string | null;
isMonorepo: boolean;
profileName: string | null;
presetName: string | null;
}
// ---------------------------------------------------------------------------
// Schema definition (TypeBox-style, plain TS for now)
// ---------------------------------------------------------------------------
interface SchemaField {
type: 'string' | 'number' | 'boolean' | 'array' | 'object';
optional?: boolean;
items?: SchemaField;
properties?: Record<string, SchemaField>;
enumValues?: string[];
min?: number;
}
// Schema for top-level config keys
const CONFIG_SCHEMA: Record<string, SchemaField> = {
mode: { type: 'string', optional: true, enumValues: ['verify', 'observe', 'qualify'] },
profile: { type: 'string', optional: true },
preset: { type: 'string', optional: true },
routes: { type: 'array', optional: true, items: { type: 'string' } },
seed: { type: 'number', optional: true },
artifactDir: { type: 'string', optional: true },
environments: {
type: 'object',
optional: true,
},
profiles: {
type: 'object',
optional: true,
properties: {},
},
presets: {
type: 'object',
optional: true,
properties: {},
},
generationProfiles: {
type: 'object',
optional: true,
properties: {},
},
packs: {
type: 'array',
optional: true,
items: { type: 'string' },
},
};
// Schema for EnvironmentPolicy values (inside environments.<name>)
const ENVIRONMENT_POLICY_SCHEMA: Record<string, SchemaField> = {
name: { type: 'string', optional: false },
allowVerify: { type: 'boolean', optional: true },
allowObserve: { type: 'boolean', optional: true },
allowQualify: { type: 'boolean', optional: true },
allowChaos: { type: 'boolean', optional: true },
allowBlocking: { type: 'boolean', optional: true },
requireSink: { type: 'boolean', optional: true },
allowedModes: { type: 'array', optional: true, items: { type: 'string' } },
blockQualify: { type: 'boolean', optional: true },
allowChaosOnProtected: { type: 'boolean', optional: true },
};
// Schema for ProfileDefinition values (inside profiles.<name>)
const PROFILE_SCHEMA: Record<string, SchemaField> = {
name: { type: 'string', optional: false },
mode: { type: 'string', optional: true, enumValues: ['verify', 'observe', 'qualify'] },
preset: { type: 'string', optional: true },
routes: { type: 'array', optional: true, items: { type: 'string' } },
seed: { type: 'number', optional: true },
artifactDir: { type: 'string', optional: true },
environment: { type: 'string', optional: true },
features: { type: 'array', optional: true, items: { type: 'string' } },
sampling: { type: 'number', optional: true },
blocking: { type: 'boolean', optional: true },
sinks: { type: 'object', optional: true },
};
// Schema for PresetDefinition values (inside presets.<name>)
const PRESET_SCHEMA: Record<string, SchemaField> = {
name: { type: 'string', optional: false },
depth: { type: 'string', optional: true, enumValues: ['quick', 'standard', 'deep'] },
timeout: { type: 'number', optional: true, min: 0 },
parallel: { type: 'boolean', optional: true },
chaos: { type: 'boolean', optional: true },
observe: { type: 'boolean', optional: true },
features: { type: 'array', optional: true, items: { type: 'string' } },
sampling: { type: 'number', optional: true },
blocking: { type: 'boolean', optional: true },
sinks: { type: 'object', optional: true },
};
const GENERATION_PROFILE_ALIAS_SCHEMA: Record<string, SchemaField> = {
base: { type: 'string', optional: false, enumValues: ['quick', 'standard', 'thorough'] },
}
// ---------------------------------------------------------------------------
// Config discovery
// ---------------------------------------------------------------------------
const CONFIG_FILES = [
'apophis.config.js',
'apophis.config.ts',
'apophis.config.json',
];
/**
* Discover config file in cwd or return null.
*/
export function discoverConfig(cwd: string): string | null {
for (const file of CONFIG_FILES) {
const fullPath = resolve(cwd, file);
if (existsSync(fullPath)) {
return fullPath;
}
}
return null;
}
/**
* Load package.json and check for "apophis" field.
*/
export function loadPackageJsonConfig(cwd: string): { config: Config | null; path: string | null } {
const pkgPath = resolve(cwd, 'package.json');
if (!existsSync(pkgPath)) {
return { config: null, path: null };
}
const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
if (pkg.apophis && typeof pkg.apophis === 'object') {
return { config: pkg.apophis as Config, path: pkgPath };
}
return { config: null, path: pkgPath };
}
// ---------------------------------------------------------------------------
// Config loading
// ---------------------------------------------------------------------------
/**
* Load a config file by path.
* Supports .js, .ts (via dynamic import, assumes tsx available), and .json.
*/
export async function loadConfigFile(configPath: string): Promise<Config> {
if (configPath.endsWith('.json')) {
const content = readFileSync(configPath, 'utf-8');
return JSON.parse(content) as Config;
}
// For .js and .ts, use dynamic import.
// tsx handles .ts files in dev environments.
const fileUrl = pathToFileURL(configPath).href;
const mod = await import(fileUrl);
// Support both default export and direct export
const config = mod.default ?? mod;
if (!config || typeof config !== 'object') {
throw new Error(`Config file at ${configPath} must export an object`);
}
return config as Config;
}
// ---------------------------------------------------------------------------
// Schema validation (unknown-key rejection)
// ---------------------------------------------------------------------------
export class ConfigValidationError extends Error {
constructor(
message: string,
public readonly path: string,
public readonly key: string,
public readonly value?: unknown,
public readonly guidance?: string,
) {
super(message);
this.name = 'ConfigValidationError';
}
}
/**
* Get the appropriate schema for a dynamic container's child objects.
* Returns the schema to validate values inside profiles.<name>, presets.<name>, or environments.<name>.
*/
function getDynamicContainerSchema(path: string): Record<string, SchemaField> | null {
if (path === 'profiles') return PROFILE_SCHEMA;
if (path === 'presets') return PRESET_SCHEMA;
if (path === 'environments') return ENVIRONMENT_POLICY_SCHEMA;
if (path === 'generationProfiles') return GENERATION_PROFILE_ALIAS_SCHEMA;
return null;
}
/**
* Check if a path is inside a dynamic container (e.g., profiles.foo, presets.bar).
*/
function isInsideDynamicContainer(path: string): boolean {
return path.startsWith('profiles.') || path.startsWith('presets.') || path.startsWith('environments.') || path.startsWith('generationProfiles.');
}
/**
* Validate that a value matches the expected type for a schema field.
* Throws ConfigValidationError on type mismatch.
*/
function validateType(
fieldValue: unknown,
fieldSchema: SchemaField,
currentPath: string,
key: string,
): void {
// Null/undefined is only valid if optional
if (fieldValue === null || fieldValue === undefined) {
if (!fieldSchema.optional) {
throw new ConfigValidationError(
`Missing required config key at ${currentPath}`,
currentPath,
key,
fieldValue,
`This field is required. Provide a ${fieldSchema.type} value.`,
);
}
return;
}
const actualType = Array.isArray(fieldValue) ? 'array' : typeof fieldValue;
if (actualType !== fieldSchema.type) {
throw new ConfigValidationError(
`Invalid type at ${currentPath}: expected ${fieldSchema.type}, got ${actualType}`,
currentPath,
key,
fieldValue,
`Expected ${fieldSchema.type}. Received ${actualType === 'object' ? JSON.stringify(fieldValue) : String(fieldValue)}.`,
);
}
// Validate enum values
if (fieldSchema.enumValues && fieldSchema.type === 'string' && typeof fieldValue === 'string') {
if (!fieldSchema.enumValues.includes(fieldValue)) {
throw new ConfigValidationError(
`Invalid value at ${currentPath}: "${fieldValue}" is not a valid ${key}. Allowed: ${fieldSchema.enumValues.join(', ')}`,
currentPath,
key,
fieldValue,
`Must be one of: ${fieldSchema.enumValues.join(', ')}.`,
);
}
}
// Validate numeric constraints
if (fieldSchema.type === 'number' && typeof fieldValue === 'number') {
if (fieldSchema.min !== undefined && fieldValue < fieldSchema.min) {
throw new ConfigValidationError(
`Invalid value at ${currentPath}: ${fieldValue} is less than minimum ${fieldSchema.min}`,
currentPath,
key,
fieldValue,
`Must be >= ${fieldSchema.min}.`,
);
}
}
// Validate array item types
if (fieldSchema.type === 'array' && Array.isArray(fieldValue) && fieldSchema.items) {
for (let i = 0; i < fieldValue.length; i++) {
const item = fieldValue[i];
const itemPath = `${currentPath}[${i}]`;
const itemType = Array.isArray(item) ? 'array' : typeof item;
if (itemType !== fieldSchema.items.type) {
throw new ConfigValidationError(
`Invalid type at ${itemPath}: expected ${fieldSchema.items.type}, got ${itemType}`,
itemPath,
`${key}[${i}]`,
item,
`Array items must be ${fieldSchema.items.type}. Received ${itemType === 'object' ? JSON.stringify(item) : String(item)}.`,
);
}
}
}
}
/**
* Recursively validate an object against a schema.
* Checks:
* - Unknown keys (hard failure)
* - Type mismatches (hard failure)
* - Enum value violations (hard failure)
* - Array item type mismatches (hard failure)
* - Numeric constraints (hard failure)
*
* Throws ConfigValidationError on any validation failure.
*/
export function validateConfigAgainstSchema(
value: unknown,
schema: Record<string, SchemaField>,
path: string = '',
): void {
if (value === null || typeof value !== 'object') {
return;
}
const obj = value as Record<string, unknown>;
for (const key of Object.keys(obj)) {
const currentPath = path ? `${path}.${key}` : key;
const fieldSchema = schema[key];
// Handle dynamic containers: profiles, presets, environments
// The keys are user-defined names; their values have specific schemas
const isDynamicContainer = path === 'profiles' || path === 'presets' || path === 'environments' || path === 'generationProfiles';
if (!fieldSchema && isDynamicContainer) {
const childSchema = getDynamicContainerSchema(path);
const fieldValue = obj[key];
if (path === 'generationProfiles' && typeof fieldValue === 'string') {
validateType(
fieldValue,
{ type: 'string', optional: false, enumValues: ['quick', 'standard', 'thorough'] },
currentPath,
key,
);
} else if (childSchema && fieldValue !== null && typeof fieldValue === 'object') {
// Validate the dynamic container value against its specific schema
validateConfigAgainstSchema(fieldValue, childSchema, currentPath);
} else if (childSchema) {
// Value is a primitive inside a dynamic container — type check it
validateType(fieldValue, { type: 'object', optional: false }, currentPath, key);
}
continue;
}
// Handle nested keys inside profile/preset/environment objects
if (!fieldSchema && isInsideDynamicContainer(path)) {
const parentContainer = path.split('.')[0] || '';
const childSchema = getDynamicContainerSchema(parentContainer);
if (childSchema) {
const nestedSchema = childSchema[key];
if (nestedSchema) {
const fieldValue = obj[key];
validateType(fieldValue, nestedSchema, currentPath, key);
// Recurse into nested objects
if (nestedSchema.type === 'object' && fieldValue !== null && typeof fieldValue === 'object') {
if (nestedSchema.properties) {
validateConfigAgainstSchema(fieldValue, nestedSchema.properties, currentPath);
}
}
if (nestedSchema.type === 'array' && Array.isArray(fieldValue) && nestedSchema.items?.properties) {
for (let i = 0; i < fieldValue.length; i++) {
const item = fieldValue[i];
if (item !== null && typeof item === 'object') {
validateConfigAgainstSchema(item, nestedSchema.items.properties, `${currentPath}[${i}]`);
}
}
}
} else {
// Unknown key inside a profile/preset/environment object
throw new ConfigValidationError(
`Unknown config key at ${currentPath}`,
currentPath,
key,
obj[key],
`Valid keys for ${parentContainer} entries: ${Object.keys(childSchema || {}).join(', ')}.`,
);
}
}
continue;
}
if (!fieldSchema) {
throw new ConfigValidationError(
`Unknown config key at ${currentPath}`,
currentPath,
key,
obj[key],
`Valid top-level keys: ${Object.keys(CONFIG_SCHEMA).join(', ')}.`,
);
}
const fieldValue = obj[key];
// Validate type for known fields
validateType(fieldValue, fieldSchema, currentPath, key);
// Recurse into objects with known properties
if (fieldSchema.type === 'object') {
if (fieldValue !== null && typeof fieldValue === 'object') {
if (fieldSchema.properties) {
validateConfigAgainstSchema(fieldValue, fieldSchema.properties, currentPath);
} else {
// For objects without explicit properties (like profiles/presets/environments),
// we still recurse to validate nested objects, but we pass the same schema
// and the skip logic above will handle dynamic container keys
validateConfigAgainstSchema(fieldValue, schema, currentPath);
}
}
}
// Recurse into array items if they are objects
if (fieldSchema.type === 'array' && fieldSchema.items && Array.isArray(fieldValue)) {
for (let i = 0; i < fieldValue.length; i++) {
const item = fieldValue[i];
if (item !== null && typeof item === 'object' && fieldSchema.items.properties) {
validateConfigAgainstSchema(item, fieldSchema.items.properties, `${currentPath}[${i}]`);
}
}
}
}
}
// ---------------------------------------------------------------------------
// Monorepo detection
// ---------------------------------------------------------------------------
/**
* Check if cwd is inside a monorepo (has workspaces in root package.json).
*/
export function detectMonorepo(cwd: string): boolean {
let current = cwd;
while (current !== dirname(current)) {
const pkgPath = resolve(current, 'package.json');
if (existsSync(pkgPath)) {
try {
const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
if (pkg.workspaces && Array.isArray(pkg.workspaces)) {
return true;
}
// Also check for pnpm workspaces
const pnpmWorkspacePath = resolve(current, 'pnpm-workspace.yaml');
if (existsSync(pnpmWorkspacePath)) {
return true;
}
} catch {
// Ignore parse errors
}
// Stop at first package.json found
return false;
}
current = dirname(current);
}
return false;
}
/**
* Find all workspace package directories under cwd.
* Supports npm workspaces (package.json workspaces field) and pnpm-workspace.yaml.
* Returns absolute paths to each package directory.
*/
export function findWorkspacePackages(cwd: string): string[] {
let root = cwd;
while (root !== dirname(root)) {
const pkgPath = resolve(root, 'package.json');
if (existsSync(pkgPath)) {
try {
const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
if (pkg.workspaces && Array.isArray(pkg.workspaces)) {
return expandWorkspacePatterns(root, pkg.workspaces);
}
} catch {
// Ignore parse errors
}
// Check for pnpm-workspace.yaml
const pnpmWorkspacePath = resolve(root, 'pnpm-workspace.yaml');
if (existsSync(pnpmWorkspacePath)) {
const patterns = parsePnpmWorkspaceYaml(pnpmWorkspacePath);
return expandWorkspacePatterns(root, patterns);
}
// Stop at first package.json found
return [];
}
root = dirname(root);
}
return [];
}
function expandWorkspacePatterns(root: string, patterns: string[]): string[] {
const packages: string[] = [];
for (const pattern of patterns) {
if (pattern.endsWith('/*')) {
const dir = pattern.slice(0, -2);
const dirPath = resolve(root, dir);
if (existsSync(dirPath)) {
const entries = readdirSync(dirPath, { withFileTypes: true });
for (const entry of entries) {
if (entry.isDirectory()) {
packages.push(resolve(dirPath, entry.name));
}
}
}
} else {
const exactPath = resolve(root, pattern);
if (existsSync(exactPath)) {
const stat = statSync(exactPath);
if (stat.isDirectory()) {
packages.push(exactPath);
}
}
}
}
return packages;
}
function parsePnpmWorkspaceYaml(yamlPath: string): string[] {
try {
const content = readFileSync(yamlPath, 'utf-8');
const patterns: string[] = [];
for (const line of content.split('\n')) {
const trimmed = line.trim();
if (trimmed.startsWith('- ')) {
patterns.push(trimmed.slice(2).trim());
}
}
return patterns;
} catch {
return [];
}
}
// ---------------------------------------------------------------------------
// Semantic validation (cross-references, value constraints)
// ---------------------------------------------------------------------------
/**
* Validate semantic constraints that go beyond schema types:
* - Profile references nonexistent preset
* - Environment policy references unknown mode
* - Timeout is a positive number
* - Routes array elements are non-empty strings
* - Seed is an integer
*
* Throws ConfigValidationError on any semantic violation.
*/
export function validateConfigSemantics(config: Config): void {
// Validate profile references
if (config.profiles) {
for (const [profileName, profile] of Object.entries(config.profiles)) {
if (profile.preset) {
const availablePresets = config.presets ? Object.keys(config.presets) : [];
if (!config.presets || !(profile.preset in config.presets)) {
throw new ConfigValidationError(
`Profile "${profileName}" references unknown preset "${profile.preset}"`,
`profiles.${profileName}.preset`,
'preset',
profile.preset,
`Available presets: ${availablePresets.join(', ') || 'none'}. Define preset "${profile.preset}" in config.presets.`,
);
}
}
}
}
// Validate preset values
if (config.presets) {
for (const [presetName, preset] of Object.entries(config.presets)) {
if (preset.timeout !== undefined) {
if (typeof preset.timeout !== 'number' || preset.timeout < 0) {
throw new ConfigValidationError(
`Preset "${presetName}" has invalid timeout: ${preset.timeout}`,
`presets.${presetName}.timeout`,
'timeout',
preset.timeout,
`Timeout must be a non-negative number (milliseconds).`,
);
}
}
if (preset.depth !== undefined) {
const validDepths = ['quick', 'standard', 'deep'];
const depthValue = preset.depth;
if (typeof depthValue === 'string' && !validDepths.includes(depthValue as string)) {
throw new ConfigValidationError(
`Preset "${presetName}" has invalid depth: "${depthValue}"`,
`presets.${presetName}.depth`,
'depth',
depthValue,
`Must be one of: ${validDepths.join(', ')}.`,
);
}
}
}
}
// Validate environment policy allowedModes
if (config.environments) {
for (const [envName, envPolicy] of Object.entries(config.environments)) {
if (envPolicy.allowedModes) {
const validModes = ['verify', 'observe', 'qualify'];
for (const mode of envPolicy.allowedModes) {
if (!validModes.includes(mode)) {
throw new ConfigValidationError(
`Environment "${envName}" has invalid allowedMode: "${mode}"`,
`environments.${envName}.allowedModes`,
'allowedModes',
mode,
`Allowed modes must be one of: ${validModes.join(', ')}.`,
);
}
}
}
}
}
// Validate routes are non-empty strings
if (config.routes) {
for (let i = 0; i < config.routes.length; i++) {
const route = config.routes[i];
if (typeof route !== 'string' || route.trim().length === 0) {
throw new ConfigValidationError(
`Invalid route at routes[${i}]: ${JSON.stringify(route)}`,
`routes[${i}]`,
'routes',
route,
`Routes must be non-empty strings like "GET /users" or "POST /api/items".`,
);
}
}
}
// Validate seed is an integer
if (config.seed !== undefined) {
if (typeof config.seed !== 'number' || !Number.isInteger(config.seed)) {
throw new ConfigValidationError(
`Invalid seed: ${config.seed}`,
'seed',
'seed',
config.seed,
`Seed must be an integer number.`,
);
}
}
}
// ---------------------------------------------------------------------------
// Profile and preset resolution
// ---------------------------------------------------------------------------
/**
* Resolve profile from config.profiles.
* Returns merged config: preset defaults + profile overrides.
*/
export function resolveProfile(
config: Config,
profileName: string | undefined,
): { config: Config; profileName: string | null; presetName: string | null } {
if (!profileName) {
return { config, profileName: null, presetName: config.preset ?? null };
}
const profiles = config.profiles ?? {};
const profile = profiles[profileName];
if (!profile) {
const available = Object.keys(profiles).join(', ');
throw new Error(
`Unknown profile "${profileName}". Available profiles: ${available || 'none'}.`,
);
}
// Start with preset if profile references one
let merged: Config = { ...config };
let presetName: string | null = null;
if (profile.preset && config.presets) {
const preset = config.presets[profile.preset];
if (preset) {
merged = { ...merged, ...preset };
presetName = profile.preset;
}
}
// Apply profile overrides
merged = {
...merged,
...profile,
// Don't overwrite the top-level preset with the profile's preset string
preset: profile.preset ? undefined : merged.preset,
};
// Clean up undefined values
if (merged.preset === undefined) {
delete merged.preset;
}
return { config: merged, profileName, presetName };
}
// ---------------------------------------------------------------------------
// Environment-specific overrides
// ---------------------------------------------------------------------------
/**
* Apply environment-specific policy overrides.
*/
export function applyEnvironmentOverrides(
config: Config,
env: string | undefined,
): Config {
if (!env || !config.environments) {
return config;
}
const envPolicy = config.environments[env];
if (!envPolicy) {
return config;
}
// Environment policy doesn't override config values directly,
// but we merge it for policy engine consumption
return {
...config,
environments: {
...config.environments,
[env]: envPolicy,
},
};
}
// ---------------------------------------------------------------------------
// Main entry point
// ---------------------------------------------------------------------------
/**
* Load and resolve config for the CLI.
*
* Discovery order:
* 1. --config override
* 2. apophis.config.js
* 3. apophis.config.ts
* 4. apophis.config.json
* 5. "apophis" field in package.json
*
* Resolution order:
* 1. Load raw config
* 2. Validate against schema (unknown keys = hard failure)
* 3. Resolve profile (preset defaults + profile overrides)
* 4. Apply environment-specific overrides
* 5. Detect monorepo
*/
export async function loadConfig(options: LoadConfigOptions): Promise<LoadConfigResult> {
const { cwd, configPath: explicitPath, profileName, env } = options;
let configPath: string | null = null;
let rawConfig: Config;
// 1. Explicit --config override
if (explicitPath) {
const resolvedPath = resolve(cwd, explicitPath);
if (!existsSync(resolvedPath)) {
throw new Error(`Config file not found: ${resolvedPath}`);
}
configPath = resolvedPath;
rawConfig = await loadConfigFile(resolvedPath);
} else {
// 2. Discover config file
const discoveredPath = discoverConfig(cwd);
if (discoveredPath) {
configPath = discoveredPath;
rawConfig = await loadConfigFile(discoveredPath);
} else {
// 3. Check package.json "apophis" field
const pkgConfig = loadPackageJsonConfig(cwd);
if (pkgConfig.config) {
configPath = pkgConfig.path;
rawConfig = pkgConfig.config;
} else {
// No config found
return {
config: {},
configPath: null,
isMonorepo: detectMonorepo(cwd),
profileName: null,
presetName: null,
};
}
}
}
// 4. Resolve protocol packs if specified
if (rawConfig.packs && Array.isArray(rawConfig.packs) && rawConfig.packs.length > 0) {
const packFragment = resolvePacks(rawConfig.packs as string[], {
seed: rawConfig.seed,
});
rawConfig = {
...packFragment,
...rawConfig,
profiles: {
...packFragment.profiles,
...rawConfig.profiles,
},
presets: {
...packFragment.presets,
...rawConfig.presets,
},
environments: {
...packFragment.environments,
...rawConfig.environments,
},
};
}
// 5. Validate against schema (unknown keys = hard failure with exact path)
validateConfigAgainstSchema(rawConfig, CONFIG_SCHEMA);
// 5b. Validate semantic constraints (cross-references, value constraints)
validateConfigSemantics(rawConfig);
// 5. Resolve profile and preset
const { config: profiledConfig, profileName: resolvedProfile, presetName } = resolveProfile(
rawConfig,
profileName,
);
// 6. Apply environment overrides
const envConfig = applyEnvironmentOverrides(profiledConfig, env);
// 7. Detect monorepo
const isMonorepo = detectMonorepo(cwd);
return {
config: envConfig,
configPath,
isMonorepo,
profileName: resolvedProfile,
presetName,
};
}
// ---------------------------------------------------------------------------
// Re-export for convenience
// ---------------------------------------------------------------------------
export { CONFIG_SCHEMA };
+130
View File
@@ -0,0 +1,130 @@
import { resolve } from 'node:path';
import type { CliContext } from './types.js';
export type { CliContext } from './types.js';
import { existsSync, readFileSync } from 'node:fs';
function detectPackageManager(cwd: string): CliContext['packageManager'] {
// Check for lock files in cwd
if (existsSync(resolve(cwd, 'bun.lockb')) || existsSync(resolve(cwd, 'bun.lock'))) {
return 'bun';
}
if (existsSync(resolve(cwd, 'pnpm-lock.yaml'))) {
return 'pnpm';
}
if (existsSync(resolve(cwd, 'yarn.lock'))) {
return 'yarn';
}
if (existsSync(resolve(cwd, 'package-lock.json'))) {
return 'npm';
}
// Check package.json packageManager field
const packageJsonPath = resolve(cwd, 'package.json');
if (existsSync(packageJsonPath)) {
try {
const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf-8')) as { packageManager?: string };
const packageManager = packageJson.packageManager || '';
if (packageManager.startsWith('bun@')) return 'bun';
if (packageManager.startsWith('pnpm@')) return 'pnpm';
if (packageManager.startsWith('yarn@')) return 'yarn';
if (packageManager.startsWith('npm@')) return 'npm';
} catch {
// Ignore parse errors
}
}
// Check environment variables
if (process.env.npm_config_user_agent) {
const ua = process.env.npm_config_user_agent;
if (ua.includes('bun')) return 'bun';
if (ua.includes('pnpm')) return 'pnpm';
if (ua.includes('yarn')) return 'yarn';
if (ua.includes('npm')) return 'npm';
}
return 'unknown';
}
function detectCI(): boolean {
const ciEnvVars = [
'CI',
'GITHUB_ACTIONS',
'GITLAB_CI',
'CIRCLECI',
'TRAVIS',
'APPVEYOR',
'BUILDKITE',
'DRONE',
'JENKINS_URL',
'TF_BUILD',
'CODEBUILD_BUILD_ID',
'TEAMCITY_VERSION',
'SEMAPHORE',
'WERCKER',
'MAGNUM',
'SNAP_CI',
'BUDDY',
'BUILDBOX',
'AGOLA',
'WOODPECKER',
];
return ciEnvVars.some(varName => process.env[varName] !== undefined);
}
export function createContext(options: Record<string, unknown> = {}): CliContext {
// Detect cwd (respect --cwd override)
const cwd = typeof options.cwd === 'string'
? resolve(options.cwd)
: process.cwd();
// Detect environment
const nodeEnv = process.env.NODE_ENV;
const apophisEnv = process.env.APOPHIS_ENV;
// Detect TTY
const isTTY = process.stdout.isTTY === true;
// Detect CI
const isCI = detectCI();
// Package manager detection
const packageManager = detectPackageManager(cwd);
// Normalize options
const format = options.format === 'json' || options.format === 'ndjson'
? options.format
: 'human';
const color = options.color === 'always' || options.color === 'never'
? options.color
: 'auto';
const generationProfile = typeof options.generationProfile === 'string'
? options.generationProfile
: undefined;
return {
cwd,
env: {
nodeEnv,
apophisEnv,
},
isTTY,
isCI,
nodeVersion: process.version,
packageManager,
selfPath: process.argv[1],
options: {
config: typeof options.config === 'string' ? options.config : undefined,
profile: typeof options.profile === 'string' ? options.profile : undefined,
generationProfile,
format,
color,
quiet: options.quiet === true,
verbose: options.verbose === true,
artifactDir: typeof options.artifactDir === 'string' ? options.artifactDir : undefined,
},
};
}
+74
View File
@@ -0,0 +1,74 @@
/**
* E0-3 / E6-1 Error Taxonomy and Precedence
*
* Taxonomic classes for failures encountered during CLI execution.
* Precedence is lowest-numbered wins (parse before discovery before runtime).
*/
export const ErrorTaxonomy = {
PARSE: 'parse',
IMPORT: 'import',
LOAD: 'load',
DISCOVERY: 'discovery',
RUNTIME: 'runtime',
USAGE: 'usage',
} as const;
export type ErrorCategory = (typeof ErrorTaxonomy)[keyof typeof ErrorTaxonomy];
/** Precedence order: lower index = higher priority. */
export const PRECEDENCE: readonly ErrorCategory[] = [
ErrorTaxonomy.PARSE,
ErrorTaxonomy.IMPORT,
ErrorTaxonomy.LOAD,
ErrorTaxonomy.DISCOVERY,
ErrorTaxonomy.USAGE,
ErrorTaxonomy.RUNTIME,
] as const;
/** Map a raw Error or string to its taxonomic category. */
export function classifyError(err: unknown): ErrorCategory {
const msg = err instanceof Error ? err.message : String(err);
const lower = msg.toLowerCase();
if (lower.includes('parse') || lower.includes('syntax') || lower.includes('unexpected token')) {
return ErrorTaxonomy.PARSE;
}
if (lower.includes('import') || lower.includes('cannot find module') || lower.includes('module not found')) {
return ErrorTaxonomy.IMPORT;
}
if (lower.includes('load') || lower.includes('config') || lower.includes('profile') || lower.includes('cannot read')) {
return ErrorTaxonomy.LOAD;
}
if (lower.includes('discovery') || lower.includes('duplicate') || lower.includes('already added') || lower.includes('decorator')) {
return ErrorTaxonomy.DISCOVERY;
}
if (lower.includes('usage') || lower.includes('argument') || lower.includes('flag') || lower.includes('unknown option') || lower.includes('required')) {
return ErrorTaxonomy.USAGE;
}
return ErrorTaxonomy.RUNTIME;
}
/** Return the highest-precedence (most important) category from a set. */
export function highestPrecedence(categories: ErrorCategory[]): ErrorCategory | undefined {
if (categories.length === 0) return undefined;
return categories.reduce((best, cat) => {
const bestIdx = PRECEDENCE.indexOf(best);
const catIdx = PRECEDENCE.indexOf(cat);
return catIdx < bestIdx ? cat : best;
});
}
/** Attach taxonomy to any diagnostic shape. */
export interface TaxonomicDiagnostic {
category: ErrorCategory;
message: string;
details?: string;
remediation?: string;
}
export function makeDiagnostic(err: unknown, overrideCategory?: ErrorCategory): TaxonomicDiagnostic {
const category = overrideCategory ?? classifyError(err);
const message = err instanceof Error ? err.message : String(err);
return { category, message };
}
+10
View File
@@ -0,0 +1,10 @@
/**
* S0: Spec Authority - Exit code constants
* Frozen contract. All implementation streams must use these constants.
*/
export const SUCCESS = 0;
export const BEHAVIORAL_FAILURE = 1;
export const USAGE_ERROR = 2;
export const INTERNAL_ERROR = 3;
export const INTERRUPTED = 130;
+51
View File
@@ -0,0 +1,51 @@
import type { Config } from './config-loader.js'
export type ResolvedGenerationProfile = 'quick' | 'standard' | 'thorough'
export class GenerationProfileResolutionError extends Error {
constructor(message: string) {
super(message)
this.name = 'GenerationProfileResolutionError'
}
}
function isBuiltInProfile(value: string): value is ResolvedGenerationProfile {
return value === 'quick' || value === 'standard' || value === 'thorough'
}
export function resolveGenerationProfileOverride(
rawProfile: string | undefined,
config: Config,
): ResolvedGenerationProfile | undefined {
if (!rawProfile) {
return undefined
}
if (isBuiltInProfile(rawProfile)) {
return rawProfile
}
const aliases = config.generationProfiles
if (!aliases) {
throw new GenerationProfileResolutionError(
`Unknown generation profile "${rawProfile}". Use one of: quick, standard, thorough, or define an alias in config.generationProfiles.`,
)
}
const alias = aliases[rawProfile]
if (!alias) {
const available = Object.keys(aliases).join(', ') || 'none'
throw new GenerationProfileResolutionError(
`Unknown generation profile "${rawProfile}". Built-ins: quick, standard, thorough. Config aliases: ${available}.`,
)
}
const target = typeof alias === 'string' ? alias : alias.base
if (!isBuiltInProfile(target)) {
throw new GenerationProfileResolutionError(
`Invalid generation profile alias "${rawProfile}". Alias must resolve to quick, standard, or thorough.`,
)
}
return target
}
+458
View File
@@ -0,0 +1,458 @@
import { cac } from 'cac';
import pc from 'picocolors';
import { createContext, type CliContext } from './context.js';
const CLI_VERSION = '2.0.0';
const HELP_HEADER = `
${pc.bold('apophis')} Contract-driven API testing for Fastify
${pc.dim('Usage:')}
apophis <command> [options]
${pc.dim('Commands:')}
init Scaffold config, scripts, and example usage
verify Run deterministic contract verification
observe Validate runtime observe configuration and reporting setup
qualify Run scenario, stateful, protocol, or chaos-driven qualification
replay Replay a failure using seed and stored trace
doctor Validate config, environment safety, docs/example correctness
migrate Check and rewrite deprecated config or API usage
${pc.dim('Global Options:')}
--config <path> Config file path
--profile <name> Profile name from config
--generation-profile <name> Generation budget profile (built-in or config alias)
--cwd <path> Working directory override
--format <mode> Output format: human | json | ndjson (default: human)
--color <mode> Color mode: auto | always | never (default: auto)
--quiet Suppress non-error output
--verbose Enable verbose logging
--artifact-dir <path> Directory for artifact output
--workspace Run command across all workspace packages
${pc.dim('Other:')}
-v, --version Show version number
-h, --help Show help
${pc.dim('Examples:')}
apophis init --preset safe-ci
apophis verify --profile quick --routes "POST /users"
apophis observe --profile staging-observe --check-config
apophis qualify --profile oauth-nightly --seed 42
apophis replay --artifact reports/apophis/failure-*.json
apophis doctor
apophis doctor --workspace
apophis migrate --dry-run
`;
function getCommandHelp(command: string): string {
const helps: Record<string, string> = {
init: `
${pc.bold('apophis init')} Scaffold config, scripts, and example usage
${pc.dim('Usage:')}
apophis init [options]
${pc.dim('Options:')}
--preset <name> Preset name (e.g. safe-ci, full)
--force Overwrite existing files
--noninteractive Skip all prompts, require explicit flags
${pc.dim('Examples:')}
apophis init --preset safe-ci
apophis init --force --noninteractive
`,
verify: `
${pc.bold('apophis verify')} Run deterministic contract verification
${pc.dim('Usage:')}
apophis verify [options]
${pc.dim('Options:')}
--profile <name> Profile name from config
--generation-profile <name> Generation budget profile (built-in or config alias)
--routes <filter> Route filter pattern
--seed <number> Deterministic seed
--changed Filter to git-modified routes
${pc.dim('Examples:')}
apophis verify --profile quick
apophis verify --routes "POST /users" --seed 42
apophis verify --changed
`,
observe: `
${pc.bold('apophis observe')} Validate runtime observe configuration and reporting setup
${pc.dim('Usage:')}
apophis observe [options]
${pc.dim('Options:')}
--profile <name> Profile name from config
--check-config Only validate, do not activate
${pc.dim('Examples:')}
apophis observe --profile staging-observe
apophis observe --check-config
`,
qualify: `
${pc.bold('apophis qualify')} Run scenario, stateful, protocol, or chaos-driven qualification
${pc.dim('Usage:')}
apophis qualify [options]
${pc.dim('Options:')}
--profile <name> Profile name from config
--generation-profile <name> Generation budget profile (built-in or config alias)
--seed <number> Deterministic seed
${pc.dim('Examples:')}
apophis qualify --profile oauth-nightly --seed 42
`,
replay: `
${pc.bold('apophis replay')} Replay a failure using seed and stored trace
${pc.dim('Usage:')}
apophis replay --artifact <path>
${pc.dim('Options:')}
--artifact <path> Path to failure artifact
${pc.dim('Examples:')}
apophis replay --artifact reports/apophis/failure-*.json
`,
doctor: `
${pc.bold('apophis doctor')} Validate config, environment safety, docs/example correctness
${pc.dim('Usage:')}
apophis doctor [options]
${pc.dim('Options:')}
--mode <mode> Focus checks on a mode: verify | observe | qualify
--strict Treat warnings as failures
${pc.dim('Examples:')}
apophis doctor
apophis doctor --mode verify
apophis doctor --strict
`,
migrate: `
${pc.bold('apophis migrate')} Check and rewrite deprecated config or API usage
${pc.dim('Usage:')}
apophis migrate [options]
${pc.dim('Options:')}
--check Detect legacy config without rewriting
--dry-run Show exact rewrites without writing
--write Perform rewrites
${pc.dim('Examples:')}
apophis migrate --check
apophis migrate --dry-run
apophis migrate --write
`,
};
return helps[command] || '';
}
function printInternalError(error: unknown): void {
console.error();
console.error(pc.red(' ╔══════════════════════════════════════════════════════════════╗'));
console.error(pc.red(' ║ INTERNAL APOPHIS ERROR ║'));
console.error(pc.red(' ╠══════════════════════════════════════════════════════════════╣'));
console.error(pc.red(`${String(error).slice(0, 56).padEnd(56)}`));
console.error(pc.red(' ╚══════════════════════════════════════════════════════════════╝'));
console.error();
console.error(pc.dim(' This is a bug in APOPHIS. Please report it with the full error'));
console.error(pc.dim(' message and the command you ran.'));
console.error();
}
function resolveRequestedFormat(argv: string[]): 'human' | 'json' | 'ndjson' {
for (let i = 0; i < argv.length; i++) {
const arg = argv[i];
if (!arg) continue;
if (arg === '--format' && argv[i + 1]) {
const value = argv[i + 1];
if (value === 'json' || value === 'ndjson') return value;
return 'human';
}
if (arg.startsWith('--format=')) {
const value = arg.slice('--format='.length);
if (value === 'json' || value === 'ndjson') return value;
return 'human';
}
}
return 'human';
}
function writeMachineRecord(
format: 'json' | 'ndjson',
payload: Record<string, unknown>,
): void {
if (format === 'json') {
process.stdout.write(JSON.stringify(payload, null, 2) + '\n');
return;
}
process.stdout.write(JSON.stringify(payload) + '\n');
}
type CommandName = 'init' | 'verify' | 'observe' | 'qualify' | 'replay' | 'doctor' | 'migrate';
type CommandHandler = (args: string[], ctx: CliContext) => Promise<number>;
const commandLoaders: Record<CommandName, () => Promise<CommandHandler>> = {
init: async () => (await import('../commands/init/index.js')).handleInit,
verify: async () => (await import('../commands/verify/index.js')).handleVerify,
observe: async () => (await import('../commands/observe/index.js')).handleObserve,
qualify: async () => (await import('../commands/qualify/index.js')).handleQualify,
replay: async () => (await import('../commands/replay/index.js')).handleReplay,
doctor: async () => (await import('../commands/doctor/index.js')).handleDoctor,
migrate: async () => (await import('../commands/migrate/index.js')).handleMigrate,
};
async function loadHandler(command: string): Promise<CommandHandler | undefined> {
const loader = commandLoaders[command as CommandName];
return loader ? loader() : undefined;
}
export async function main(argv: string[] = process.argv.slice(2)): Promise<number> {
const cli = cac('apophis');
const requestedFormat = resolveRequestedFormat(argv);
const machineMode = requestedFormat === 'json' || requestedFormat === 'ndjson';
// Global flags
cli.option('--config <path>', 'Config file path');
cli.option('--profile <name>', 'Profile name from config');
cli.option('--generation-profile <name>', 'Generation budget profile (built-in or config alias)');
cli.option('--cwd <path>', 'Working directory override');
cli.option('--format <mode>', 'Output format: human | json | ndjson', { default: 'human' });
cli.option('--color <mode>', 'Color mode: auto | always | never', { default: 'auto' });
cli.option('--quiet', 'Suppress non-error output');
cli.option('--verbose', 'Enable verbose logging');
cli.option('--artifact-dir <path>', 'Directory for artifact output');
cli.option('--workspace', 'Run command across all workspace packages');
// Version
cli.version(CLI_VERSION);
// Override help to use our custom format
// Note: cac's help() returns the CAC instance for chaining, but we just want to print
cli.help = () => {
console.log(HELP_HEADER);
return cli;
};
// Prevent cac from handling --version (we handle it manually)
// cac.version() registers --version but we intercept it before cac processes it
// Commands
const commands = [
'init',
'verify',
'observe',
'qualify',
'replay',
'doctor',
'migrate',
];
for (const command of commands) {
const cmd = cli.command(command, getCommandHelp(command).split('\n')[1]?.trim() || `${command} command`);
// Add command-specific options
switch (command) {
case 'init':
cmd.option('--preset <name>', 'Preset name (e.g. safe-ci, full)');
cmd.option('--force', 'Overwrite existing files');
cmd.option('--noninteractive', 'Skip all prompts, require explicit flags');
break;
case 'verify':
cmd.option('--profile <name>', 'Profile name from config');
cmd.option('--generation-profile <name>', 'Generation budget profile (built-in or config alias)');
cmd.option('--routes <filter>', 'Route filter pattern');
cmd.option('--seed <number>', 'Deterministic seed');
cmd.option('--changed', 'Filter to git-modified routes');
break;
case 'observe':
cmd.option('--profile <name>', 'Profile name from config');
cmd.option('--check-config', 'Only validate, do not activate');
break;
case 'qualify':
cmd.option('--profile <name>', 'Profile name from config');
cmd.option('--generation-profile <name>', 'Generation budget profile (built-in or config alias)');
cmd.option('--seed <number>', 'Deterministic seed');
break;
case 'replay':
cmd.option('--artifact <path>', 'Path to failure artifact');
break;
case 'doctor':
cmd.option('--mode <mode>', 'Focus checks on a specific mode: verify | observe | qualify');
cmd.option('--strict', 'Treat warnings as failures');
break;
case 'migrate':
cmd.option('--check', 'Detect legacy config without rewriting');
cmd.option('--dry-run', 'Show exact rewrites without writing');
cmd.option('--write', 'Perform rewrites');
break;
}
cmd.action(async (options) => {
const ctx = createContext(options);
const handler = await loadHandler(command);
if (!handler) {
console.error(pc.red(`Unknown command: ${command}`));
return 2;
}
// Pass raw argv so doctor/migrate can parse extra flags
const result = await handler(argv, ctx);
// Ensure we always return a number (cac may swallow undefined)
return typeof result === 'number' ? result : 0;
});
}
try {
// Handle --help globally (before parsing)
if (argv.includes('-h') || argv.includes('--help')) {
const commandArg = argv.find(arg => commands.includes(arg));
if (commandArg) {
const helpText = getCommandHelp(commandArg);
if (helpText) {
if (machineMode) {
writeMachineRecord(requestedFormat, {
command: commandArg,
help: helpText,
});
} else {
console.log(helpText);
}
return 0;
}
}
if (machineMode) {
writeMachineRecord(requestedFormat, { help: HELP_HEADER });
} else {
cli.help();
}
return 0;
}
// Handle --version (before parsing)
if (argv.includes('-v') || argv.includes('--version')) {
if (machineMode) {
writeMachineRecord(requestedFormat, { version: CLI_VERSION });
} else {
console.log(CLI_VERSION);
}
return 0;
}
// Check for unknown commands
const firstArg = argv[0];
if (firstArg && !firstArg.startsWith('-') && !commands.includes(firstArg)) {
if (machineMode) {
writeMachineRecord(requestedFormat, {
error: `Unknown command: ${firstArg}`,
availableCommands: commands,
next: 'Run "apophis --help" for usage information.',
});
} else {
console.error(pc.red(`Unknown command: ${firstArg}`));
console.error();
console.error(pc.dim('Available commands:'));
for (const cmd of commands) {
console.error(pc.dim(` ${cmd}`));
}
console.error();
console.error(pc.dim('Run "apophis --help" for usage information.'));
}
return 2;
}
// Handle unknown flags
const knownGlobalFlags = new Set([
'--config', '--profile', '--cwd', '--format', '--color',
'--generation-profile',
'--quiet', '--verbose', '--artifact-dir', '--workspace',
'-v', '--version', '-h', '--help',
]);
const commandSpecificFlags: Record<string, Set<string>> = {
init: new Set(['--preset', '--force', '--noninteractive']),
verify: new Set(['--profile', '--generation-profile', '--routes', '--seed', '--changed', '--workspace']),
observe: new Set(['--profile', '--check-config', '--workspace']),
qualify: new Set(['--profile', '--generation-profile', '--seed', '--workspace']),
replay: new Set(['--artifact']),
doctor: new Set(['--mode', '--strict', '--workspace']),
migrate: new Set(['--check', '--dry-run', '--write']),
};
const activeCommand = firstArg && commands.includes(firstArg) ? firstArg : undefined;
const activeCmdFlags = activeCommand ? commandSpecificFlags[activeCommand] : undefined;
const allowedFlags = activeCmdFlags
? new Set([...knownGlobalFlags, ...activeCmdFlags])
: knownGlobalFlags;
const unknownFlags: string[] = [];
for (const arg of argv) {
if (arg.startsWith('--') || (arg.startsWith('-') && arg.length > 1)) {
const flagName = arg.split('=')[0]!;
if (!allowedFlags.has(flagName)) {
unknownFlags.push(flagName);
}
}
}
if (unknownFlags.length > 0) {
if (machineMode) {
writeMachineRecord(requestedFormat, {
error: `Unknown flag: ${unknownFlags[0]}`,
next: 'Run "apophis --help" for available options.',
});
} else {
console.error(pc.red(`Unknown flag: ${unknownFlags[0]}`));
console.error();
console.error(pc.dim('Run "apophis --help" for available options.'));
}
return 2;
}
// If no command provided, show help
if (!firstArg || firstArg.startsWith('-')) {
if (machineMode) {
writeMachineRecord(requestedFormat, { help: HELP_HEADER });
} else {
cli.help();
}
return 0;
}
// Parse options for the command
const parsed = cli.parse(['node', 'apophis', ...argv], { run: false });
// Directly dispatch to handler (bypass cac's runMatchedCommand which has issues)
const handler = await loadHandler(firstArg);
if (!handler) {
console.error(pc.red(`Unknown command: ${firstArg}`));
return 2;
}
const ctx = createContext(parsed.options);
const result = await handler(argv, ctx);
return typeof result === 'number' ? result : 0;
} catch (error) {
if (machineMode) {
writeMachineRecord(requestedFormat, {
error: 'Internal APOPHIS error',
detail: String(error),
});
} else {
printInternalError(error);
}
return 3;
}
}
// src/cli/core/index.ts is the CLI logic module. The direct entrypoint is src/cli/index.ts.
// Do NOT add a direct main() call here — that belongs in the entrypoint file only.
+296
View File
@@ -0,0 +1,296 @@
/**
* Tests for policy-engine.ts
*/
import { test } from 'node:test';
import assert from 'node:assert';
import {
PolicyEngine,
isModeAllowed,
checkProfile,
detectEnvironment,
} from './policy-engine.js';
import type { Config } from './config-loader.js';
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function createConfig(overrides: Partial<Config> = {}): Config {
return {
mode: 'verify',
profiles: {},
presets: {},
environments: {},
...overrides,
};
}
// ---------------------------------------------------------------------------
// PolicyEngine.check
// ---------------------------------------------------------------------------
test('verify allowed in local', () => {
const engine = new PolicyEngine({
config: createConfig(),
env: 'local',
mode: 'verify',
});
const result = engine.check();
assert.strictEqual(result.allowed, true);
assert.strictEqual(result.errors.length, 0);
});
test('qualify blocked in production', () => {
const engine = new PolicyEngine({
config: createConfig(),
env: 'production',
mode: 'qualify',
});
const result = engine.check();
assert.strictEqual(result.allowed, false);
assert.ok(result.errors.some((e) => e.includes('Qualify mode is blocked')));
});
test('observe allowed in production with warning', () => {
const engine = new PolicyEngine({
config: createConfig(),
env: 'production',
mode: 'observe',
});
const result = engine.check();
assert.strictEqual(result.allowed, true);
assert.ok(result.warnings.some((w) => w.includes('production')));
});
test('qualify allowed in local', () => {
const engine = new PolicyEngine({
config: createConfig(),
env: 'local',
mode: 'qualify',
});
const result = engine.check();
assert.strictEqual(result.allowed, true);
});
test('qualify allowed in staging', () => {
const engine = new PolicyEngine({
config: createConfig(),
env: 'staging',
mode: 'qualify',
});
const result = engine.check();
assert.strictEqual(result.allowed, true);
});
// ---------------------------------------------------------------------------
// Profile feature checks
// ---------------------------------------------------------------------------
test('profile with chaos blocked in production', () => {
const config = createConfig({
profiles: {
chaos: {
features: ['chaos'],
},
},
});
const engine = new PolicyEngine({
config,
env: 'production',
mode: 'qualify',
profileName: 'chaos',
});
const result = engine.check();
assert.strictEqual(result.allowed, false);
assert.ok(result.errors.some((e) => e.includes('Chaos on protected routes')));
});
test('profile with chaos allowed in local', () => {
const config = createConfig({
profiles: {
chaos: {
features: ['chaos'],
},
},
});
const engine = new PolicyEngine({
config,
env: 'local',
mode: 'qualify',
profileName: 'chaos',
});
const result = engine.check();
assert.strictEqual(result.allowed, true);
});
test('qualify-only feature in verify mode is blocked', () => {
const config = createConfig({
profiles: {
bad: {
features: ['stateful'],
},
},
});
const engine = new PolicyEngine({
config,
env: 'local',
mode: 'verify',
profileName: 'bad',
});
const result = engine.check();
assert.strictEqual(result.allowed, false);
assert.ok(result.errors.some((e) => e.includes('qualify-only')));
});
// ---------------------------------------------------------------------------
// Preset/profile combination
// ---------------------------------------------------------------------------
test('unknown preset referenced by profile is blocked', () => {
const config = createConfig({
profiles: {
quick: {
preset: 'missing',
},
},
});
const engine = new PolicyEngine({
config,
env: 'local',
mode: 'verify',
profileName: 'quick',
presetName: 'missing',
});
const result = engine.check();
assert.strictEqual(result.allowed, false);
assert.ok(result.errors.some((e) => e.includes('Unknown preset')));
});
test('preset mode mismatch produces warning', () => {
const config = createConfig({
presets: {
safe: { mode: 'observe' },
},
profiles: {
quick: {
preset: 'safe',
},
},
});
const engine = new PolicyEngine({
config,
env: 'local',
mode: 'verify',
profileName: 'quick',
presetName: 'safe',
});
const result = engine.check();
assert.strictEqual(result.allowed, true);
assert.ok(result.warnings.some((w) => w.includes('mode')));
});
// ---------------------------------------------------------------------------
// isModeAllowed
// ---------------------------------------------------------------------------
test('isModeAllowed for verify in local', () => {
assert.strictEqual(isModeAllowed('verify', 'local'), true);
});
test('isModeAllowed for qualify in production', () => {
assert.strictEqual(isModeAllowed('qualify', 'production'), false);
});
test('isModeAllowed for observe in production', () => {
assert.strictEqual(isModeAllowed('observe', 'production'), true);
});
// ---------------------------------------------------------------------------
// checkProfile
// ---------------------------------------------------------------------------
test('checkProfile with chaos in production', () => {
const config = createConfig({
profiles: {
nightly: {
features: ['chaos', 'scenario'],
},
},
});
const result = checkProfile('nightly', config, 'production', 'qualify');
assert.strictEqual(result.allowed, false);
assert.ok(result.errors.length > 0);
});
// ---------------------------------------------------------------------------
// detectEnvironment
// ---------------------------------------------------------------------------
test('detectEnvironment reads NODE_ENV', () => {
const original = process.env.NODE_ENV;
process.env.NODE_ENV = 'test';
assert.strictEqual(detectEnvironment(), 'test');
process.env.NODE_ENV = original;
});
test('detectEnvironment defaults to local', () => {
const original = process.env.NODE_ENV;
delete process.env.NODE_ENV;
assert.strictEqual(detectEnvironment(), 'local');
process.env.NODE_ENV = original;
});
test('detectEnvironment maps production', () => {
const original = process.env.NODE_ENV;
process.env.NODE_ENV = 'production';
assert.strictEqual(detectEnvironment(), 'production');
process.env.NODE_ENV = original;
});
test('detectEnvironment maps prod alias', () => {
const original = process.env.NODE_ENV;
process.env.NODE_ENV = 'prod';
assert.strictEqual(detectEnvironment(), 'production');
process.env.NODE_ENV = original;
});
// ---------------------------------------------------------------------------
// Custom environment policies
// ---------------------------------------------------------------------------
test('custom environment policy overrides default', () => {
const config = createConfig({
environments: {
production: {
allowedModes: ['verify', 'observe', 'qualify'],
blockQualify: false,
},
},
});
const engine = new PolicyEngine({
config,
env: 'production',
mode: 'qualify',
});
const result = engine.check();
assert.strictEqual(result.allowed, true);
});
test('custom environment can block verify', () => {
const config = createConfig({
environments: {
readonly: {
allowedModes: ['observe'],
},
},
});
const engine = new PolicyEngine({
config,
env: 'readonly',
mode: 'verify',
});
const result = engine.check();
assert.strictEqual(result.allowed, false);
assert.ok(result.errors.some((e) => e.includes('not allowed')));
});
+446
View File
@@ -0,0 +1,446 @@
/**
* Policy engine for APOPHIS CLI.
*
* Responsibilities:
* - Environment safety matrix enforcement
* - Mode gating (verify/observe/qualify per environment)
* - Profile feature validation against environment
* - Preset/profile combination validation
* - Clear error messages on policy violations
*/
import type { Config, EnvironmentPolicy, ProfileDefinition, PresetDefinition } from './config-loader.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface PolicyCheckResult {
allowed: boolean;
errors: string[];
warnings: string[];
}
export interface PolicyEngineOptions {
config: Config;
env: string;
mode: 'verify' | 'observe' | 'qualify';
profileName?: string;
presetName?: string;
}
// ---------------------------------------------------------------------------
// Default environment policies
// ---------------------------------------------------------------------------
/**
* Default safety matrix for environments.
*
* verify: allowed in local, test, CI, optional in staging/prod
* observe: allowed everywhere, blocking in prod requires explicit policy
* qualify: blocked in prod by default, allowed in local/test/staging with restrictions
*/
const DEFAULT_ENV_POLICIES: Record<string, EnvironmentPolicy> = {
local: {
allowedModes: ['verify', 'observe', 'qualify'],
blockQualify: false,
allowChaosOnProtected: true,
},
test: {
allowedModes: ['verify', 'observe', 'qualify'],
blockQualify: false,
allowChaosOnProtected: true,
},
ci: {
allowedModes: ['verify', 'observe', 'qualify'],
blockQualify: false,
allowChaosOnProtected: false,
},
staging: {
allowedModes: ['verify', 'observe', 'qualify'],
blockQualify: false,
allowChaosOnProtected: false,
},
production: {
allowedModes: ['verify', 'observe'],
blockQualify: true,
allowChaosOnProtected: false,
},
prod: {
allowedModes: ['verify', 'observe'],
blockQualify: true,
allowChaosOnProtected: false,
},
};
// ---------------------------------------------------------------------------
// Feature sets per mode
// ---------------------------------------------------------------------------
/**
* Features that are only available in qualify mode.
*/
const QUALIFY_ONLY_FEATURES = new Set<string>([
'chaos',
'stateful',
'scenario',
'outbound-mocks',
'protocol-flow',
]);
/**
* Features that require explicit allowlist in production.
*/
const PROD_RESTRICTED_FEATURES = new Set<string>([
'chaos',
'outbound-mocks',
'protocol-flow',
]);
// ---------------------------------------------------------------------------
// Policy engine
// ---------------------------------------------------------------------------
export class PolicyEngine {
private config: Config;
private env: string;
private mode: 'verify' | 'observe' | 'qualify';
private profileName?: string;
private presetName?: string;
constructor(options: PolicyEngineOptions) {
this.config = options.config;
this.env = options.env;
this.mode = options.mode;
this.profileName = options.profileName;
this.presetName = options.presetName;
}
/**
* Run all policy checks.
* Returns result with errors and warnings.
*/
check(): PolicyCheckResult {
const errors: string[] = [];
const warnings: string[] = [];
// 1. Check if mode is allowed in current environment
const modeCheck = this.checkModeAllowed();
if (!modeCheck.allowed) {
errors.push(...modeCheck.errors);
}
warnings.push(...modeCheck.warnings);
// 2. Check if profile references features not allowed in current env
const profileCheck = this.checkProfileFeatures();
if (!profileCheck.allowed) {
errors.push(...profileCheck.errors);
}
warnings.push(...profileCheck.warnings);
// 3. Check preset/profile combination validity
const comboCheck = this.checkPresetProfileCombination();
if (!comboCheck.allowed) {
errors.push(...comboCheck.errors);
}
warnings.push(...comboCheck.warnings);
// 4. Check observe-specific safety
if (this.mode === 'observe') {
const observeCheck = this.checkObserveSafety();
if (!observeCheck.allowed) {
errors.push(...observeCheck.errors);
}
warnings.push(...observeCheck.warnings);
}
// 5. Check qualify-specific safety
if (this.mode === 'qualify') {
const qualifyCheck = this.checkQualifySafety();
if (!qualifyCheck.allowed) {
errors.push(...qualifyCheck.errors);
}
warnings.push(...qualifyCheck.warnings);
}
return {
allowed: errors.length === 0,
errors,
warnings,
};
}
// -------------------------------------------------------------------------
// Individual checks
// -------------------------------------------------------------------------
/**
* Check if current mode is allowed in current environment.
*/
private checkModeAllowed(): PolicyCheckResult {
const errors: string[] = [];
const warnings: string[] = [];
const envPolicy = this.getEnvironmentPolicy();
const allowedModes = envPolicy.allowedModes ?? [];
if (!allowedModes.includes(this.mode)) {
errors.push(
`Mode "${this.mode}" is not allowed in environment "${this.env}". ` +
`Allowed modes: ${allowedModes.join(', ') || 'none'}.`,
);
}
// Warn about observe in prod
if (this.mode === 'observe' && (this.env === 'production' || this.env === 'prod')) {
warnings.push(
`Observe mode in production requires explicit policy configuration. ` +
`Ensure blocking behavior is disabled and sampling rate is configured.`,
);
}
return { allowed: errors.length === 0, errors, warnings };
}
/**
* Check if profile references features not allowed in current environment.
*/
private checkProfileFeatures(): PolicyCheckResult {
const errors: string[] = [];
const warnings: string[] = [];
if (!this.profileName || !this.config.profiles) {
return { allowed: true, errors, warnings };
}
const profile = this.config.profiles[this.profileName];
if (!profile) {
// This should be caught by config loader, but be defensive
return { allowed: true, errors, warnings };
}
// Resolve preset features if profile references a preset
let features = profile.features ?? [];
if (profile.preset && this.config.presets) {
const preset = this.config.presets[profile.preset];
if (preset && preset.features) {
// Merge preset features with profile features (profile takes precedence)
const presetFeatures = preset.features.filter(f => !features.includes(f));
features = [...presetFeatures, ...features];
}
}
const envPolicy = this.getEnvironmentPolicy();
for (const feature of features) {
// Check qualify-only features in non-qualify mode
if (QUALIFY_ONLY_FEATURES.has(feature) && this.mode !== 'qualify') {
errors.push(
`Profile "${this.profileName}" references qualify-only feature "${feature}" ` +
`but current mode is "${this.mode}".`,
);
}
// Check prod-restricted features
if (PROD_RESTRICTED_FEATURES.has(feature) && (this.env === 'production' || this.env === 'prod')) {
if (feature === 'chaos' && !envPolicy.allowChaosOnProtected) {
errors.push(
`Feature "${feature}" from profile "${this.profileName}" is blocked in production. ` +
`Chaos on protected routes requires explicit allowlist configuration.`,
);
} else if (feature !== 'chaos') {
errors.push(
`Feature "${feature}" from profile "${this.profileName}" is restricted in production. ` +
`Requires explicit break-glass policy.`,
);
}
}
}
return { allowed: errors.length === 0, errors, warnings };
}
/**
* Check preset/profile combination validity.
*/
private checkPresetProfileCombination(): PolicyCheckResult {
const errors: string[] = [];
const warnings: string[] = [];
if (!this.presetName || !this.profileName) {
return { allowed: true, errors, warnings };
}
if (!this.config.presets) {
errors.push(`Preset "${this.presetName}" referenced but no presets defined in config.`);
return { allowed: false, errors, warnings };
}
const preset = this.config.presets[this.presetName];
if (!preset) {
errors.push(`Unknown preset "${this.presetName}".`);
return { allowed: false, errors, warnings };
}
// Check mode compatibility between preset and current mode
if (preset.mode && preset.mode !== this.mode) {
warnings.push(
`Preset "${this.presetName}" is configured for mode "${preset.mode}" ` +
`but current mode is "${this.mode}".`,
);
}
// Check profile features against preset features
const profile = this.config.profiles?.[this.profileName];
if (profile && preset.features && profile.features) {
const presetFeatures = new Set(preset.features);
const profileFeatures = new Set(profile.features);
for (const feature of profileFeatures) {
if (!presetFeatures.has(feature)) {
warnings.push(
`Profile "${this.profileName}" includes feature "${feature}" ` +
`not present in preset "${this.presetName}".`,
);
}
}
}
return { allowed: errors.length === 0, errors, warnings };
}
/**
* Check observe-specific safety constraints.
*/
private checkObserveSafety(): PolicyCheckResult {
const errors: string[] = [];
const warnings: string[] = [];
const envPolicy = this.getEnvironmentPolicy();
// In prod, observe must be non-blocking
if ((this.env === 'production' || this.env === 'prod') && envPolicy.blockQualify) {
// blockQualify being true in prod is expected, but we should ensure
// observe doesn't have blocking behavior
warnings.push(
`Observe mode in production: ensure non-blocking semantics and proper sampling rate.`,
);
}
return { allowed: errors.length === 0, errors, warnings };
}
/**
* Check qualify-specific safety constraints.
*/
private checkQualifySafety(): PolicyCheckResult {
const errors: string[] = [];
const warnings: string[] = [];
const envPolicy = this.getEnvironmentPolicy();
// Check if qualify is blocked in this environment
if (envPolicy.blockQualify) {
errors.push(
`Qualify mode is blocked in environment "${this.env}". ` +
`This environment does not support scenario, stateful, or chaos execution.`,
);
}
// Check for chaos on protected routes
const profile = this.profileName ? this.config.profiles?.[this.profileName] : undefined;
if (profile?.features?.includes('chaos') && !envPolicy.allowChaosOnProtected) {
errors.push(
`Chaos on protected routes is not allowed in environment "${this.env}". ` +
`Add routes to allowlist or use a different environment.`,
);
}
return { allowed: errors.length === 0, errors, warnings };
}
// -------------------------------------------------------------------------
// Helpers
// -------------------------------------------------------------------------
/**
* Get environment policy, falling back to defaults.
*/
private getEnvironmentPolicy(): EnvironmentPolicy {
const userPolicy = this.config.environments?.[this.env];
const defaultPolicy = DEFAULT_ENV_POLICIES[this.env];
return {
...defaultPolicy,
...userPolicy,
};
}
}
// ---------------------------------------------------------------------------
// Convenience functions
// ---------------------------------------------------------------------------
/**
* Check if a mode is allowed in an environment.
* Standalone function for simple checks.
*/
export function isModeAllowed(
mode: 'verify' | 'observe' | 'qualify',
env: string,
config?: Config,
): boolean {
const engine = new PolicyEngine({
config: config ?? {},
env,
mode,
});
const result = engine.check();
return result.allowed;
}
/**
* Check a profile against environment policy.
*/
export function checkProfile(
profileName: string,
config: Config,
env: string,
mode: 'verify' | 'observe' | 'qualify',
): PolicyCheckResult {
const engine = new PolicyEngine({
config,
env,
mode,
profileName,
});
return engine.check();
}
/**
* Get the default environment name from NODE_ENV.
*/
export function detectEnvironment(): string {
const nodeEnv = process.env.NODE_ENV ?? 'local';
switch (nodeEnv) {
case 'test':
return 'test';
case 'ci':
case 'CI':
return 'ci';
case 'staging':
return 'staging';
case 'production':
case 'prod':
return 'production';
default:
return 'local';
}
}
// ---------------------------------------------------------------------------
// Re-export types
// ---------------------------------------------------------------------------
export type { Config, EnvironmentPolicy, ProfileDefinition, PresetDefinition };
+403
View File
@@ -0,0 +1,403 @@
/**
* S0: Spec Authority - Core types for APOPHIS CLI
* Frozen contract. All implementation streams code against these types.
*/
// ─────────────────────────────────────────────────────────────────────────────
// CLI Context (injected, never optional imports)
// ─────────────────────────────────────────────────────────────────────────────
export interface CliContext {
/** Absolute path to current working directory */
cwd: string;
/** Normalized environment detection */
env: {
nodeEnv: string | undefined;
apophisEnv: string | undefined;
};
/** Is stdout a TTY? */
isTTY: boolean;
/** Is running in CI? (CI=true, GITHUB_ACTIONS, etc.) */
isCI: boolean;
/** Node.js version string */
nodeVersion?: string;
/** Package manager detected (npm, yarn, pnpm, bun) */
packageManager: "npm" | "yarn" | "pnpm" | "bun" | "unknown";
/** Absolute path to the CLI binary (for self-reference) */
selfPath?: string;
/** Parsed global CLI options */
options: {
config: string | undefined;
profile: string | undefined;
generationProfile?: string;
format: OutputFormat;
color: ColorMode;
quiet: boolean;
verbose: boolean;
artifactDir: string | undefined;
};
}
// ─────────────────────────────────────────────────────────────────────────────
// Exit Codes
// ─────────────────────────────────────────────────────────────────────────────
export const ExitCodes = {
SUCCESS: 0,
BEHAVIORAL_FAILURE: 1,
USAGE_ERROR: 2,
INTERNAL_ERROR: 3,
INTERRUPTED: 130,
} as const;
export type ExitCode = (typeof ExitCodes)[keyof typeof ExitCodes];
// ─────────────────────────────────────────────────────────────────────────────
// Config Schema (TypeBox-style: plain TS interfaces with JSON Schema metadata)
// ─────────────────────────────────────────────────────────────────────────────
/**
* @jsonSchema { type: "string", enum: ["verify", "observe", "qualify"] }
*/
export type ApophisMode = "verify" | "observe" | "qualify";
/**
* @jsonSchema { type: "string", enum: ["human", "json", "ndjson", "json-summary", "ndjson-summary"] }
*/
export type OutputFormat = "human" | "json" | "ndjson" | "json-summary" | "ndjson-summary";
/**
* @jsonSchema { type: "string", enum: ["auto", "always", "never"] }
*/
export type ColorMode = "auto" | "always" | "never";
/**
* Environment policy: safety gates for running commands in specific environments.
* @jsonSchema {
* type: "object",
* required: ["name"],
* properties: {
* name: { type: "string" },
* allowVerify: { type: "boolean", default: true },
* allowObserve: { type: "boolean", default: true },
* allowQualify: { type: "boolean", default: false },
* allowChaos: { type: "boolean", default: false },
* allowBlocking: { type: "boolean", default: false },
* requireSink: { type: "boolean", default: false }
* },
* additionalProperties: false
* }
*/
export interface EnvironmentPolicy {
name: string;
allowVerify?: boolean;
allowObserve?: boolean;
allowQualify?: boolean;
allowChaos?: boolean;
allowBlocking?: boolean;
requireSink?: boolean;
}
/**
* Profile: a named configuration for a specific run mode.
* @jsonSchema {
* type: "object",
* required: ["name"],
* properties: {
* name: { type: "string" },
* mode: { type: "string", enum: ["verify", "observe", "qualify"] },
* preset: { type: "string" },
* routes: { type: "array", items: { type: "string" } },
* seed: { type: "number" },
* artifactDir: { type: "string" },
* environment: { type: "string" }
* },
* additionalProperties: false
* }
*/
export interface ProfileDefinition {
name: string;
mode?: ApophisMode;
preset?: string;
routes?: string[];
seed?: number;
artifactDir?: string;
environment?: string;
}
/**
* Preset: a reusable base configuration that profiles can extend.
* @jsonSchema {
* type: "object",
* required: ["name"],
* properties: {
* name: { type: "string" },
* depth: { type: "string", enum: ["quick", "standard", "deep"] },
* timeout: { type: "number" },
* parallel: { type: "boolean" },
* chaos: { type: "boolean" },
* observe: { type: "boolean" }
* },
* additionalProperties: false
* }
*/
export interface PresetDefinition {
name: string;
depth?: "quick" | "standard" | "deep";
timeout?: number;
parallel?: boolean;
chaos?: boolean;
observe?: boolean;
}
/**
* Root configuration object for apophis.config.js|ts|json
* @jsonSchema {
* type: "object",
* required: [],
* properties: {
* mode: { type: "string", enum: ["verify", "observe", "qualify"] },
* profile: { type: "string" },
* preset: { type: "string" },
* routes: { type: "array", items: { type: "string" } },
* seed: { type: "number" },
* artifactDir: { type: "string" },
* environments: { type: "object", additionalProperties: { $ref: "#/definitions/EnvironmentPolicy" } },
* profiles: { type: "object", additionalProperties: { $ref: "#/definitions/ProfileDefinition" } },
* presets: { type: "object", additionalProperties: { $ref: "#/definitions/PresetDefinition" } }
* },
* additionalProperties: false
* }
*/
export interface ApophisConfig {
mode?: ApophisMode;
profile?: string;
preset?: string;
routes?: string[];
seed?: number;
artifactDir?: string;
environments?: Record<string, EnvironmentPolicy>;
profiles?: Record<string, ProfileDefinition>;
presets?: Record<string, PresetDefinition>;
}
// ─────────────────────────────────────────────────────────────────────────────
// Artifact Schema
// ─────────────────────────────────────────────────────────────────────────────
/**
* Single contract failure record.
*/
export interface FailureRecord {
route: string;
contract: string;
expected: string;
observed: string;
seed: number;
replayCommand: string;
category?: string;
diff?: string;
actual?: string;
}
/**
* Artifact document emitted by verify, observe, and qualify runs.
* @jsonSchema {
* type: "object",
* required: ["version", "command", "cwd", "startedAt", "durationMs", "summary"],
* properties: {
* version: { type: "string", const: "apophis-artifact/1" },
* command: { type: "string" },
* mode: { type: "string" },
* cwd: { type: "string" },
* configPath: { type: "string" },
* profile: { type: "string" },
* preset: { type: "string" },
* env: { type: "string" },
* seed: { type: "number" },
* startedAt: { type: "string", format: "date-time" },
* durationMs: { type: "number" },
* summary: {
* type: "object",
* properties: {
* total: { type: "number" },
* passed: { type: "number" },
* failed: { type: "number" }
* }
* },
* failures: { type: "array", items: { type: "object" } },
* artifacts: { type: "array", items: { type: "string" } },
* warnings: { type: "array", items: { type: "string" } },
* exitReason: { type: "string" }
* }
* }
*/
export interface StepTrace {
step: number;
name: string;
route: string;
durationMs: number;
status: "passed" | "failed" | "skipped";
error?: string;
}
export interface CleanupOutcome {
resource: string;
cleaned: boolean;
error?: string;
}
export interface ExecutionSummary {
totalPlanned: number;
totalExecuted: number;
totalPassed: number;
totalFailed: number;
scenariosRun: number;
statefulTestsRun: number;
chaosRunsRun: number;
totalSteps: number;
}
export interface RouteExecutionInfo {
route: string;
executed: boolean;
reason?: string;
}
export interface ProfileGates {
scenario: boolean;
stateful: boolean;
chaos: boolean;
}
export interface WorkspaceRun {
package: string;
cwd: string;
artifact: Artifact;
}
export interface WorkspaceResult {
exitCode: ExitCode;
runs: WorkspaceRun[];
message?: string;
warnings?: string[];
}
export interface Artifact {
version: "apophis-artifact/1";
cliVersion?: string;
command: string;
mode?: string;
cwd: string;
configPath?: string;
profile?: string;
preset?: string;
env?: string;
seed?: number;
startedAt: string;
durationMs: number;
summary: {
total: number;
passed: number;
failed: number;
};
executionSummary?: ExecutionSummary;
executedRoutes?: string[];
skippedRoutes?: RouteExecutionInfo[];
stepTraces?: StepTrace[];
cleanupOutcomes?: CleanupOutcome[];
profileGates?: ProfileGates;
deterministicParams?: Record<string, unknown>;
failures: FailureRecord[];
artifacts: string[];
warnings: string[];
exitReason: string;
package?: string;
}
// ─────────────────────────────────────────────────────────────────────────────
// Command Result
// ─────────────────────────────────────────────────────────────────────────────
/**
* Result returned by every command handler.
* Renderers consume this to produce human/json/ndjson output.
*/
export interface CommandResult {
exitCode: ExitCode;
artifact?: Artifact;
message?: string;
warnings?: string[];
}
// ─────────────────────────────────────────────────────────────────────────────
// Shared types for commands
// ─────────────────────────────────────────────────────────────────────────────
/**
* Parsed CLI arguments (from cac or similar).
*/
export interface ParsedArgs {
command: string;
args: string[];
flags: Record<string, unknown>;
}
/**
* Global flags every command must accept.
*/
export interface GlobalFlags {
config?: string;
profile?: string;
cwd?: string;
format?: OutputFormat;
color?: ColorMode;
quiet?: boolean;
verbose?: boolean;
artifactDir?: string;
}
/**
* Route descriptor used for filtering and discovery.
*/
export interface RouteDescriptor {
method: string;
path: string;
schema?: unknown;
contracts?: string[];
}
/**
* Contract evaluation result for a single route.
*/
export interface RouteResult {
route: string;
passed: boolean;
durationMs: number;
failures?: FailureRecord[];
warnings?: string[];
}
/**
* NDJSON event types for streaming output.
*/
export type NdjsonEvent =
| { type: "run.started"; command: string; seed?: number; timestamp: string }
| { type: "route.started"; route: string; timestamp: string }
| { type: "route.passed"; route: string; durationMs: number; timestamp: string }
| { type: "route.failed"; route: string; failure: FailureRecord; timestamp: string }
| { type: "run.completed"; summary: Artifact["summary"]; timestamp: string };
/**
* Human output section for canonical failure rendering.
*/
export interface HumanFailureSection {
route: string;
profile?: string;
seed: number;
expected: string;
observed: string;
whyItMatters: string;
replayCommand: string;
nextSteps: string;
}
+201
View File
@@ -0,0 +1,201 @@
/**
* Workspace runner for APOPHIS CLI commands.
*
* Responsibilities:
* - Fan out a command across all workspace packages
* - Collect per-package artifacts with package attribution
* - Aggregate results into a single workspace result
* - Support json, ndjson, and human output formats
* - Preserve exit codes: fail if any package fails
*
* Architecture:
* - Dependency injection: all dependencies passed explicitly
* - No optional imports everything is required or injected
* - Inline comments for documentation
*/
import type { CliContext } from './context.js';
import { findWorkspacePackages } from './config-loader.js';
import type { Artifact, WorkspaceRun, WorkspaceResult, ExitCode } from './types.js';
import { SUCCESS } from './exit-codes.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export type RunCommandFn = (ctx: CliContext) => Promise<{ exitCode: number; artifact?: Artifact; warnings?: string[] }>;
export interface WorkspaceRunnerDeps {
runCommand: RunCommandFn;
findPackages?: (cwd: string) => string[];
}
// ---------------------------------------------------------------------------
// Workspace package discovery
// ---------------------------------------------------------------------------
/**
* Discover workspace packages using config-loader.
* Falls back to empty array if no workspaces found.
*/
function discoverPackages(cwd: string, findPackages?: (cwd: string) => string[]): string[] {
if (findPackages) {
return findPackages(cwd);
}
return findWorkspacePackages(cwd);
}
// ---------------------------------------------------------------------------
// Package name extraction
// ---------------------------------------------------------------------------
/**
* Extract package name from absolute path.
* Uses basename of the directory.
*/
function getPackageName(pkgPath: string): string {
const parts = pkgPath.split('/');
return parts[parts.length - 1] || 'unknown';
}
// ---------------------------------------------------------------------------
// Workspace execution
// ---------------------------------------------------------------------------
/**
* Run a command across all workspace packages.
*
* Flow:
* 1. Discover workspace packages
* 2. Run command for each package with package-attributed context
* 3. Collect artifacts and warnings
* 4. Determine overall exit code (fail if any package fails)
* 5. Return workspace result with all runs
*/
export async function runWorkspace(
deps: WorkspaceRunnerDeps,
ctx: CliContext,
): Promise<WorkspaceResult> {
const packages = discoverPackages(ctx.cwd, deps.findPackages);
if (packages.length === 0) {
return {
exitCode: SUCCESS as ExitCode,
runs: [],
message: 'No workspace packages found.',
};
}
const runs: WorkspaceRun[] = [];
let overallExitCode = SUCCESS;
const allWarnings: string[] = [];
for (const pkgPath of packages) {
const pkgName = getPackageName(pkgPath);
// Create a context scoped to this package's directory
const pkgCtx: CliContext = {
...ctx,
cwd: pkgPath,
};
const pkgResult = await deps.runCommand(pkgCtx);
if (pkgResult.artifact) {
// Attach package name to artifact for attribution
const attributedArtifact: Artifact = {
...pkgResult.artifact,
package: pkgName,
};
runs.push({
package: pkgName,
cwd: pkgPath,
artifact: attributedArtifact,
});
}
if (pkgResult.exitCode !== SUCCESS) {
overallExitCode = pkgResult.exitCode as ExitCode;
}
if (pkgResult.warnings) {
allWarnings.push(...pkgResult.warnings.map(w => `[${pkgName}] ${w}`));
}
}
return {
exitCode: overallExitCode as ExitCode,
runs,
warnings: allWarnings.length > 0 ? allWarnings : undefined,
};
}
// ---------------------------------------------------------------------------
// Output formatting
// ---------------------------------------------------------------------------
/**
* Format workspace results for human-readable output.
* Shows per-package summary with pass/fail status.
*/
export function formatWorkspaceHuman(result: WorkspaceResult): string {
const lines: string[] = [];
lines.push('Workspace results');
lines.push('');
for (const run of result.runs) {
const a = run.artifact;
const status = a.exitReason === 'success' ? '✓' : '✗';
lines.push(` ${status} ${run.package}: ${a.summary.passed}/${a.summary.total} passed`);
if (a.summary.failed > 0) {
lines.push(` ${a.summary.failed} failed`);
}
}
lines.push('');
lines.push(`Overall: ${result.exitCode === SUCCESS ? 'passed' : 'failed'}`);
return lines.join('\n');
}
/**
* Format workspace results as JSON.
* Includes all runs with full artifacts.
*/
export function formatWorkspaceJson(result: WorkspaceResult): string {
return JSON.stringify({
exitCode: result.exitCode,
runs: result.runs.map(r => ({
package: r.package,
cwd: r.cwd,
artifact: r.artifact,
})),
warnings: result.warnings,
}, null, 2);
}
/**
* Format workspace results as NDJSON.
* Emits one event per package plus a completion event.
*/
export function formatWorkspaceNdjson(result: WorkspaceResult): string {
const lines: string[] = [];
for (const run of result.runs) {
lines.push(JSON.stringify({
type: 'workspace.run.completed',
package: run.package,
cwd: run.cwd,
summary: run.artifact.summary,
exitReason: run.artifact.exitReason,
}));
}
lines.push(JSON.stringify({
type: 'workspace.completed',
exitCode: result.exitCode,
packages: result.runs.length,
}));
return lines.join('\n');
}
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env node
// src/cli/index.ts — canonical CLI entrypoint
// Imports main from core and executes it when run directly.
import { main } from './core/index.js';
main().then(code => {
process.exit(code);
}).catch(error => {
console.error(error);
process.exit(3);
});
+466
View File
@@ -0,0 +1,466 @@
/**
* S10: Renderers thread - Human renderer
*
* Responsibilities:
* - Render canonical failure output matching golden snapshot exactly
* - Render progress/summary for verify/observe/qualify
* - Render doctor check results
* - Render migrate rewrite reports
* - Handle large payload truncation
* - Use picocolors for styling
* - No spinners in CI
* - Color respects --color flag
*/
import pc from 'picocolors';
import type { Artifact, FailureRecord, HumanFailureSection } from '../core/types.js';
import type { OutputContext } from './shared.js';
import { shouldUseColor, getColors, truncate, indent, formatDuration } from './shared.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface HumanRendererOptions {
ctx: OutputContext;
profile?: string;
seed?: number;
}
// ---------------------------------------------------------------------------
// Color setup
// ---------------------------------------------------------------------------
/**
* Get the colors instance for this render context.
*/
function getColorizer(ctx: OutputContext) {
const enabled = shouldUseColor(ctx);
return getColors(enabled);
}
// ---------------------------------------------------------------------------
// Canonical failure output
// ---------------------------------------------------------------------------
/**
* Render canonical failure output matching golden snapshot exactly.
*
* Golden snapshot format:
* Contract violation
* POST /users
* Profile: quick
* Seed: 42
*
* Expected
* response_code(GET /users/{response_body(this).id}) == 200
*
* Observed
* GET /users/usr-123 returned 404
*
* Why this matters
* The resource created by POST /users is not retrievable.
*
* Replay
* apophis replay --artifact reports/apophis/failure-2026-04-28T12-30-22Z.json
*
* Next
* Check the create/read consistency for POST /users and GET /users/{id}.
*/
export function renderCanonicalFailure(
failure: FailureRecord,
options: HumanRendererOptions,
): string {
const c = getColorizer(options.ctx);
const lines: string[] = [];
// Title
lines.push(c.red('Contract violation'));
// Route
lines.push(c.bold(failure.route));
// Profile and Seed
lines.push(`Profile: ${options.profile || 'default'}`);
lines.push(`Seed: ${failure.seed}`);
lines.push('');
// Expected
lines.push('Expected');
lines.push(indent(failure.contract, 2));
lines.push('');
// Observed
lines.push('Observed');
// Truncate observed if very long
const observed = failure.observed.length > 500
? truncate(failure.observed, { maxLength: 500 })
: failure.observed;
lines.push(indent(observed, 2));
lines.push('');
// Diff (if available)
if (failure.diff) {
lines.push('Diff');
for (const line of failure.diff.split('\n')) {
lines.push(indent(line, 2));
}
lines.push('');
}
// Actual value (if available, different from observed)
if (failure.actual && failure.actual !== failure.observed) {
lines.push('Actual value');
const actual = failure.actual.length > 500
? truncate(failure.actual, { maxLength: 500 })
: failure.actual;
lines.push(indent(actual, 2));
lines.push('');
}
// Why this matters
lines.push('Why this matters');
lines.push(indent(generateWhyItMatters(failure), 2));
lines.push('');
// Replay
lines.push('Replay');
lines.push(indent(failure.replayCommand, 2));
lines.push('');
// Next
lines.push('Next');
lines.push(indent(generateNextSteps(failure), 2));
return lines.join('\n');
}
/**
* Generate "Why this matters" text from failure context.
*/
function generateWhyItMatters(failure: FailureRecord): string {
const route = failure.route;
const method = route.split(' ')[0];
const path = route.split(' ')[1] || route;
// For POST /users with GET follow-up contract
if (method === 'POST' && failure.contract.includes('GET')) {
return `The resource created by ${route} is not retrievable.`;
}
// For GET requests
if (method === 'GET') {
return `The resource at ${path} does not exist or is inaccessible.`;
}
// Generic fallback
return `The contract for ${route} was violated.`;
}
/**
* Generate "Next" steps text from failure context.
*/
function generateNextSteps(failure: FailureRecord): string {
const route = failure.route;
const method = route.split(' ')[0];
const path = route.split(' ')[1] || route;
// For POST /users with GET follow-up
if (method === 'POST' && failure.contract.includes('GET')) {
const getPath = failure.contract.match(/GET\s+([^\s{]+)/)?.[1] || path;
// Ensure the path ends with /{id} for the canonical format
// Remove trailing slash before adding /{id} to avoid double slashes
const basePath = getPath.endsWith('/') ? getPath.slice(0, -1) : getPath;
const normalizedPath = basePath.endsWith('/{id}') ? basePath : `${basePath}/{id}`;
return `Check the create/read consistency for ${route} and GET ${normalizedPath}.`;
}
// Generic fallback
return `Review the contract and implementation for ${route}.`;
}
// ---------------------------------------------------------------------------
// Progress and summary rendering
// ---------------------------------------------------------------------------
/**
* Render progress for a running command.
* Safe for CI (no spinners, just text updates).
*/
export function renderProgress(
current: number,
total: number,
label: string,
ctx: OutputContext,
): string {
const c = getColorizer(ctx);
const pct = total > 0 ? Math.round((current / total) * 100) : 0;
if (ctx.isCI || !ctx.isTTY) {
// CI mode: simple text, no spinner
return `${label} [${current}/${total}] ${pct}%`;
}
// TTY mode: with color
const bar = renderProgressBar(current, total, 20, ctx);
return `${c.dim(label)} ${bar} ${c.bold(`${pct}%`)}`;
}
/**
* Render a simple ASCII progress bar.
*/
function renderProgressBar(
current: number,
total: number,
width: number,
ctx: OutputContext,
): string {
const c = getColorizer(ctx);
if (total === 0) return c.dim('[' + ' '.repeat(width) + ']');
const filled = Math.round((current / total) * width);
const empty = width - filled;
const filledChar = '█';
const emptyChar = '░';
return '[' + c.green(filledChar.repeat(filled)) + c.dim(emptyChar.repeat(empty)) + ']';
}
/**
* Render summary for verify/observe/qualify results.
*/
export function renderSummary(
artifact: Artifact,
ctx: OutputContext,
): string {
const c = getColorizer(ctx);
const lines: string[] = [];
const { summary } = artifact;
lines.push('');
lines.push(c.bold('Summary'));
lines.push(` Total: ${summary.total}`);
lines.push(` ${c.green('Passed:')} ${summary.passed}`);
if (summary.failed > 0) {
lines.push(` ${c.red('Failed:')} ${summary.failed}`);
} else {
lines.push(` Failed: ${summary.failed}`);
}
lines.push(` Duration: ${formatDuration(artifact.durationMs)}`);
if (artifact.seed !== undefined) {
lines.push(` Seed: ${artifact.seed}`);
}
return lines.join('\n');
}
// ---------------------------------------------------------------------------
// Doctor check results rendering
// ---------------------------------------------------------------------------
/**
* Render doctor check results.
*/
export function renderDoctorChecks(
checks: Array<{ name: string; status: 'pass' | 'fail' | 'warn'; message: string; detail?: string }>,
ctx: OutputContext,
): string {
const c = getColorizer(ctx);
const lines: string[] = [];
lines.push(c.bold('Doctor Results'));
lines.push('');
for (const check of checks) {
const icon = check.status === 'pass'
? c.green('✓')
: check.status === 'warn'
? c.yellow('⚠')
: c.red('✗');
lines.push(` ${icon} ${check.name}: ${check.message}`);
if (check.detail) {
lines.push(indent(check.detail, 4));
}
}
// Overall status
const failedCount = checks.filter(c => c.status === 'fail').length;
const warnCount = checks.filter(c => c.status === 'warn').length;
lines.push('');
if (failedCount > 0) {
lines.push(c.red(`Failed: ${failedCount} check(s)`));
} else if (warnCount > 0) {
lines.push(c.yellow(`Warnings: ${warnCount} check(s)`));
} else {
lines.push(c.green('All checks passed.'));
}
return lines.join('\n');
}
// ---------------------------------------------------------------------------
// Migrate rewrite report rendering
// ---------------------------------------------------------------------------
/**
* Render migrate rewrite report.
*/
export function renderMigrateReport(
items: Array<{ type: string; file: string; line?: number; legacy: string; replacement: string; guidance?: string; ambiguous?: boolean }>,
completed: typeof items,
remaining: typeof items,
mode: 'check' | 'dry-run' | 'write',
ctx: OutputContext,
): string {
const c = getColorizer(ctx);
const lines: string[] = [];
if (mode === 'check') {
lines.push(c.bold('Legacy config patterns detected:'));
lines.push('');
for (const item of items) {
const location = item.line ? `${item.file}:${item.line}` : item.file;
lines.push(` ${c.dim(location)}`);
lines.push(` Legacy: ${c.yellow(item.legacy)}`);
lines.push(` Replace: ${c.green(item.replacement)}`);
if (item.guidance) {
lines.push(` Guidance: ${c.dim(item.guidance)}`);
}
if (item.ambiguous) {
lines.push(` ${c.yellow('⚠ Ambiguous — requires manual choice')}`);
}
lines.push('');
}
lines.push(`Found ${items.length} item(s) to migrate.`);
lines.push('');
lines.push('Run "apophis migrate --dry-run" to preview rewrites.');
lines.push('Run "apophis migrate --write" to apply rewrites.');
} else if (mode === 'dry-run') {
lines.push(c.bold('Dry run — the following rewrites would be applied:'));
lines.push('');
for (const item of items) {
const location = item.line ? `${item.file}:${item.line}` : item.file;
lines.push(` ${c.dim(location)}`);
lines.push(` ${c.red('- ' + item.legacy)}`);
lines.push(` ${c.green('+ ' + item.replacement)}`);
if (item.guidance) {
lines.push(` ${c.dim('# ' + item.guidance)}`);
}
if (item.ambiguous) {
lines.push(` ${c.yellow('⚠ Skipped (ambiguous — requires manual choice)')}`);
}
lines.push('');
}
lines.push(`Total: ${items.length} item(s) to migrate.`);
lines.push('');
lines.push('Run "apophis migrate --write" to apply these rewrites.');
} else {
// write mode
lines.push(c.bold('Migration complete:'));
lines.push('');
if (completed.length > 0) {
lines.push(` ${c.green(`Completed (${completed.length}):`)}`);
for (const item of completed) {
const location = item.line ? `${item.file}:${item.line}` : item.file;
lines.push(` ${c.green('✓')} ${location}${item.legacy}${item.replacement}`);
}
lines.push('');
}
if (remaining.length > 0) {
lines.push(` ${c.yellow(`Remaining (${remaining.length}):`)}`);
for (const item of remaining) {
const location = item.line ? `${item.file}:${item.line}` : item.file;
lines.push(` - ${location}${item.legacy}`);
if (item.ambiguous) {
lines.push(` ${c.yellow('⚠ Ambiguous — requires manual choice')}`);
} else if (item.guidance) {
lines.push(` ${c.dim('# ' + item.guidance)}`);
}
}
lines.push('');
}
if (remaining.length === 0) {
lines.push(c.green('All items migrated successfully.'));
} else {
lines.push('Run "apophis migrate --check" to review remaining items.');
}
}
return lines.join('\n');
}
// ---------------------------------------------------------------------------
// Full artifact rendering (human format)
// ---------------------------------------------------------------------------
/**
* Render a full artifact as human-readable output.
* This is the main entry point for --format human.
*/
export function renderHumanArtifact(
artifact: Artifact,
ctx: OutputContext,
): string {
const c = getColorizer(ctx);
const lines: string[] = [];
// Header
lines.push(c.bold(`apophis ${artifact.command}`));
lines.push('');
// Failures
if (artifact.failures.length > 0) {
for (const failure of artifact.failures) {
lines.push(renderCanonicalFailure(failure, {
ctx,
profile: artifact.profile,
seed: artifact.seed,
}));
lines.push('');
}
}
// Warnings
if (artifact.warnings.length > 0) {
lines.push(c.yellow('Warnings:'));
for (const warning of artifact.warnings) {
lines.push(`${warning}`);
}
lines.push('');
}
// Summary
lines.push(renderSummary(artifact, ctx));
// Expansion path guidance
lines.push('');
lines.push(c.bold('Next steps'));
if (artifact.command === 'verify') {
if (artifact.summary.failed === 0) {
lines.push(` ${c.green('✓')} All contracts passed.`);
lines.push(` ${c.dim('→ Add more behavioral contracts with')} x-ensures ${c.dim('and')} x-requires ${c.dim('to cover more routes.')}`);
lines.push(` ${c.dim('→ Run')} apophis observe ${c.dim('to enable runtime contract monitoring in production.')}`);
lines.push(` ${c.dim('→ Run')} apophis qualify --profile standard ${c.dim('for stateful/chaos testing.')}`);
} else {
lines.push(` ${c.yellow('!')} Fix failing contracts and rerun with:`);
lines.push(` ${c.dim('→')} apophis verify --seed ${artifact.seed} ${artifact.profile ? `--profile ${artifact.profile}` : ''}`);
lines.push(` ${c.dim('→ Or replay the artifact:')} apophis replay --artifact <path>`);
}
}
return lines.join('\n');
}
+210
View File
@@ -0,0 +1,210 @@
/**
* S10: Renderers thread - JSON renderer
*
* Responsibilities:
* - Render artifact schema as single JSON document
* - Include all required fields
* - Stable field ordering
* - No ANSI codes
*/
import type { Artifact, CommandResult } from '../core/types.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface JsonRendererOptions {
indent?: number;
}
// ---------------------------------------------------------------------------
// Stable field ordering
// ---------------------------------------------------------------------------
/**
* Ordered keys for the artifact schema.
* This ensures stable output regardless of object creation order.
*/
const ARTIFACT_KEY_ORDER: (keyof Artifact)[] = [
'version',
'command',
'mode',
'cwd',
'configPath',
'profile',
'preset',
'env',
'seed',
'startedAt',
'durationMs',
'summary',
'failures',
'artifacts',
'warnings',
'exitReason',
];
/**
* Ordered keys for the summary object.
*/
const SUMMARY_KEY_ORDER = ['total', 'passed', 'failed'];
/**
* Ordered keys for failure records.
*/
const FAILURE_KEY_ORDER: (keyof Artifact['failures'][number])[] = [
'route',
'contract',
'expected',
'observed',
'seed',
'replayCommand',
];
// ---------------------------------------------------------------------------
// Ordering helpers
// ---------------------------------------------------------------------------
/**
* Create an object with stable key ordering.
*/
function orderKeys<T extends Record<string, unknown>>(
obj: T,
keyOrder: string[],
): Record<string, unknown> {
const ordered: Record<string, unknown> = {};
// Add keys in specified order
for (const key of keyOrder) {
if (key in obj) {
ordered[key] = obj[key];
}
}
// Add any remaining keys not in the order list
for (const key of Object.keys(obj)) {
if (!(key in ordered)) {
ordered[key] = obj[key];
}
}
return ordered;
}
/**
* Order artifact fields for stable JSON output.
*/
function orderArtifact(artifact: Artifact): Record<string, unknown> {
const ordered = orderKeys(artifact as unknown as Record<string, unknown>, ARTIFACT_KEY_ORDER);
// Order summary fields
if (ordered.summary && typeof ordered.summary === 'object') {
ordered.summary = orderKeys(
ordered.summary as Record<string, unknown>,
SUMMARY_KEY_ORDER,
);
}
// Order failure fields
if (Array.isArray(ordered.failures)) {
ordered.failures = ordered.failures.map((failure) =>
orderKeys(failure as Record<string, unknown>, FAILURE_KEY_ORDER),
);
}
return ordered;
}
// ---------------------------------------------------------------------------
// JSON rendering
// ---------------------------------------------------------------------------
/**
* Render an artifact as a single stable JSON document.
*/
export function renderJsonArtifact(
artifact: Artifact,
options: JsonRendererOptions = {},
): string {
const { indent = 2 } = options;
// Order fields for stability
const ordered = orderArtifact(artifact);
// Serialize with stable field ordering
return JSON.stringify(ordered, null, indent);
}
/**
* Render a CommandResult as JSON.
* If an artifact is present, it is rendered.
* Otherwise, a minimal JSON with the message and exit code is returned.
*/
export function renderJsonResult(
result: CommandResult,
options: JsonRendererOptions = {},
): string {
if (result.artifact) {
return renderJsonArtifact(result.artifact, options);
}
// Minimal JSON for results without artifacts
const minimal = {
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
};
return JSON.stringify(minimal, null, options.indent ?? 2);
}
/**
* Render a concise summary artifact for CI/machine parsers.
* Omits stepTraces, cleanupOutcomes, and profileGates to reduce noise.
* Keeps summary, failures, warnings, and deterministicParams.
*/
export function renderJsonSummaryArtifact(
artifact: Artifact,
options: JsonRendererOptions = {},
): string {
const { indent = 2 } = options;
// Build a minimal artifact with only essential fields
const minimal: Record<string, unknown> = {
version: artifact.version,
command: artifact.command,
mode: artifact.mode,
cwd: artifact.cwd,
configPath: artifact.configPath,
profile: artifact.profile,
preset: artifact.preset,
env: artifact.env,
seed: artifact.seed,
startedAt: artifact.startedAt,
durationMs: artifact.durationMs,
summary: artifact.summary,
failures: artifact.failures,
artifacts: artifact.artifacts,
warnings: artifact.warnings,
exitReason: artifact.exitReason,
};
// Only include executionSummary and deterministicParams if present
if (artifact.executionSummary) {
minimal.executionSummary = artifact.executionSummary;
}
if (artifact.deterministicParams) {
minimal.deterministicParams = artifact.deterministicParams;
}
return JSON.stringify(minimal, null, indent);
}
/**
* Render any value as JSON (for generic use).
* Ensures no ANSI codes are present.
*/
export function renderJson(value: unknown, indent = 2): string {
return JSON.stringify(value, null, indent);
}
+240
View File
@@ -0,0 +1,240 @@
/**
* S10: Renderers thread - NDJSON renderer
*
* Responsibilities:
* - Render step events as NDJSON lines
* - Event types: run.started, route.started, route.passed, route.failed, run.completed
* - Include timestamps
* - Flush after each event
*/
import type { Artifact, FailureRecord, NdjsonEvent } from '../core/types.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface NdjsonRendererOptions {
/** Output stream to write to (defaults to process.stdout) */
output?: NodeJS.WriteStream;
}
// ---------------------------------------------------------------------------
// Timestamp helper
// ---------------------------------------------------------------------------
/**
* Get current ISO timestamp.
*/
function getTimestamp(): string {
return new Date().toISOString();
}
// ---------------------------------------------------------------------------
// Event creation
// ---------------------------------------------------------------------------
/**
* Create a run.started event.
*/
export function createRunStartedEvent(
command: string,
seed?: number,
): NdjsonEvent {
return {
type: 'run.started',
command,
seed,
timestamp: getTimestamp(),
};
}
/**
* Create a route.started event.
*/
export function createRouteStartedEvent(route: string): NdjsonEvent {
return {
type: 'route.started',
route,
timestamp: getTimestamp(),
};
}
/**
* Create a route.passed event.
*/
export function createRoutePassedEvent(
route: string,
durationMs: number,
): NdjsonEvent {
return {
type: 'route.passed',
route,
durationMs,
timestamp: getTimestamp(),
};
}
/**
* Create a route.failed event.
*/
export function createRouteFailedEvent(
route: string,
failure: FailureRecord,
): NdjsonEvent {
return {
type: 'route.failed',
route,
failure,
timestamp: getTimestamp(),
};
}
/**
* Create a run.completed event.
*/
export function createRunCompletedEvent(
summary: Artifact['summary'],
): NdjsonEvent {
return {
type: 'run.completed',
summary,
timestamp: getTimestamp(),
};
}
// ---------------------------------------------------------------------------
// NDJSON rendering
// ---------------------------------------------------------------------------
/**
* Render a single NDJSON event as a JSON line.
*/
export function renderNdjsonEvent(event: NdjsonEvent): string {
return JSON.stringify(event);
}
/**
* Write an NDJSON event to the output stream.
* Flushes after each write.
*/
export function writeNdjsonEvent(
event: NdjsonEvent,
options: NdjsonRendererOptions = {},
): void {
const output = options.output || process.stdout;
const line = renderNdjsonEvent(event) + '\n';
output.write(line);
// Flush if possible (Node.js streams)
if ('flush' in output && typeof (output as any).flush === 'function') {
(output as any).flush();
}
}
/**
* Render a full artifact as NDJSON events.
* Emits the complete event sequence for a run.
*/
export function renderNdjsonArtifact(
artifact: Artifact,
options: NdjsonRendererOptions = {},
): void {
const output = options.output || process.stdout;
// run.started
writeNdjsonEvent(
createRunStartedEvent(artifact.command, artifact.seed),
options,
);
// Route events
for (const failure of artifact.failures) {
// For failed routes, emit started then failed
writeNdjsonEvent(createRouteStartedEvent(failure.route), options);
writeNdjsonEvent(createRouteFailedEvent(failure.route, failure), options);
}
// run.completed
writeNdjsonEvent(createRunCompletedEvent(artifact.summary), options);
}
/**
* Create all NDJSON events for an artifact without writing.
* Useful for testing.
*/
export function createNdjsonEvents(artifact: Artifact): NdjsonEvent[] {
const events: NdjsonEvent[] = [];
events.push(createRunStartedEvent(artifact.command, artifact.seed));
for (const failure of artifact.failures) {
events.push(createRouteStartedEvent(failure.route));
events.push(createRouteFailedEvent(failure.route, failure));
}
events.push(createRunCompletedEvent(artifact.summary));
return events;
}
// ---------------------------------------------------------------------------
// Concise / summary NDJSON rendering
// ---------------------------------------------------------------------------
/**
* Render a concise NDJSON artifact for CI/machine parsers.
* Emits only: run.started, run.summary, run.completed.
* Omits per-route events to reduce log volume.
*/
export function renderNdjsonSummaryArtifact(
artifact: Artifact,
options: NdjsonRendererOptions = {},
): void {
const output = options.output || process.stdout;
// run.started
writeNdjsonEvent(
createRunStartedEvent(artifact.command, artifact.seed),
options,
);
// run.summary with execution counts and gate info
writeNdjsonEvent(
{
type: 'run.summary',
summary: artifact.summary,
executionSummary: artifact.executionSummary,
profileGates: artifact.profileGates,
deterministicParams: artifact.deterministicParams,
timestamp: getTimestamp(),
} as unknown as NdjsonEvent,
options,
);
// run.completed
writeNdjsonEvent(createRunCompletedEvent(artifact.summary), options);
}
/**
* Create concise NDJSON events for an artifact without writing.
* Useful for testing summary mode.
*/
export function createNdjsonSummaryEvents(artifact: Artifact): NdjsonEvent[] {
const events: NdjsonEvent[] = [];
events.push(createRunStartedEvent(artifact.command, artifact.seed));
events.push({
type: 'run.summary',
summary: artifact.summary,
executionSummary: artifact.executionSummary,
profileGates: artifact.profileGates,
deterministicParams: artifact.deterministicParams,
timestamp: getTimestamp(),
} as unknown as NdjsonEvent);
events.push(createRunCompletedEvent(artifact.summary));
return events;
}
+193
View File
@@ -0,0 +1,193 @@
/**
* S10: Renderers thread - Shared utilities
*
* Shared utilities for all renderers:
* - Truncation for large payloads
* - Indentation helpers
* - Color detection logic
* - TTY/CI aware output helpers
* - Formatting utilities
*/
import pc from 'picocolors';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface TruncationOptions {
maxLength?: number;
suffix?: string;
}
export interface OutputContext {
isTTY: boolean;
isCI: boolean;
colorMode: 'auto' | 'always' | 'never';
}
// ---------------------------------------------------------------------------
// Color detection
// ---------------------------------------------------------------------------
/**
* Determine if colors should be enabled based on context.
* Respects --color flag: always=force on, never=force off, auto=detect.
*/
export function shouldUseColor(ctx: OutputContext): boolean {
if (ctx.colorMode === 'always') return true;
if (ctx.colorMode === 'never') return false;
// auto: use color if TTY and not CI
return ctx.isTTY && !ctx.isCI;
}
/**
* Get picocolors instance based on color preference.
* Returns a no-op proxy when colors are disabled.
*/
export function getColors(enabled: boolean): typeof pc {
if (enabled) return pc;
// Return a proxy that returns strings unchanged
return new Proxy(pc, {
get(target, prop) {
if (typeof target[prop as keyof typeof pc] === 'function') {
return (str: string) => str;
}
return target[prop as keyof typeof pc];
},
}) as typeof pc;
}
// ---------------------------------------------------------------------------
// Truncation
// ---------------------------------------------------------------------------
/**
* Truncate a string to maxLength, adding suffix if truncated.
*/
export function truncate(str: string, options: TruncationOptions = {}): string {
const { maxLength = 500, suffix = '...' } = options;
if (str.length <= maxLength) return str;
const truncatedLength = maxLength - suffix.length;
if (truncatedLength <= 0) return suffix;
return str.slice(0, truncatedLength) + suffix;
}
/**
* Truncate an object for terminal display.
* Converts to JSON and truncates.
*/
export function truncateObject(obj: unknown, options: TruncationOptions = {}): string {
const str = typeof obj === 'string' ? obj : JSON.stringify(obj, null, 2);
return truncate(str, options);
}
// ---------------------------------------------------------------------------
// Indentation
// ---------------------------------------------------------------------------
/**
* Indent each line of a string by n spaces.
*/
export function indent(str: string, spaces: number = 2): string {
const prefix = ' '.repeat(spaces);
return str
.split('\n')
.map(line => (line ? prefix + line : line))
.join('\n');
}
// ---------------------------------------------------------------------------
// Formatting helpers
// ---------------------------------------------------------------------------
/**
* Format a duration in milliseconds for human reading.
*/
export function formatDuration(ms: number): string {
if (ms < 1000) return `${ms}ms`;
return `${(ms / 1000).toFixed(2)}s`;
}
/**
* Format a timestamp as ISO string.
*/
export function formatTimestamp(date?: Date): string {
return (date || new Date()).toISOString();
}
/**
* Strip ANSI escape codes from a string.
*/
export function stripAnsi(str: string): string {
// eslint-disable-next-line no-control-regex
return str.replace(/\u001b\[[0-9;]*m/g, '');
}
/**
* Check if a string contains ANSI escape codes.
*/
export function hasAnsi(str: string): boolean {
// eslint-disable-next-line no-control-regex
return /\u001b\[[0-9;]*m/.test(str);
}
// ---------------------------------------------------------------------------
// TTY/CI output helpers
// ---------------------------------------------------------------------------
/**
* Determine if spinners should be shown.
* Never show spinners in CI or non-TTY environments.
*/
export function shouldShowSpinner(ctx: OutputContext): boolean {
return ctx.isTTY && !ctx.isCI;
}
/**
* Write to stdout with optional flushing.
* In non-TTY mode, always flush.
*/
export function writeStdout(str: string): void {
process.stdout.write(str);
}
/**
* Write line to stdout.
*/
export function writeLine(str: string = ''): void {
process.stdout.write(str + '\n');
}
// ---------------------------------------------------------------------------
// Progress helpers
// ---------------------------------------------------------------------------
/**
* Format a progress indicator (no spinner, just text).
* Safe for CI/non-TTY.
*/
export function formatProgress(current: number, total: number, label?: string): string {
const pct = total > 0 ? Math.round((current / total) * 100) : 0;
const prefix = label ? `${label} ` : '';
return `${prefix}[${current}/${total}] ${pct}%`;
}
// ---------------------------------------------------------------------------
// Summary formatting
// ---------------------------------------------------------------------------
/**
* Format a summary block for human output.
*/
export function formatSummary(total: number, passed: number, failed: number): string {
const lines: string[] = [];
lines.push(`Total: ${total}`);
lines.push(`Passed: ${passed}`);
lines.push(`Failed: ${failed}`);
return lines.join('\n');
}
+97
View File
@@ -0,0 +1,97 @@
import type { OperationCategory } from '../types.js'
/**
* Category inference for route contracts.
* Pure functions, no side effects.
* Optimized: direct string comparison over Set for hot path.
*/
// Fast path: direct string comparison for most common utility paths
const isUtilityPath = (path: string): boolean => {
// Check exact matches first (fastest)
if (
path === '/reset' ||
path === '/health' ||
path === '/ping' ||
path === '/login' ||
path === '/logout' ||
path === '/auth' ||
path === '/callback' ||
path === '/purge' ||
path === '/clear' ||
path === '/initialize' ||
path === '/setup' ||
path === '/webhook'
) {
return true
}
// Check trailing slash variants
const last = path.charCodeAt(path.length - 1)
if (last === 47) {
const base = path.slice(0, -1)
return (
base === '/reset' ||
base === '/health' ||
base === '/ping' ||
base === '/login' ||
base === '/logout' ||
base === '/auth' ||
base === '/callback' ||
base === '/purge' ||
base === '/clear' ||
base === '/initialize' ||
base === '/setup' ||
base === '/webhook'
)
}
return false
}
// Fast path: check last 7 chars for common suffixes
const isObserverSuffix = (path: string): boolean => {
const len = path.length
if (len >= 7) {
const end = path.slice(-7)
if (end === '/search') return true
}
if (len >= 6) {
const end = path.slice(-6)
if (end === '/count' || end === '/stats') return true
}
if (len >= 7) {
const end = path.slice(-7)
if (end === '/status') return true
}
return false
}
const isCollectionPath = (path: string): boolean => {
const len = path.length
let i = len - 1
// Skip trailing slash
while (i > 0 && path.charCodeAt(i) === 47) i--
// Find last segment
const lastSlash = path.lastIndexOf('/', i)
const segment = path.slice(lastSlash + 1, i + 1)
return segment.length > 0 && segment.charCodeAt(0) !== 58 // ':'.charCodeAt(0) === 58
}
const hasPathParam = (path: string): boolean => path.includes(':')
export const inferCategory = (
path: string,
method: string,
override: string | undefined
): OperationCategory => {
if (override !== undefined && override !== '') {
return override as OperationCategory
}
if (isUtilityPath(path)) {
return 'utility'
}
const upperMethod = method.toUpperCase()
if (upperMethod === 'GET' || isObserverSuffix(path)) {
return 'observer'
}
if (upperMethod === 'POST' && isCollectionPath(path)) {
return 'constructor'
}
if (upperMethod === 'PUT' || upperMethod === 'PATCH' || upperMethod === 'DELETE' || (upperMethod === 'POST' && hasPathParam(path))) {
return 'mutator'
}
return 'observer'
}
+382
View File
@@ -0,0 +1,382 @@
/**
* Contract Validation with Rich Error Context
* Validates postconditions and returns structured errors.
* Backward compatible: error is always a string, violation is optional.
*/
import { parse } from '../formula/parser.js'
import { evaluateAsync, evaluateBooleanResult, evaluateWithExtensions } from '../formula/evaluator.js'
import { getSuggestion, formatDiff } from './error-suggestions.js'
import { getErrorMessage } from '../infrastructure/http-executor.js'
import type { ExtensionRegistry } from '../extension/types.js'
import type { FormulaNode } from './formula.js'
import type { ContractViolation, EvalContext, EvalResult, RouteContract } from '../types.js'
// ---------------------------------------------------------------------------
// Core helpers
// ---------------------------------------------------------------------------
const makeViolation = (
partial: Omit<ContractViolation, 'type' | 'kind' | 'suggestion'> & { kind?: ContractViolation['kind']; suggestion?: string }
): ContractViolation => ({
type: 'contract-violation',
kind: partial.kind ?? 'postcondition',
suggestion: partial.suggestion ?? 'Contract violation detected. Review the formula and response.',
...partial,
})
const getFieldValue = (obj: unknown, path: string): unknown => {
const parts = path.split('.')
let current: unknown = obj
for (const part of parts) {
if (current === null || current === undefined || typeof current !== 'object') {
return undefined
}
current = (current as Record<string, unknown>)[part]
}
return current
}
const extractExpectedFromEquality = (formula: string): string | undefined => {
const match = formula.match(/==\s*["']?([^"']+)["']?/)
return match?.[1]
}
const extractFieldPath = (formula: string): string | undefined => {
const match = formula.match(/(?:response_body\(this\)|response_payload\(this\)|request_body\(this\)|request_headers\(this\)|response_headers\(this\)|query_params\(this\)|request_params\(this\)|cookies\(this\)|response_time\(this\))(?:\.?([\w.\[\]]+))?/)
return match?.[1]
}
const isLegacyPreconditionSyntax = (formula: string): boolean =>
/^[a-zA-Z][a-zA-Z0-9_-]*:[^\s()]+$/.test(formula) &&
!formula.startsWith('status:')
const clauseLabelForKind = (kind: ContractViolation['kind']): 'x-requires' | 'x-ensures' =>
kind === 'precondition' ? 'x-requires' : 'x-ensures'
const formatRouteClauseContext = (
kind: ContractViolation['kind'],
route: RouteContract | { method: string; path: string } | undefined,
index: number,
formula: string
): string => {
const routeCtx = route ? `${route.method} ${route.path}` : 'unknown route'
return `${routeCtx} ${clauseLabelForKind(kind)}[${index}] "${formula}"`
}
const buildLegacyPreconditionMessage = (formula: string): string =>
`Legacy precondition syntax is no longer supported: "${formula}". ` +
'Use APOSTL formulas in x-requires, for example request_params(this).id != null or response_code(GET /users/{request_params(this).id}) == 200.'
const parseFormula = (formula: string, extensionRegistry?: ExtensionRegistry): FormulaNode => {
const extensionHeaders = extensionRegistry?.getExtensionHeaders() ?? []
return parse(formula, extensionHeaders).ast
}
const evaluateParsedFormula = (
ast: FormulaNode,
ctx: EvalContext,
route?: RouteContract | { method: string; path: string },
extensionRegistry?: ExtensionRegistry
): boolean => {
if (extensionRegistry && route && 'category' in route) {
const evalResult = evaluateWithExtensions(ast, ctx, route as RouteContract, extensionRegistry)
if (!evalResult.success) {
throw new Error(evalResult.error)
}
return Boolean(evalResult.value)
}
return evaluateBooleanResult(ast, ctx)
}
const evaluateParsedFormulaAsync = async (
ast: FormulaNode,
ctx: EvalContext,
route?: RouteContract | { method: string; path: string },
extensionRegistry?: ExtensionRegistry
): Promise<boolean> => {
const evalResult = await evaluateAsync(
ast,
ctx,
route && 'category' in route ? route as RouteContract : undefined,
extensionRegistry
)
if (!evalResult.success) {
throw new Error(evalResult.error)
}
return Boolean(evalResult.value)
}
// ---------------------------------------------------------------------------
// Violation builders — extracted from makeConditionFailure to reduce nesting
// ---------------------------------------------------------------------------
const resolveStatusExpectation = (
formula: string,
ast: FormulaNode | undefined,
statusCode: number
): { expected: string; actual: string } => {
if (ast?.type === 'status') {
return { expected: String(ast.code), actual: String(statusCode) }
}
const statusMatch = formula.match(/status:(\d+)/)
if (statusMatch) {
return { expected: statusMatch[1] ?? 'true', actual: String(statusCode) }
}
return { expected: 'true', actual: 'false' }
}
const resolveFieldNullExpectation = (
formula: string,
body: unknown
): { expected: string; actual: string } | null => {
const fieldMatch = formula.match(/(?:response_body\(this\)|response_payload\(this\)|request_body\(this\)|request_headers\(this\)|response_headers\(this\)|query_params\(this\)|request_params\(this\))\.(\w[\w.\[\]]*)/)
if (!fieldMatch || !formula.includes('!= null')) return null
const fieldPath = fieldMatch[1]
if (!fieldPath) return null
const parts = fieldPath.split('.')
let current: unknown = body
let exists = true
for (const part of parts) {
if (current === null || current === undefined || typeof current !== 'object') {
exists = false
break
}
current = (current as Record<string, unknown>)[part]
}
if (!exists || current === undefined) {
return { expected: 'non-null value', actual: 'undefined (field missing)' }
}
if (current === null) {
return { expected: 'non-null value', actual: 'null' }
}
return null
}
const buildDiff = (formula: string, body: unknown): string | null => {
if (!formula.includes('==') || formula.includes('!=')) return null
const fieldPath = extractFieldPath(formula)
const expectedValue = extractExpectedFromEquality(formula)
if (!fieldPath || !expectedValue) return null
const actualValue = getFieldValue(body, fieldPath)
return formatDiff(expectedValue, String(actualValue ?? 'undefined'))
}
const makeConditionFailure = (
kind: ContractViolation['kind'],
formula: string,
ctx: EvalContext,
route: RouteContract | { method: string; path: string } | undefined,
ast?: FormulaNode
): ContractViolation => {
const statusExpectation = resolveStatusExpectation(formula, ast, ctx.response.statusCode)
const fieldExpectation = resolveFieldNullExpectation(formula, ctx.response.body)
const expected = fieldExpectation?.expected ?? statusExpectation.expected
const actual = fieldExpectation?.actual ?? statusExpectation.actual
const diff = buildDiff(formula, ctx.response.body)
return makeViolation({
route: route ?? { method: '', path: '' },
formula,
kind,
request: {
body: ctx.request.body,
headers: ctx.request.headers,
query: ctx.request.query,
params: ctx.request.params,
},
response: {
statusCode: ctx.response.statusCode,
headers: ctx.response.headers,
body: ctx.response.body,
},
context: { expected, actual, diff },
})
}
const makeFormulaError = (
kind: ContractViolation['kind'],
formula: string,
ctx: EvalContext,
route: RouteContract | { method: string; path: string } | undefined,
message: string
): ContractViolation => {
return makeViolation({
route: route ?? { method: '', path: '' },
formula,
kind,
request: {
body: ctx.request.body,
headers: ctx.request.headers,
query: ctx.request.query,
params: ctx.request.params,
},
response: {
statusCode: ctx.response.statusCode,
headers: ctx.response.headers,
body: ctx.response.body,
},
context: { expected: 'valid formula', actual: `parse error: ${message}`, diff: null },
suggestion: `Formula evaluation failed: ${message}. Check your contract syntax.`,
})
}
// ---------------------------------------------------------------------------
// Shared validation body — extracted to avoid duplication between sync/async
// ---------------------------------------------------------------------------
const runValidationBody = (
kind: ContractViolation['kind'],
formula: string,
ctx: EvalContext,
route: RouteContract | { method: string; path: string } | undefined,
extensionRegistry: ExtensionRegistry | undefined
): EvalResult | null => {
if (kind === 'precondition' && isLegacyPreconditionSyntax(formula)) {
throw new Error(buildLegacyPreconditionMessage(formula))
}
const ast = parseFormula(formula, extensionRegistry)
const result = evaluateParsedFormula(ast, ctx, route, extensionRegistry)
if (!result) {
const violation = makeConditionFailure(kind, formula, ctx, route, ast)
if (extensionRegistry) {
extensionRegistry.runViolationHooks(violation).catch((err: unknown) => {
console.warn(`Extension violation hook failed: ${getErrorMessage(err)}`)
})
}
return {
success: false,
error: `Contract violation: ${formula}`,
violation: { ...violation, suggestion: getSuggestion(violation) },
}
}
return null
}
const runValidationBodyAsync = async (
kind: ContractViolation['kind'],
formula: string,
ctx: EvalContext,
route: RouteContract | { method: string; path: string } | undefined,
extensionRegistry: ExtensionRegistry | undefined
): Promise<EvalResult | null> => {
if (kind === 'precondition' && isLegacyPreconditionSyntax(formula)) {
throw new Error(buildLegacyPreconditionMessage(formula))
}
const ast = parseFormula(formula, extensionRegistry)
const result = await evaluateParsedFormulaAsync(ast, ctx, route, extensionRegistry)
if (!result) {
const violation = makeConditionFailure(kind, formula, ctx, route, ast)
if (extensionRegistry) {
extensionRegistry.runViolationHooks(violation).catch((err: unknown) => {
console.warn(`Extension violation hook failed: ${getErrorMessage(err)}`)
})
}
return {
success: false,
error: `Contract violation: ${formula}`,
violation: { ...violation, suggestion: getSuggestion(violation) },
}
}
return null
}
// ---------------------------------------------------------------------------
// Sync / Async wrappers — thin loops over the shared body
// ---------------------------------------------------------------------------
const runValidationSync = (
kind: ContractViolation['kind'],
formulas: string[],
ctx: EvalContext,
route?: RouteContract | { method: string; path: string },
extensionRegistry?: ExtensionRegistry
): EvalResult => {
for (let index = 0; index < formulas.length; index++) {
const formula = formulas[index] ?? ''
try {
const stepResult = runValidationBody(kind, formula, ctx, route, extensionRegistry)
if (stepResult) {
return stepResult
}
} catch (err) {
const msg = getErrorMessage(err)
const violation = makeFormulaError(kind, formula, ctx, route, msg)
const routeCtx = formatRouteClauseContext(kind, route, index, formula)
return {
success: false,
error: `Formula error in ${routeCtx}: ${msg}`,
violation,
}
}
}
return { success: true, value: ctx.response.statusCode }
}
const runValidationAsync = async (
kind: ContractViolation['kind'],
formulas: string[],
ctx: EvalContext,
route?: RouteContract | { method: string; path: string },
extensionRegistry?: ExtensionRegistry
): Promise<EvalResult> => {
for (let index = 0; index < formulas.length; index++) {
const formula = formulas[index] ?? ''
try {
const stepResult = await runValidationBodyAsync(kind, formula, ctx, route, extensionRegistry)
if (stepResult) {
return stepResult
}
} catch (err) {
const msg = getErrorMessage(err)
const violation = makeFormulaError(kind, formula, ctx, route, msg)
const routeCtx = formatRouteClauseContext(kind, route, index, formula)
return {
success: false,
error: `Formula error in ${routeCtx}: ${msg}`,
violation,
}
}
}
return { success: true, value: ctx.response.statusCode }
}
/**
* Validate a set of postcondition formulas against an evaluation context.
* Returns string error for backward compatibility, with optional rich violation.
*/
export const validatePostconditions = (
ensures: string[],
ctx: EvalContext,
route?: RouteContract | { method: string; path: string },
extensionRegistry?: ExtensionRegistry
): EvalResult => {
return runValidationSync('postcondition', ensures, ctx, route, extensionRegistry)
}
export const validatePostconditionsAsync = (
ensures: string[],
ctx: EvalContext,
route?: RouteContract | { method: string; path: string },
extensionRegistry?: ExtensionRegistry
): Promise<EvalResult> => {
return runValidationAsync('postcondition', ensures, ctx, route, extensionRegistry)
}
export const validatePreconditionsAsync = (
requires: string[],
ctx: EvalContext,
route?: RouteContract | { method: string; path: string },
extensionRegistry?: ExtensionRegistry
): Promise<EvalResult> => {
return runValidationAsync('precondition', requires, ctx, route, extensionRegistry)
}
+117
View File
@@ -0,0 +1,117 @@
import { inferCategory } from './category.js'
import { inferContractsFromRouteSchema } from './schema-to-contract.js'
// Reuse empty arrays to avoid allocation
import type { HttpMethod, OutboundBinding, RouteContract, ValidatedFormula } from '../types.js'
const EMPTY_REQUIRES: ValidatedFormula[] = []
const EMPTY_ENSURES: ValidatedFormula[] = []
const EMPTY_INVARIANTS: ValidatedFormula[] = []
// Two-level cache: WeakMap<schema, Map<"METHOD path", RouteContract>>
// Preserves automatic GC of schema objects while correctly caching per-route contracts
const contractCache = new WeakMap<Record<string, unknown>, Map<string, RouteContract>>()
export const extractContract = (
path: string,
method: string,
schema: Record<string, unknown> | undefined
): RouteContract => {
const s = schema ?? {}
// Fast path: two-level cache lookup (guard against null — WeakMap rejects null keys)
if (schema != null) {
let routeMap = contractCache.get(schema)
if (routeMap === undefined) {
routeMap = new Map()
contractCache.set(schema, routeMap)
}
const key = `${method.toUpperCase()} ${path}`
const cached = routeMap.get(key)
if (cached !== undefined) {
return cached
}
}
const override = typeof s['x-category'] === 'string' ? s['x-category'] : undefined
const category = inferCategory(path, method, override)
// APOPHIS annotations may live on the top-level schema OR nested inside
// response.statusCode (e.g. schema.response[200]['x-ensures']).
// We merge both levels so contracts are never silently dropped.
const responseSchema = (s.response ?? {}) as Record<string, Record<string, unknown>>
const firstStatus = Object.values(responseSchema)[0] ?? {}
const topRequires = s['x-requires']
const nestedRequires = firstStatus['x-requires']
const requires = Array.isArray(topRequires) && topRequires.length > 0
? (topRequires as string[])
: Array.isArray(nestedRequires) && nestedRequires.length > 0
? (nestedRequires as string[])
: EMPTY_REQUIRES
const topEnsures = s['x-ensures']
const nestedEnsures = firstStatus['x-ensures']
const explicitEnsures = Array.isArray(topEnsures) && topEnsures.length > 0
? (topEnsures as string[])
: Array.isArray(nestedEnsures) && nestedEnsures.length > 0
? (nestedEnsures as string[])
: []
// Infer contracts from JSON Schema constraints (required, minimum, maximum, pattern, etc.)
// These supplement explicit x-ensures — never replace them.
const inferred = inferContractsFromRouteSchema(s)
const inferredSet = new Set(inferred)
const explicitSet = new Set(explicitEnsures)
// Deduplicate: don't add inferred formulas that the user already wrote explicitly
const additionalInferred = inferred.filter(f => !explicitSet.has(f))
// Merge: explicit first, then inferred
const ensures = explicitEnsures.length > 0 || additionalInferred.length > 0
? [...explicitEnsures, ...additionalInferred]
: EMPTY_ENSURES
const validateRuntime =
(s['x-validate-runtime'] !== false) &&
(firstStatus['x-validate-runtime'] !== false)
// Extract timeout from schema annotation
const timeoutValue = s['x-timeout'] ?? firstStatus['x-timeout']
const timeout = typeof timeoutValue === 'number' && timeoutValue > 0
? timeoutValue
: undefined
// Parse x-outbound annotation
const outboundRaw = s['x-outbound']
const outbound: OutboundBinding[] | undefined = Array.isArray(outboundRaw)
? (outboundRaw as OutboundBinding[])
: undefined
// Parse x-variants annotation
const variantsRaw = s['x-variants']
const variants = Array.isArray(variantsRaw)
? variantsRaw.map((v: unknown) => {
if (typeof v === 'string') {
return { name: v }
}
if (v !== null && typeof v === 'object') {
const vo = v as Record<string, unknown>
return {
name: String(vo.name || 'unnamed'),
headers: vo.headers as Record<string, string> | undefined,
when: vo.when as string | undefined,
}
}
return { name: String(v) }
})
: undefined
const contract: RouteContract = {
path,
method: method.toUpperCase() as HttpMethod,
category,
requires: requires as ValidatedFormula[],
ensures: ensures as ValidatedFormula[],
invariants: EMPTY_INVARIANTS,
regexPatterns: {},
validateRuntime,
schema: s,
timeout,
outbound,
variants,
}
if (schema !== undefined) {
const key = `${method.toUpperCase()} ${path}`
let routeMap = contractCache.get(schema)
if (routeMap === undefined) {
routeMap = new Map()
contractCache.set(schema, routeMap)
}
routeMap.set(key, contract)
}
return contract
}
+95
View File
@@ -0,0 +1,95 @@
/**
* Route discovery from a Fastify instance.
* Pure functions, no side effects.
*
* Fastify 5 removed the public `routes` array. We capture routes via the `onRoute`
* hook during plugin registration and store them in a WeakMap keyed by the instance.
*/
import { extractContract } from './contract.js'
import type { RouteContract } from '../types.js'
interface CapturedRoute {
method: string
url: string
schema?: Record<string, unknown>
prefix?: string
}
// WeakMap to store captured routes per Fastify instance (no memory leaks)
const capturedRoutes = new WeakMap<object, CapturedRoute[]>()
/**
* Capture a route for discovery.
* Called from the plugin's `onRoute` hook.
*/
export const captureRoute = (
instance: object,
route: CapturedRoute
): void => {
const existing = capturedRoutes.get(instance) ?? []
existing.push(route)
capturedRoutes.set(instance, existing)
}
/**
* Fallback route discovery for Fastify 5 when routes were registered before
* the APOPHIS plugin (e.g., external apps loaded by CLI).
* Uses hasRoute to test known route patterns.
*/
function discoverRoutesFallback(
instance: { hasRoute?: (opts: { method: string; url: string }) => boolean }
): RouteContract[] {
if (typeof instance.hasRoute !== 'function') {
return []
}
// Common HTTP methods to test
const methods = ['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD', 'OPTIONS']
// We can't enumerate all possible routes, but we can check if the instance
// has any routes at all by testing a few common patterns
// This is a best-effort fallback
const routes: RouteContract[] = []
// Try to extract routes from the instance's internal state
// Fastify stores routes in find-my-way router, but it's not directly accessible
// We'll use a heuristic: check if the instance responds to common route methods
// Check if instance has any routes by looking at prototype methods
const hasRouting = typeof (instance as any).routing === 'function'
if (!hasRouting) {
return []
}
// Since we can't enumerate routes in Fastify 5 without the onRoute hook,
// we return empty and let the caller handle the "no routes" case
return []
}
/**
* Discover routes from a Fastify instance.
*
* First checks captured routes (from onRoute hook), then falls back to
* the legacy `routes` array for Fastify 4 compatibility.
*/
export const discoverRoutes = (instance: { routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> }>; hasRoute?: (opts: { method: string; url: string }) => boolean }): RouteContract[] => {
// Fastify 5: routes captured via onRoute hook
const captured = capturedRoutes.get(instance)
if (captured && captured.length > 0) {
return captured.map((route) =>
extractContract(route.url, route.method, route.schema)
)
}
// Fastify 4 fallback
if (Array.isArray(instance.routes) && instance.routes.length > 0) {
return instance.routes.map((route) =>
extractContract(route.url, route.method, route.schema)
)
}
// Fastify 5 fallback: routes registered before plugin
return discoverRoutesFallback(instance)
}
/**
* Clear captured routes for an instance (useful for testing).
*/
export const clearCapturedRoutes = (instance: object): void => {
capturedRoutes.delete(instance)
}
+157
View File
@@ -0,0 +1,157 @@
import type { ContractViolation } from '../types.js'
/**
* Error Suggestions Engine
* Maps common contract violation patterns to actionable developer guidance.
* Pure functions: no side effects.
*/
// ─── Extractors ────────────────────────────────────────────────────────────
const extractField = (formula: string): string | undefined => {
const match = formula.match(/(?:response_body\(this\)|response_payload\(this\)|request_body\(this\)|request_headers\(this\)|response_headers\(this\)|query_params\(this\))\.(\w[\w.]*)\s*!=\s*null/)
return match?.[1]
}
const extractExpectedValue = (formula: string): string | undefined => {
const match = formula.match(/==\s*["']?([^"']+)["']?/)
return match?.[1]
}
const extractFieldPath = (formula: string): string | undefined => {
const match = formula.match(/(?:response_body\(this\)|response_payload\(this\)|request_body\(this\)|request_headers\(this\)|response_headers\(this\)|query_params\(this\)|cookies\(this\)|response_time\(this\))\.(\w[\w.]*)/)
return match?.[1]
}
const STATUS_CODE_NAMES: Record<number, string> = {
200: 'OK', 201: 'Created', 204: 'No Content',
400: 'Bad Request', 401: 'Unauthorized', 403: 'Forbidden',
404: 'Not Found', 409: 'Conflict', 422: 'Unprocessable Entity',
500: 'Internal Server Error',
}
// ─── Pattern Matchers ──────────────────────────────────────────────────────
/** A matcher returns a suggestion string, or undefined if it doesn't match. */
type Matcher = (v: ContractViolation) => string | undefined
const statusCodeMatcher: Matcher = (v) => {
if (v.kind !== 'postcondition' || !v.formula.startsWith('status:')) return
const expected = parseInt(v.context.expected, 10)
const actual = v.response.statusCode
const expectedName = STATUS_CODE_NAMES[expected] ?? ''
const actualName = STATUS_CODE_NAMES[actual] ?? ''
if (actual === 400 || actual === 422) {
return `Your handler rejected the request with ${actual} ${actualName}, but the contract expects ${expected} ${expectedName}. This usually means validation failed. Check that your request body matches the schema constraints.`
}
if (actual === 401 || actual === 403) {
return `Authentication/authorization failed (${actual} ${actualName}). Check that required headers (authorization, x-tenant-id) are present in the request.`
}
if (actual === 404) {
return `Resource not found (404). This may indicate a precondition failure — the test tried to access a resource that was not created in a previous constructor route.`
}
if (actual === 500) {
return `Server error (500). Your route handler threw an unhandled exception. Check server logs for the stack trace.`
}
return `Expected status ${expected} ${expectedName}, got ${actual} ${actualName}. Check your route handler's reply.status() call.`
}
const missingFieldMatcher: Matcher = (v) => {
if (!v.formula.includes('!= null') || !v.context.actual.startsWith('undefined')) return
const field = extractField(v.formula) ?? extractFieldPath(v.formula)
if (!field) return
return `Field '${field}' is missing from the response body. Ensure your handler returns all required fields. Check that the field name matches exactly (case-sensitive).`
}
const nullFieldMatcher: Matcher = (v) => {
if (!v.formula.includes('!= null') || v.context.actual !== 'null') return
const field = extractField(v.formula) ?? extractFieldPath(v.formula)
if (!field) return
return `Field '${field}' is explicitly null in the response. If null is valid, change the contract to allow it. Otherwise, ensure your handler populates this field.`
}
const temporalMatcher: Matcher = (v) => {
if (!v.formula.includes('previous(')) return
return `Temporal contract failed. The current response does not satisfy the relationship with the previous request's response. Ensure state transitions are valid.`
}
const equalityMatcher: Matcher = (v) => {
if (!v.formula.includes('==')) return
const field = extractFieldPath(v.formula)
const expected = extractExpectedValue(v.formula)
if (field && expected) {
return `Field '${field}' does not match expected value '${expected}'. Check for typos, case sensitivity, or missing data transformations in your handler.`
}
return `Values do not match. Check for typos, case sensitivity, or type mismatches (string vs number).`
}
const regexMatcher: Matcher = (v) => {
if (!v.formula.includes(' matches ')) return
return `String does not match the required pattern. Check that the regex in your contract is correct and that the response format matches expectations.`
}
const comparisonMatcher: Matcher = (v) => {
if (!v.formula.includes('>') && !v.formula.includes('<')) return
if (v.formula.includes('=>')) return
return `Numeric comparison failed. Check that calculated values (counts, timestamps, sizes) are within expected bounds.`
}
const headerMatcher: Matcher = (v) => {
if (!v.formula.includes('request_headers(this)') && !v.formula.includes('response_headers(this)')) return
return `Header check failed. Ensure the expected header is present, correctly named (headers are case-insensitive), and has the expected value. Check that your Fastify hook or middleware sets the header.`
}
const authMatcher: Matcher = (v) => {
if (!v.formula.includes('authorization') && !v.formula.includes('x-tenant-id') && !v.formula.includes('tenantId')) return
return `Authorization or tenant isolation check failed. Ensure required headers are present and the scope registry is configured correctly.`
}
const implicationMatcher: Matcher = (v) => {
if (!v.formula.includes('=>')) return
return `Conditional contract failed. The 'if' condition was true but the 'then' condition was false. Review the logic in your contract.`
}
const justinImplicationMatcher: Matcher = (v) => {
if (!v.formula.includes('||')) return
return `Conditional contract failed. The 'if' condition was true but the 'then' condition was false. Review the logic in your contract.`
}
const responseTimeMatcher: Matcher = (v) => {
if (!v.formula.includes('response_time(this)')) return
return `Response time check failed. The request took longer than expected. Consider optimizing database queries, adding caching, or increasing the timeout threshold in your contract.`
}
const cookieMatcher: Matcher = (v) => {
if (!v.formula.includes('cookies')) return
return `Cookie check failed. Ensure the response includes the expected Set-Cookie header and that cookies are properly configured in your Fastify instance.`
}
const queryParamMatcher: Matcher = (v) => {
if (!v.formula.includes('query_params(this)')) return
return `Query parameter check failed. Verify that query parameters are correctly parsed and validated in your route handler. Check for missing or malformed query strings.`
}
// Matchers are evaluated in order — more specific patterns first
const matchers: Matcher[] = [
statusCodeMatcher,
missingFieldMatcher,
nullFieldMatcher,
temporalMatcher,
equalityMatcher,
regexMatcher,
comparisonMatcher,
headerMatcher,
authMatcher,
implicationMatcher,
justinImplicationMatcher,
responseTimeMatcher,
cookieMatcher,
queryParamMatcher,
]
// ─── Public API ────────────────────────────────────────────────────────────
/**
* Generate a human-readable suggestion for a contract violation.
*/
export const getSuggestion = (violation: ContractViolation): string => {
for (const matcher of matchers) {
const suggestion = matcher(violation)
if (suggestion !== undefined) return suggestion
}
return `Review the contract formula and your route handler implementation for mismatches.`
}
/**
* Generate a formatted diff for display.
*/
export const formatDiff = (expected: string, actual: string): string => {
if (expected === actual) {
return 'Values appear identical. Check for hidden characters or type differences (string vs number).'
}
if (typeof expected === 'string' && typeof actual === 'string' && expected.length < 100 && actual.length < 100) {
const maxLen = Math.max(expected.length, actual.length)
let diff = ''
for (let i = 0; i < maxLen; i++) {
if (expected[i] !== actual[i]) {
diff += ` Position ${i}: expected '${expected[i] ?? '(end)' }', got '${actual[i] ?? '(end)'}'\n`
}
}
return diff || 'Values are identical'
}
return `Expected: ${expected}\nActual: ${actual}`
}
+51
View File
@@ -0,0 +1,51 @@
/**
* APOSTL AST Types
* Formula parsing and evaluation types for the APOSTL contract language.
*/
export type FormulaNode =
| { type: 'literal'; value: boolean | number | string | null }
| { type: 'variable'; name: string; accessor?: string[] | undefined }
| { type: 'comparison'; op: Comparator; left: FormulaNode; right: FormulaNode }
| { type: 'boolean'; op: BooleanOperator; left: FormulaNode; right: FormulaNode }
| { type: 'conditional'; condition: FormulaNode; then: FormulaNode; else: FormulaNode }
| { type: 'quantified'; quantifier: 'for' | 'exists'; variable: string; collection: OperationCall; body: FormulaNode }
| { type: 'operation'; header: OperationHeader; parameter: OperationParameter; accessor?: string[] | undefined }
| { type: 'previous'; inner: FormulaNode }
| { type: 'status'; code: number }
export type Comparator = '==' | '!=' | '<=' | '>=' | '<' | '>' | 'matches'
export type BooleanOperator = '&&' | '||' | '=>'
export type OperationPathSegment =
| { type: 'text'; value: string }
| { type: 'expression'; expression: FormulaNode }
export type OperationHeader =
| 'request_body'
| 'response_body'
| 'response_payload'
| 'response_code'
| 'request_headers'
| 'response_headers'
| 'query_params'
| 'cookies'
| 'response_time'
| 'request_params'
| 'redirect_count'
| 'redirect_url'
| 'redirect_status'
| 'timeout_occurred'
| 'timeout_value'
| 'request_files'
| 'request_fields'
| 'stream_chunks'
| 'stream_duration'
| string
export type OperationParameter =
| { type: 'this' }
| { type: 'call'; method: 'GET'; path: OperationPathSegment[] }
export type OperationCall = { header: OperationHeader; parameter: OperationParameter; accessor?: string[] | undefined }
export interface ParseResult {
readonly ast: FormulaNode
readonly raw: string
}
+110
View File
@@ -0,0 +1,110 @@
/**
* Invariant Registry Cross-route assertions for stateful testing
* Generic: no domain-specific assumptions
*/
import type { ModelState } from './stateful.js'
import type { EvalContext } from '../types.js'
export interface InvariantResult {
readonly success: boolean
readonly error?: string
}
export interface Invariant {
readonly name: string
readonly description: string
readonly check: (state: ModelState, history: ReadonlyArray<EvalContext>) => InvariantResult
}
/**
* Check if all parent references in state point to existing resources.
*/
const checkParentReferences = (state: ModelState): InvariantResult => {
for (const [resourceType, resources] of state.resources) {
for (const [id, resource] of resources) {
if (resource.parentId && resource.parentType) {
const parents = state.resources.get(resource.parentType)
if (!parents || !parents.has(resource.parentId)) {
return {
success: false,
error: `Resource ${resourceType}(${id}) references non-existent parent ${resource.parentType}(${resource.parentId})`,
}
}
}
}
}
return { success: true }
}
/**
* Check for orphaned resources (resources that should have parents but don't).
*/
const checkOrphanedResources = (state: ModelState): InvariantResult => {
for (const [resourceType, resources] of state.resources) {
for (const [id, resource] of resources) {
if (resource.parentType && !resource.parentId) {
return {
success: false,
error: `Resource ${resourceType}(${id}) declares parentType ${resource.parentType} but has no parentId`,
}
}
}
}
return { success: true }
}
/**
* Check that all resources have required fields.
*/
const checkResourceIntegrity = (state: ModelState): InvariantResult => {
for (const [resourceType, resources] of state.resources) {
for (const [id, resource] of resources) {
if (!resource.id || !resource.type) {
return {
success: false,
error: `Resource ${resourceType}(${id}) is missing required fields`,
}
}
}
}
return { success: true }
}
/**
* Built-in invariants that apply to any API.
*/
export const BUILTIN_INVARIANTS: Invariant[] = [
{
name: 'parent-reference-integrity',
description: 'All parent references must point to existing resources',
check: checkParentReferences,
},
{
name: 'no-orphaned-resources',
description: 'Resources with parentType must have a parentId',
check: checkOrphanedResources,
},
{
name: 'resource-integrity',
description: 'All resources must have required fields',
check: checkResourceIntegrity,
},
]
/**
* Resolve which invariants to check based on config.
*/
export function resolveInvariants(
config: string[] | false | undefined
): Invariant[] {
if (config === false) return []
if (config === undefined) return BUILTIN_INVARIANTS
return BUILTIN_INVARIANTS.filter(inv => config.includes(inv.name))
}
/**
* Check a set of invariants against current state.
*/
export const checkInvariants = (
invariants: ReadonlyArray<Invariant>,
state: ModelState,
history: ReadonlyArray<EvalContext>
): Array<{ name: string; result: InvariantResult }> => {
return invariants.map((inv) => ({
name: inv.name,
result: inv.check(state, history),
}))
}
+62
View File
@@ -0,0 +1,62 @@
import type { OutboundBinding, OutboundContractSpec, ResolvedOutboundContract } from '../types.js'
/**
* Outbound Contract Registry
*
* Normalizes, resolves, and validates `x-outbound` bindings against the shared
* plugin-level registry. Pure functions no side effects.
*/
export class OutboundContractRegistry {
private contracts: Map<string, OutboundContractSpec> = new Map()
register(name: string, spec: OutboundContractSpec): void {
this.contracts.set(name, spec)
}
registerAll(contracts: Record<string, OutboundContractSpec>): void {
for (const [name, spec] of Object.entries(contracts)) {
this.contracts.set(name, spec)
}
}
get(name: string): OutboundContractSpec | undefined {
return this.contracts.get(name)
}
has(name: string): boolean {
return this.contracts.has(name)
}
resolve(bindings: readonly OutboundBinding[]): ResolvedOutboundContract[] {
return bindings.map((b) => this.resolveBinding(b))
}
private resolveBinding(binding: OutboundBinding): ResolvedOutboundContract {
if (typeof binding === 'string') {
const spec = this.contracts.get(binding)
if (!spec) {
throw new Error(
`Outbound contract '${binding}' not found in registry. ` +
`Did you forget to register it via outboundContracts?`
)
}
return { name: binding, ...spec }
}
if ('ref' in binding) {
const spec = this.contracts.get(binding.ref)
if (!spec) {
throw new Error(
`Outbound contract '${binding.ref}' not found in registry. ` +
`Did you forget to register it via outboundContracts?`
)
}
return {
name: binding.ref,
...spec,
chaos: binding.chaos ?? spec.chaos,
}
}
// Inline contract
return {
name: binding.name,
target: binding.target,
method: binding.method,
request: binding.request,
response: binding.response,
chaos: binding.chaos,
}
}
}
+190
View File
@@ -0,0 +1,190 @@
/**
* Plugin Contract System
*
* Enables Fastify plugins to declare APOPHIS contracts that are
* automatically merged into route contracts at test time.
*/
import type { PluginContractSpec, ComposedContract } from '../plugin/contracts.js'
// ============================================================================
import type { RouteContract } from '../types.js'
// Pattern Matching
// ============================================================================
function matchPattern(pattern: string, path: string): boolean {
// Exact match
if (pattern === path) return true
// Double-wildcard: matches everything
if (pattern === '**') return true
// Wildcard match: '/api/**' matches '/api/users', '/api/users/:id'
if (pattern.endsWith('/**')) {
const prefix = pattern.slice(0, -3)
if (prefix === '') return true // '/**' matches everything
return path.startsWith(prefix)
}
// Prefix match: '/api/*' matches '/api/users' but not '/api/users/:id'
if (pattern.endsWith('/*')) {
const prefix = pattern.slice(0, -2)
if (!path.startsWith(prefix)) return false
const remainder = path.slice(prefix.length)
// remainder should be empty or a single segment without nested slashes
if (remainder === '') return true
// Remove leading slash and check no more slashes
const trimmed = remainder.startsWith('/') ? remainder.slice(1) : remainder
return !trimmed.includes('/')
}
return false
}
// ============================================================================
// Plugin Contract Registry
// ============================================================================
export class PluginContractRegistry {
private contracts = new Map<string, PluginContractSpec>()
private availableExtensions = new Set<string>()
/**
* Register a plugin's contract specification.
* Idempotent: registering the same plugin twice updates the spec.
*/
register(name: string, spec: PluginContractSpec): void {
this.contracts.set(name, spec)
}
/**
* Register available Apophis extensions.
* Called by the extension registry when extensions are added.
*/
registerAvailableExtension(name: string): void {
this.availableExtensions.add(name)
}
/**
* Check if all required extensions for a plugin are available.
*/
checkExtensions(spec: PluginContractSpec): { available: boolean; missing: string[] } {
const missing: string[] = []
for (const ext of spec.extensions ?? []) {
if (ext.required !== false && !this.availableExtensions.has(ext.name)) {
missing.push(ext.name)
}
}
return { available: missing.length === 0, missing }
}
/**
* Find all plugin contracts that apply to a given route.
* Skips plugins whose required extensions are not available.
*/
findContractsForRoute(route: RouteContract): Array<{ plugin: string; spec: PluginContractSpec }> {
const matches: Array<{ plugin: string; spec: PluginContractSpec }> = []
for (const [plugin, spec] of this.contracts) {
if (!matchPattern(spec.appliesTo, route.path)) continue
const extCheck = this.checkExtensions(spec)
if (!extCheck.available) {
console.warn(
`Plugin '${plugin}' requires extensions [${extCheck.missing.join(', ')}] which are not registered. Skipping its contracts.`
)
continue
}
matches.push({ plugin, spec })
}
return matches
}
/**
* Merge route contracts with applicable plugin contracts.
*/
composeContracts(route: RouteContract): ComposedContract {
const pluginContracts = this.findContractsForRoute(route)
const phases: ComposedContract['phases'] = {}
// Route-level contracts go into 'route' phase
phases.route = {
requires: route.requires.map((f) => ({ formula: f, source: 'route' as const })),
ensures: route.ensures.map((f) => ({ formula: f, source: 'route' as const })),
}
// Merge plugin contracts by phase
for (const { plugin, spec } of pluginContracts) {
for (const [phase, contracts] of Object.entries(spec.hooks)) {
if (!phases[phase]) {
phases[phase] = { requires: [], ensures: [] }
}
for (const req of contracts.requires ?? []) {
phases[phase]!.requires.push({
formula: req,
source: `plugin:${plugin}` as const,
})
}
for (const ens of contracts.ensures ?? []) {
phases[phase]!.ensures.push({
formula: ens,
source: `plugin:${plugin}` as const,
})
}
}
}
return { route, phases }
}
/**
* Get all registered plugin names.
*/
getPluginNames(): string[] {
return Array.from(this.contracts.keys())
}
/**
* Check if a plugin is registered.
*/
hasPlugin(name: string): boolean {
return this.contracts.has(name)
}
/**
* Get a plugin's spec.
*/
getPluginSpec(name: string): PluginContractSpec | undefined {
return this.contracts.get(name)
}
/**
* Get all available extension names.
*/
getAvailableExtensions(): string[] {
return Array.from(this.availableExtensions)
}
}
// ============================================================================
// Built-in Plugin Contracts
// ============================================================================
export const BUILTIN_PLUGIN_CONTRACTS: Record<string, PluginContractSpec> = {
'@fastify/auth': {
appliesTo: '**',
hooks: {
onRequest: {
requires: ['request_headers(this).authorization != null'],
},
},
},
'@fastify/compress': {
appliesTo: '**',
hooks: {
onSend: {
ensures: ['response_headers(this).content-encoding != null'],
},
},
},
'@fastify/cors': {
appliesTo: '**',
hooks: {
onRequest: {
ensures: ['response_headers(this).access-control-allow-origin != null'],
},
},
},
'@fastify/rate-limit': {
appliesTo: '**',
hooks: {
onRequest: {
ensures: [
'response_headers(this).x-ratelimit-limit != null',
'response_headers(this).x-ratelimit-remaining != null',
],
},
},
},
}
// ============================================================================
// Factory
// ============================================================================
export function createPluginContractRegistry(): PluginContractRegistry {
return new PluginContractRegistry()
}
+219
View File
@@ -0,0 +1,219 @@
/**
* Request Builder Schema-aware request construction with path/body/query/header discrimination
*/
import type { ResourceHierarchy, ModelState } from './stateful.js'
import { SeededRng } from '../infrastructure/seeded-rng.js'
import { CONTENT_TYPE } from '../infrastructure/http-executor.js'
import type { RouteContract } from '../types.js'
export interface RequestStructure {
method: string
url: string
headers: Record<string, string>
query?: Record<string, string>
body?: unknown
contentType?: string
multipart?: {
fields: Record<string, unknown>
files: Record<string, { originalname: string; mimetype: string; size: number; buffer: Buffer } | Array<{ originalname: string; mimetype: string; size: number; buffer: Buffer }>>
}
}
export const parseRouteParams = (path: string): string[] => {
const params: string[] = []
const segments = path.split('/')
for (const segment of segments) {
if (segment.startsWith(':')) {
params.push(segment.slice(1))
} else if (segment.startsWith('{') && segment.endsWith('}')) {
params.push(segment.slice(1, -1))
}
}
return params
}
const extractBodyParams = (
data: Record<string, unknown>,
bodySchema: Record<string, unknown>
): Record<string, unknown> => {
const properties = bodySchema.properties as Record<string, Record<string, unknown>> | undefined
if (!properties) return data
const body: Record<string, unknown> = {}
for (const key of Object.keys(properties)) {
if (key in data) {
const propSchema = properties[key]
if (propSchema?.type === 'object' && propSchema.properties) {
body[key] = extractBodyParams(data, propSchema)
} else {
body[key] = data[key]
}
}
}
return body
}
const extractQueryParams = (
data: Record<string, unknown>,
querySchema: Record<string, unknown>
): Record<string, string> => {
const properties = querySchema.properties as Record<string, Record<string, unknown>> | undefined
if (!properties) return {}
const query: Record<string, string> = {}
for (const key of Object.keys(properties)) {
if (key in data) {
query[key] = String(data[key])
}
}
return query
}
const extractRemainingParams = (
data: Record<string, unknown>,
pathParams: string[],
body?: Record<string, unknown>
): Record<string, string> => {
const usedKeys = new Set(pathParams)
if (body) {
Object.keys(body).forEach((k) => usedKeys.add(k))
}
const query: Record<string, string> = {}
for (const [key, value] of Object.entries(data)) {
if (!usedKeys.has(key)) {
query[key] = String(value)
}
}
return query
}
const PARAM_PATTERN = /:([a-zA-Z_][a-zA-Z0-9_]*)/g
const validateParamName = (paramName: string): void => {
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(paramName)) {
throw new Error(`Invalid path parameter name: ${paramName}. Must match /^[a-zA-Z_][a-zA-Z0-9_]*$/`)
}
}
/**
* Infer resource type from path parameter name.
* Supports patterns like: tenantId tenant, user_id user, id resource
*/
const inferResourceTypeFromParam = (param: string): string | null => {
// Pattern: tenantId, projectId, userId → tenant, project, user
if (param.endsWith('Id')) {
return param.replace(/Id$/, '').toLowerCase()
}
// Pattern: tenant_id, user_id → tenant, user
if (param.endsWith('_id')) {
return param.replace(/_id$/, '').toLowerCase()
}
// Pattern: just 'id' — infer from context or return null
if (param === 'id') {
return null // Can't infer from just 'id'
}
return null
}
const sanitizeParamValue = (value: unknown): string => {
const str = String(value)
// URL-encode the value to prevent injection
return encodeURIComponent(str)
}
const substitutePathParams = (
path: string,
data: Record<string, unknown>,
state: ModelState,
rng?: SeededRng
): string => {
let url = path
const pathParams = parseRouteParams(path)
for (const param of pathParams) {
// Validate param name against whitelist
validateParamName(param)
let value = data[param]
// If param is an ID reference, try to find it in state
if (value === undefined) {
// Try various patterns: tenantId, tenant_id, id, userId, etc.
const resourceType = inferResourceTypeFromParam(param)
if (resourceType) {
const resources = state.resources.get(resourceType)
if (resources && resources.size > 0) {
const ids = Array.from(resources.keys())
value = rng ? rng.pick(ids) : ids[0] // Deterministic fallback: use first ID
}
}
}
if (value !== undefined) {
// Sanitize value before substitution to prevent injection
url = url.replace(`:${param}`, sanitizeParamValue(value))
}
}
return url
}
const buildHeaders = (
route: RouteContract,
scopeHeaders: Record<string, string>,
data: Record<string, unknown>,
_state: ModelState
): Record<string, string> => {
const headers: Record<string, string> = { ...scopeHeaders }
// Content-Type for body requests
if (route.schema?.body) {
headers['content-type'] = CONTENT_TYPE.JSON
}
return headers
}
const getString = (obj: Record<string, unknown>, key: string): string | undefined => {
const val = obj[key]
return typeof val === 'string' ? val : undefined
}
export const buildRequest = (
route: RouteContract,
generatedData: Record<string, unknown>,
scopeHeaders: Record<string, string>,
state: ModelState,
rng?: SeededRng
): RequestStructure => {
const url = substitutePathParams(route.path, generatedData, state, rng)
// Extract body params from schema
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
// Check for multipart
const isMultipart = bodySchema && getString(bodySchema, 'x-content-type') === CONTENT_TYPE.MULTIPART
if (isMultipart) {
const multipartData = (generatedData ?? {}) as Record<string, unknown>
const headers = buildHeaders(route, scopeHeaders, generatedData, state)
headers['content-type'] = CONTENT_TYPE.MULTIPART
const files = multipartData['files']
const fields = multipartData['fields']
return {
method: route.method,
url,
headers,
query: extractRemainingParams(generatedData, parseRouteParams(route.path)),
multipart: {
fields: (fields ?? {}) as Record<string, unknown>,
files: (files ?? {}) as NonNullable<RequestStructure['multipart']>['files'],
},
contentType: CONTENT_TYPE.MULTIPART,
}
}
const body = bodySchema
? extractBodyParams(generatedData, bodySchema)
: undefined
// Extract query params from schema
const querySchema = route.schema?.querystring as Record<string, unknown> | undefined
const query = querySchema
? extractQueryParams(generatedData, querySchema)
: extractRemainingParams(generatedData, parseRouteParams(route.path), body)
// Build headers
const headers = buildHeaders(route, scopeHeaders, generatedData, state)
// Determine content type
const contentType = body ? CONTENT_TYPE.JSON : undefined
return { method: route.method, url, headers, query, body, contentType }
}
export const extractPathParams = (routePath: string, url: string): Record<string, unknown> => {
const routeSegments = routePath.split('/').filter(Boolean)
const urlSegments = url.split('/').filter(Boolean)
const params: Record<string, unknown> = {}
for (let i = 0; i < routeSegments.length; i++) {
const routeSeg = routeSegments[i]
const urlSeg = urlSegments[i]
if (routeSeg?.startsWith(':')) {
params[routeSeg.slice(1)] = urlSeg
} else if (routeSeg?.startsWith('{') && routeSeg.endsWith('}')) {
params[routeSeg.slice(1, -1)] = urlSeg
}
}
return params
}
+208
View File
@@ -0,0 +1,208 @@
/**
* Resource Inference Schema-driven identity extraction and hierarchy detection
* Extracts resource identity from responses with parent-child awareness
* Fully generic: relationships declared in schema annotations, no hardcoded assumptions
*/
import type { ResourceHierarchy } from './stateful.js'
import type { RouteContract } from '../types.js'
export interface ResourceRelationship {
relation: 'parent' | 'owner' | 'reference'
resourceType: string
field: string
}
export interface ResourceIdentity {
resourceType: string
id: string
parentType?: string | undefined
parentId?: string | undefined
scope: string | null
}
const GENERIC_IDENTITY_FIELDS = ['id', 'uuid', '_id', 'resourceId']
const IDENTITY_SUFFIXES = ['Id', '_id', 'UUID', 'Uuid']
interface SchemaResourceAnnotation {
type?: string
identityField?: string
relationships?: ResourceRelationship[]
}
const extractSchemaAnnotation = (schema?: Record<string, unknown>): SchemaResourceAnnotation | undefined => {
const annotation = schema?.['x-apophis-resource']
if (annotation && typeof annotation === 'object') {
return annotation as SchemaResourceAnnotation
}
return undefined
}
const findIdentityField = (
body: Record<string, unknown>,
responseSchema?: Record<string, unknown>,
annotation?: SchemaResourceAnnotation
): { field: string; value: string } | null => {
// 0. Use annotation-specified identity field if available
if (annotation?.identityField) {
const value = body[annotation.identityField]
if (typeof value === 'string' && value.length > 0) {
return { field: annotation.identityField, value }
}
}
// 1. Check schema-required fields first
if (responseSchema?.required && Array.isArray(responseSchema.required)) {
for (const field of responseSchema.required as string[]) {
const value = body[field]
if (typeof value === 'string' && value.length > 0) {
if (GENERIC_IDENTITY_FIELDS.includes(field) || IDENTITY_SUFFIXES.some((s) => field.endsWith(s))) {
return { field, value }
}
}
}
}
// 2. Check generic identity fields
for (const field of GENERIC_IDENTITY_FIELDS) {
const value = body[field]
if (typeof value === 'string' && value.length > 0) {
return { field, value }
}
}
// 3. Check schema properties for fields ending in Id
if (responseSchema?.properties && typeof responseSchema.properties === 'object') {
const properties = responseSchema.properties as Record<string, unknown>
for (const [field, prop] of Object.entries(properties)) {
if (typeof prop === 'object' && prop !== null) {
const propObj = prop as Record<string, unknown>
if (propObj.type === 'string') {
if (IDENTITY_SUFFIXES.some((s) => field.endsWith(s))) {
const value = body[field]
if (typeof value === 'string' && value.length > 0) {
return { field, value }
}
}
}
}
}
}
// 4. Check body for any string field that looks like an identity
for (const [field, value] of Object.entries(body)) {
if (typeof value === 'string' && value.length > 0) {
if (IDENTITY_SUFFIXES.some((s) => field.endsWith(s))) {
return { field, value }
}
}
}
return null
}
export const inferResourceHierarchy = (path: string): {
resourceType: string
parentType?: string
isNested: boolean
} => {
const segments = path.split('/').filter(Boolean)
if (segments.length === 0) {
return { resourceType: 'resource', isNested: false }
}
// Find parameter indices
const paramIndices = segments
.map((s, i) => ({ segment: s, index: i }))
.filter((x) => x.segment.startsWith(':') || (x.segment.startsWith('{') && x.segment.endsWith('}')))
.map((x) => x.index)
if (paramIndices.length > 0) {
const lastParamIdx = paramIndices[paramIndices.length - 1]!
const resourceSegment = segments[lastParamIdx + 1]
if (resourceSegment) {
const resourceType = resourceSegment.replace(/s$/, '')
// Parent is the segment immediately before the last parameter
const parentSegment = segments[lastParamIdx - 1]
if (parentSegment) {
return {
resourceType,
parentType: parentSegment.replace(/s$/, ''),
isNested: true,
}
}
return { resourceType, isNested: true }
}
}
// Top-level: /users, /items, etc.
const lastSegment = segments[segments.length - 1]
return {
resourceType: lastSegment?.replace(/s$/, '') ?? 'resource',
isNested: false,
}
}
const extractParentFromBody = (
body: Record<string, unknown>,
hierarchy: { resourceType: string; parentType?: string }
): { parentId?: string } => {
const result: { parentId?: string } = {}
if (hierarchy.parentType) {
const possibleParentFields = [
`${hierarchy.parentType}Id`,
`${hierarchy.parentType}_id`,
'parentId',
'parent_id',
]
for (const field of possibleParentFields) {
const value = body[field]
if (typeof value === 'string' && value.length > 0) {
result.parentId = value
break
}
}
}
return result
}
export const extractResourceIdentity = (
route: RouteContract,
responseBody: unknown,
responseSchema?: Record<string, unknown>
): ResourceIdentity | null => {
// Only constructors create trackable resources
if (route.category !== 'constructor') return null
if (responseBody === null || typeof responseBody !== 'object') {
return null
}
const body = responseBody as Record<string, unknown>
const annotation = extractSchemaAnnotation(responseSchema)
// Use schema annotation for resource type if available, otherwise infer from path
const resourceType = annotation?.type ?? inferResourceHierarchy(route.path).resourceType
const identity = findIdentityField(body, responseSchema, annotation)
if (!identity) {
return null
}
// Use schema-defined relationships if available
let parentType: string | undefined
let parentId: string | undefined
if (annotation?.relationships && annotation.relationships.length > 0) {
const parentRel = annotation.relationships.find((r) => r.relation === 'parent')
if (parentRel) {
parentType = parentRel.resourceType
const value = body[parentRel.field]
if (typeof value === 'string' && value.length > 0) {
parentId = value
}
}
} else {
// Fall back to path-based inference
const hierarchy = inferResourceHierarchy(route.path)
parentType = hierarchy.parentType
const parentInfo = extractParentFromBody(body, hierarchy)
parentId = parentInfo.parentId
}
return {
resourceType,
id: identity.value,
parentType,
parentId,
scope: parentId ?? null,
}
}
export const createResourceHierarchy = (
identity: ResourceIdentity,
body: Record<string, unknown>
): ResourceHierarchy => ({
id: identity.id,
type: identity.resourceType,
parentId: identity.parentId,
parentType: identity.parentType,
scope: {},
data: body,
createdAt: Date.now(),
})
+528
View File
@@ -0,0 +1,528 @@
/**
* Convert JSON Schema to fast-check arbitraries.
* Pure functions for data transformation.
*/
import type { Arbitrary } from 'fast-check'
import * as fc from 'fast-check'
import { CONTENT_TYPE } from '../infrastructure/http-executor.js'
export type GenerationProfile = 'quick' | 'standard' | 'thorough'
export interface SchemaToArbOptions {
/** 'request' skips readOnly, 'response' skips writeOnly */
readonly context: 'request' | 'response'
/** Generation budget profile: quick favors speed, thorough favors breadth */
readonly generationProfile?: GenerationProfile
}
interface ContextCache {
request?: Arbitrary<unknown>
response?: Arbitrary<unknown>
}
const schemaArbitraryCache = new WeakMap<Record<string, unknown>, ContextCache>()
const schemaFingerprintCache = new WeakMap<Record<string, unknown>, string | null>()
const stableSchemaArbitraryCache = new Map<string, Arbitrary<unknown>>()
const patternRegexCache = new Map<string, RegExp>()
const STABLE_SCHEMA_CACHE_LIMIT = 512
const PATTERN_REGEX_CACHE_LIMIT = 256
function normalizeProfile(profile: GenerationProfile | undefined): GenerationProfile {
return profile ?? 'standard'
}
function defaultStringMaxLength(profile: GenerationProfile): number | undefined {
if (profile === 'quick') return 48
if (profile === 'standard') return 128
return undefined
}
function defaultArrayMaxLength(profile: GenerationProfile): number | undefined {
if (profile === 'quick') return 4
if (profile === 'standard') return 10
return undefined
}
function additionalPropsMaxKeys(profile: GenerationProfile): number {
if (profile === 'quick') return 3
if (profile === 'standard') return 6
return 10
}
function buildFallbackAnyArb(profile: GenerationProfile): Arbitrary<unknown> {
if (profile === 'thorough') {
return fc.anything()
}
const stringMax = profile === 'quick' ? 24 : 64
const arrayMax = profile === 'quick' ? 3 : 6
const dictMax = profile === 'quick' ? 2 : 4
return fc.oneof(
fc.constant(null),
fc.boolean(),
fc.integer(),
fc.double({ noNaN: true }),
fc.string({ maxLength: stringMax }),
fc.array(fc.string({ maxLength: 16 }), { maxLength: arrayMax }),
fc.dictionary(fc.string({ maxLength: 16 }), fc.string({ maxLength: 24 }), { maxKeys: dictMax }),
)
}
const isObject = (v: unknown): v is Record<string, unknown> =>
typeof v === 'object' && v !== null
const getString = (schema: unknown, key: string): string | undefined => {
if (!isObject(schema)) return undefined
const v = schema[key]
return typeof v === 'string' ? v : undefined
}
const getNumber = (schema: unknown, key: string): number | undefined => {
if (!isObject(schema)) return undefined
const v = schema[key]
return typeof v === 'number' ? v : undefined
}
const getBoolean = (schema: unknown, key: string): boolean | undefined => {
if (!isObject(schema)) return undefined
const v = schema[key]
return typeof v === 'boolean' ? v : undefined
}
const getArray = (schema: unknown, key: string): unknown[] | undefined => {
if (!isObject(schema)) return undefined
const v = schema[key]
return Array.isArray(v) ? v : undefined
}
const getObject = (schema: unknown, key: string): Record<string, unknown> | undefined => {
if (!isObject(schema)) return undefined
const v = schema[key]
return isObject(v) ? v : undefined
}
const buildStringArb = (
schema: Record<string, unknown>,
profile: GenerationProfile,
): Arbitrary<string> => {
const minLength = getNumber(schema, 'minLength')
const maxLength = getNumber(schema, 'maxLength')
const pattern = getString(schema, 'pattern')
const format = getString(schema, 'format')
if (format === 'email') {
return fc.emailAddress()
}
if (format === 'uuid') {
return fc.uuid()
}
if (format === 'date-time') {
return fc.date().filter((d) => !Number.isNaN(d.getTime())).map((d) => d.toISOString())
}
if (pattern !== undefined) {
const optimizedPatternArb = tryBuildSimplePatternArb(pattern, minLength, maxLength)
if (optimizedPatternArb) {
return optimizedPatternArb
}
let compiled = patternRegexCache.get(pattern)
if (!compiled) {
compiled = new RegExp(pattern)
if (patternRegexCache.size >= PATTERN_REGEX_CACHE_LIMIT) {
const firstKey = patternRegexCache.keys().next().value
if (firstKey !== undefined) {
patternRegexCache.delete(firstKey)
}
}
patternRegexCache.set(pattern, compiled)
}
return fc.stringMatching(compiled)
}
const constraints: { minLength?: number; maxLength?: number } = {}
if (minLength !== undefined) constraints.minLength = minLength
if (maxLength !== undefined) constraints.maxLength = maxLength
else {
const capped = defaultStringMaxLength(profile)
if (capped !== undefined) constraints.maxLength = capped
}
return fc.string(constraints)
}
function tryBuildSimplePatternArb(
pattern: string,
schemaMinLength: number | undefined,
schemaMaxLength: number | undefined,
): Arbitrary<string> | null {
const match = pattern.match(/^\^\[([^\]]+)\](\+|\*|\{\d+(?:,\d*)?\})\$$/)
if (!match) {
return null
}
const charClass = expandSimpleCharClass(match[1]!)
if (!charClass || charClass.length === 0) {
return null
}
const quantifier = parseQuantifier(match[2]!)
if (!quantifier) {
return null
}
const minLength = Math.max(schemaMinLength ?? 0, quantifier.minLength)
const maxLength =
schemaMaxLength !== undefined && quantifier.maxLength !== undefined
? Math.min(schemaMaxLength, quantifier.maxLength)
: (schemaMaxLength ?? quantifier.maxLength)
if (maxLength !== undefined && minLength > maxLength) {
return null
}
const charArb = fc.constantFrom(...charClass)
return fc.array(charArb, { minLength, maxLength }).map((chars) => chars.join(''))
}
function expandSimpleCharClass(rawClass: string): string[] | null {
const chars: string[] = []
let i = 0
while (i < rawClass.length) {
const start = rawClass[i]!
const dash = rawClass[i + 1]
const end = rawClass[i + 2]
if (dash === '-' && end !== undefined) {
const startCode = start.charCodeAt(0)
const endCode = end.charCodeAt(0)
if (startCode > endCode) {
return null
}
for (let code = startCode; code <= endCode; code++) {
chars.push(String.fromCharCode(code))
}
i += 3
continue
}
chars.push(start)
i += 1
}
return chars
}
function parseQuantifier(
rawQuantifier: string,
): { minLength: number; maxLength: number | undefined } | null {
if (rawQuantifier === '+') {
return { minLength: 1, maxLength: undefined }
}
if (rawQuantifier === '*') {
return { minLength: 0, maxLength: undefined }
}
const exactMatch = rawQuantifier.match(/^\{(\d+)\}$/)
if (exactMatch) {
const exact = Number.parseInt(exactMatch[1]!, 10)
return Number.isFinite(exact) ? { minLength: exact, maxLength: exact } : null
}
const rangeMatch = rawQuantifier.match(/^\{(\d+),(\d*)\}$/)
if (!rangeMatch) {
return null
}
const min = Number.parseInt(rangeMatch[1]!, 10)
const max =
rangeMatch[2] && rangeMatch[2].length > 0
? Number.parseInt(rangeMatch[2], 10)
: undefined
if (!Number.isFinite(min)) {
return null
}
if (max !== undefined && (!Number.isFinite(max) || min > max)) {
return null
}
return { minLength: min, maxLength: max }
}
function stableSerializeSchema(value: unknown, seen: Set<unknown>): string {
if (value === null) return 'null'
if (typeof value === 'string') return JSON.stringify(value)
if (typeof value === 'number' || typeof value === 'boolean') return String(value)
if (Array.isArray(value)) {
return `[${value.map((item) => stableSerializeSchema(item, seen)).join(',')}]`
}
if (typeof value === 'object') {
if (seen.has(value)) {
throw new Error('Cyclic schema object is not supported for stable serialization')
}
seen.add(value)
const record = value as Record<string, unknown>
const keys = Object.keys(record).sort()
const pairs = keys.map((key) => `${JSON.stringify(key)}:${stableSerializeSchema(record[key], seen)}`)
seen.delete(value)
return `{${pairs.join(',')}}`
}
return JSON.stringify(String(value))
}
function getSchemaFingerprint(schema: Record<string, unknown>): string | undefined {
const cached = schemaFingerprintCache.get(schema)
if (cached !== undefined) {
return cached === null ? undefined : cached
}
try {
const fingerprint = stableSerializeSchema(schema, new Set())
schemaFingerprintCache.set(schema, fingerprint)
return fingerprint
} catch {
schemaFingerprintCache.set(schema, null)
return undefined
}
}
function getStableCachedArbitrary(
schema: Record<string, unknown>,
context: SchemaToArbOptions['context'],
profile: GenerationProfile,
): Arbitrary<unknown> | undefined {
const fingerprint = getSchemaFingerprint(schema)
if (!fingerprint) {
return undefined
}
const key = `${context}:${profile}:${fingerprint}`
const cached = stableSchemaArbitraryCache.get(key)
if (!cached) {
return undefined
}
stableSchemaArbitraryCache.delete(key)
stableSchemaArbitraryCache.set(key, cached)
return cached
}
function setStableCachedArbitrary(
schema: Record<string, unknown>,
context: SchemaToArbOptions['context'],
profile: GenerationProfile,
arbitrary: Arbitrary<unknown>,
): void {
const fingerprint = getSchemaFingerprint(schema)
if (!fingerprint) {
return
}
const key = `${context}:${profile}:${fingerprint}`
if (stableSchemaArbitraryCache.has(key)) {
stableSchemaArbitraryCache.delete(key)
}
stableSchemaArbitraryCache.set(key, arbitrary)
if (stableSchemaArbitraryCache.size > STABLE_SCHEMA_CACHE_LIMIT) {
const oldestKey = stableSchemaArbitraryCache.keys().next().value
if (oldestKey !== undefined) {
stableSchemaArbitraryCache.delete(oldestKey)
}
}
}
const buildIntegerArb = (schema: Record<string, unknown>): Arbitrary<number> => {
const minimum = getNumber(schema, 'minimum')
const maximum = getNumber(schema, 'maximum')
const constraints: { min?: number; max?: number } = {}
if (minimum !== undefined) constraints.min = minimum
if (maximum !== undefined) constraints.max = maximum
return fc.integer(constraints)
}
const buildArrayArb = (
schema: Record<string, unknown>,
options: SchemaToArbOptions,
profile: GenerationProfile,
): Arbitrary<unknown[]> => {
const itemsSchema = getObject(schema, 'items')
const itemArb = itemsSchema !== undefined
? convertSchemaInternal(itemsSchema, options, false)
: buildFallbackAnyArb(profile)
const minItems = getNumber(schema, 'minItems')
const maxItems = getNumber(schema, 'maxItems')
const constraints: { minLength?: number; maxLength?: number } = {}
if (minItems !== undefined) constraints.minLength = minItems
if (maxItems !== undefined) constraints.maxLength = maxItems
else {
const capped = defaultArrayMaxLength(profile)
if (capped !== undefined) constraints.maxLength = capped
}
return fc.array(itemArb, constraints)
}
const buildObjectArb = (
schema: Record<string, unknown>,
options: SchemaToArbOptions,
profile: GenerationProfile,
): Arbitrary<Record<string, unknown>> => {
const properties = getObject(schema, 'properties') ?? {}
const required = new Set(getArray(schema, 'required') as string[] ?? [])
const additionalProperties = getBoolean(schema, 'additionalProperties')
const arbs: Record<string, Arbitrary<unknown>> = {}
for (const [key, propSchema] of Object.entries(properties)) {
if (!isObject(propSchema)) continue
const readOnly = getBoolean(propSchema, 'readOnly')
const writeOnly = getBoolean(propSchema, 'writeOnly')
if (options.context === 'request' && readOnly) continue
if (options.context === 'response' && writeOnly) continue
const propArb = convertSchemaInternal(propSchema, options, false)
arbs[key] = required.has(key) ? propArb : fc.option(propArb, { nil: undefined })
}
const baseArb = fc.record(arbs)
if (additionalProperties === true) {
const extraValueArb = buildFallbackAnyArb(profile)
const keyMaxLength = profile === 'quick' ? 16 : 32
return fc.tuple(
baseArb,
fc.dictionary(
fc.string({ maxLength: keyMaxLength }),
extraValueArb,
{ maxKeys: additionalPropsMaxKeys(profile) },
),
).map(([base, extra]) => ({
...base,
...extra,
}))
}
return baseArb
}
const buildMultipartArb = (
schema: Record<string, unknown>,
profile: GenerationProfile,
): Arbitrary<{ fields: Record<string, unknown>; files: Record<string, { originalname: string; mimetype: string; size: number; buffer: Buffer } | { originalname: string; mimetype: string; size: number; buffer: Buffer }[]> }> => {
const fieldsSchema = getObject(schema, 'x-multipart-fields') ?? {}
const filesSchema = getObject(schema, 'x-multipart-files') ?? {}
const fieldArbs: Record<string, Arbitrary<unknown>> = {}
for (const [key, propSchema] of Object.entries(fieldsSchema)) {
if (isObject(propSchema)) {
fieldArbs[key] = convertSchemaInternal(propSchema, { context: 'request', generationProfile: profile }, false)
}
}
const fileArbs: Record<string, Arbitrary<{ originalname: string; mimetype: string; size: number; buffer: Buffer } | { originalname: string; mimetype: string; size: number; buffer: Buffer }[]>> = {}
for (const [key, fileConfig] of Object.entries(filesSchema)) {
if (isObject(fileConfig)) {
const mimeTypes = getArray(fileConfig, 'mimeTypes') as string[] ?? ['application/octet-stream']
const maxSize = getNumber(fileConfig, 'maxSize') ?? 1024 * 1024
const maxCount = getNumber(fileConfig, 'maxCount') ?? 1
const fileArb = fc.record({
originalname: fc.string({ minLength: 1, maxLength: 100 }),
mimetype: fc.constantFrom(...mimeTypes),
size: fc.integer({ min: 1, max: maxSize }),
buffer: fc.uint8Array({ minLength: 1, maxLength: Math.min(maxSize, 1024) }).map(buf => Buffer.from(buf)),
})
if (maxCount > 1) {
fileArbs[key] = fc.array(fileArb, { minLength: 1, maxLength: maxCount })
} else {
fileArbs[key] = fileArb
}
}
}
return fc.record({
fields: fc.record(fieldArbs),
files: fc.record(fileArbs),
})
}
const convertSchemaInternal = (
schema: Record<string, unknown>,
options: SchemaToArbOptions,
useStableCache: boolean,
): Arbitrary<unknown> => {
const profile = normalizeProfile(options.generationProfile)
const cacheKey = options.context
const cachedBySchema = schemaArbitraryCache.get(schema)
const cached = cachedBySchema?.[cacheKey]
if (cached) {
return cached
}
if (useStableCache) {
const stableCached = getStableCachedArbitrary(schema, cacheKey, profile)
if (stableCached) {
const contextCache = cachedBySchema ?? {}
contextCache[cacheKey] = stableCached
schemaArbitraryCache.set(schema, contextCache)
return stableCached
}
}
const type = getString(schema, 'type')
const enumValues = getArray(schema, 'enum')
const nullable = getBoolean(schema, 'nullable')
const contentType = getString(schema, 'x-content-type')
let arb: Arbitrary<unknown>
if (contentType === CONTENT_TYPE.MULTIPART) {
arb = buildMultipartArb(schema, profile)
} else if (enumValues !== undefined && enumValues.length > 0) {
arb = fc.constantFrom(...enumValues)
} else if (type === 'string') {
arb = buildStringArb(schema, profile)
} else if (type === 'integer') {
arb = buildIntegerArb(schema)
} else if (type === 'number') {
arb = fc.float()
} else if (type === 'boolean') {
arb = fc.boolean()
} else if (type === 'array') {
arb = buildArrayArb(schema, options, profile)
} else if (type === 'object') {
arb = buildObjectArb(schema, options, profile)
} else {
arb = buildFallbackAnyArb(profile)
}
if (nullable === true) {
arb = fc.option(arb, { nil: null })
}
const contextCache = cachedBySchema ?? {}
contextCache[cacheKey] = arb
schemaArbitraryCache.set(schema, contextCache)
if (useStableCache) {
setStableCachedArbitrary(schema, cacheKey, profile, arb)
}
return arb
}
export const convertSchema = (
schema: Record<string, unknown>,
options: SchemaToArbOptions = { context: 'request' }
): Arbitrary<unknown> => convertSchemaInternal(schema, options, true)
+196
View File
@@ -0,0 +1,196 @@
/**
* Schema-to-Contract Inference
*
* Derives APOSTL contract formulas from JSON Schema constraints.
* If you declare a field as required with a minimum value in JSON Schema,
* APOPHIS will automatically test that constraint no x-ensures needed.
*
* Inferred contracts are additive: they supplement, never replace, explicit x-ensures.
*
* Supported inference:
* - required fields response_body(this).field != null
* - minimum (number/integer) response_body(this).field >= N
* - maximum (number/integer) response_body(this).field <= N
* - pattern (string) response_body(this).field matches "..."
* - const response_body(this).field == value
* - enum (small) response_body(this).field == "a" || response_body(this).field == "b"
*
* Not inferred (leave to x-ensures for business logic):
* - minLength/maxLength
* - uniqueItems (array-level, hard to express)
* - deep object/array shape (fast-check already generates valid shapes)
*/
interface SchemaInferenceOptions {
/** Base accessor path, e.g. 'response_body(this)' */
accessor: string
/** Maximum enum values to expand into OR chain */
maxEnumValues?: number
}
/** Escape a regex pattern for safe embedding in APOSTL string literals */
const escapePattern = (pattern: string): string =>
pattern.replace(/\\/g, '\\\\').replace(/"/g, '\\"')
function inferFromProperty(
key: string,
propSchema: Record<string, unknown>,
path: string,
opts: SchemaInferenceOptions
): string[] {
const formulas: string[] = []
const fullPath = path ? `${path}.${key}` : key
const accessor = `${opts.accessor}.${fullPath}`
// Nullable fields skip null checks
const isNullable = propSchema.nullable === true
// Required check: field must not be null or undefined
// (We infer this separately via required array, but also check here for const/enum)
// Type-based constraints
const type = propSchema.type
// minimum / maximum
if (type === 'number' || type === 'integer') {
const minimum = propSchema.minimum
if (typeof minimum === 'number') {
formulas.push(`${accessor} >= ${minimum}`)
}
const maximum = propSchema.maximum
if (typeof maximum === 'number') {
formulas.push(`${accessor} <= ${maximum}`)
}
}
// pattern
if (type === 'string' && typeof propSchema.pattern === 'string') {
const safePattern = escapePattern(propSchema.pattern)
formulas.push(`${accessor} matches "${safePattern}"`)
}
// const
if ('const' in propSchema) {
const val = propSchema.const
if (typeof val === 'string') {
formulas.push(`${accessor} == "${val.replace(/"/g, '\\"')}"`)
} else if (typeof val === 'number' || typeof val === 'boolean') {
formulas.push(`${accessor} == ${val}`)
}
}
// enum (small)
if (Array.isArray(propSchema.enum) && propSchema.enum.length > 0) {
const maxEnum = opts.maxEnumValues ?? 5
if (propSchema.enum.length <= maxEnum) {
const conditions = propSchema.enum.map((v: unknown) => {
if (typeof v === 'string') return `${accessor} == "${v.replace(/"/g, '\\"')}"`
if (typeof v === 'number' || typeof v === 'boolean') return `${accessor} == ${v}`
return null
}).filter((c: unknown): c is string => typeof c === 'string')
if (conditions.length === 1) {
formulas.push(conditions[0]!)
} else if (conditions.length > 1) {
formulas.push(conditions.join(' || '))
}
}
}
// Recurse into nested object properties
if (type === 'object' && propSchema.properties) {
const nested = inferFromSchema(
propSchema.properties as Record<string, Record<string, unknown>>,
propSchema.required as string[] | undefined,
fullPath,
opts
)
formulas.push(...nested)
}
// Array items — disabled: generates invalid APOSTL with [] notation
// See: https://github.com/anomalyco/apophis/issues/426
// Arrays of objects are better handled with explicit x-ensures formulas
// if (type === 'array' && propSchema.items) {
// const itemSchema = propSchema.items as Record<string, unknown>
// if (itemSchema.type === 'object' && itemSchema.properties) {
// ...
// }
// }
return formulas
}
function inferFromSchema(
properties: Record<string, Record<string, unknown>> | undefined,
required: string[] | undefined,
path: string,
opts: SchemaInferenceOptions
): string[] {
if (!properties) return []
const formulas: string[] = []
const requiredSet = new Set(required ?? [])
for (const [key, propSchema] of Object.entries(properties)) {
const fullPath = path ? `${path}.${key}` : key
const accessor = `${opts.accessor}.${fullPath}`
// Required check
if (requiredSet.has(key) && propSchema.nullable !== true) {
formulas.push(`${accessor} != null`)
}
// Property-specific constraints
formulas.push(...inferFromProperty(key, propSchema, path, opts))
}
return formulas
}
/**
* Infer contract formulas from a route's response JSON Schema.
*
* @param responseSchema - The response schema for a specific status code (e.g. schema.response[200])
* @returns Array of APOSTL formula strings
*/
export function inferContractsFromResponseSchema(
responseSchema: Record<string, unknown> | undefined
): string[] {
if (!responseSchema) return []
// Handle schema wrappers: { type: 'object', properties: { ... } }
const bodySchema = responseSchema.type === 'object' ? responseSchema : undefined
if (!bodySchema?.properties) return []
const properties = bodySchema.properties as Record<string, Record<string, unknown>>
const required = bodySchema.required as string[] | undefined
return inferFromSchema(properties, required, '', { accessor: 'response_body(this)' })
}
/**
* Infer contract formulas from a route's full schema.
* Looks at success response codes (2xx) and infers from their schemas.
*
* @param schema - The full route schema object
* @returns Array of APOSTL formula strings
*/
export function inferContractsFromRouteSchema(
schema: Record<string, unknown> | undefined
): string[] {
if (!schema) return []
const responseSchema = (schema.response ?? {}) as Record<string, Record<string, unknown>>
const formulas: string[] = []
for (const [statusCode, statusSchema] of Object.entries(responseSchema)) {
const code = parseInt(statusCode, 10)
if (code >= 200 && code < 300) {
const inferred = inferContractsFromResponseSchema(statusSchema)
formulas.push(...inferred)
}
}
return formulas
}
+61
View File
@@ -0,0 +1,61 @@
import { extractResourceIdentity, createResourceHierarchy } from './resource-inference.js'
import type { ResourceHierarchy, ModelState } from './stateful.js'
import type { TrackedResource } from '../infrastructure/cleanup-manager.js'
import type { EvalContext, RouteContract } from '../types.js'
/**
* Updates the model state with a newly created resource from a constructor route.
* Only processes routes with category === 'constructor'.
* Extracts resource identity from the response body, creates a resource hierarchy,
* and stores it in the typed resource map. Tracks parent-child relationships
* when parentId and parentType are present in the identity.
*/
export const updateModelState = (
route: RouteContract,
ctx: EvalContext,
state: ModelState
): ModelState => {
if (route.category !== 'constructor') return state
const body = ctx.response.body as Record<string, unknown> | undefined
if (body === undefined) return state
const identity = extractResourceIdentity(
route,
body,
route.schema?.response as Record<string, unknown> | undefined
)
if (!identity) return state
const hierarchy: ResourceHierarchy = createResourceHierarchy(identity, body)
// Store in typed resource map
const existing = state.resources.get(identity.resourceType) ?? new Map<string, ResourceHierarchy>()
const updated = new Map(existing)
updated.set(identity.id, hierarchy)
const newResources = new Map(state.resources)
newResources.set(identity.resourceType, updated)
return { ...state, resources: newResources }
}
/**
* Creates a TrackedResource from a constructor route's response.
* Only processes routes with category === 'constructor'.
* Returns null if the response body is missing or no resource identity can be extracted.
*/
export const makeTrackedResource = (
route: RouteContract,
ctx: EvalContext
): TrackedResource | null => {
if (route.category !== 'constructor') return null
const body = ctx.response.body as Record<string, unknown> | undefined
if (body === undefined) return null
const identity = extractResourceIdentity(
route,
body,
route.schema?.response as Record<string, unknown> | undefined
)
if (!identity) return null
return {
type: identity.resourceType,
id: identity.id,
url: route.path,
scope: identity.scope,
timestamp: Date.now(),
}
}
+24
View File
@@ -0,0 +1,24 @@
import type { OperationCategory, RouteContract } from '../types.js'
/**
* Stateful Testing Types
* Types for model-based stateful testing with resource hierarchies.
*/
export interface ResourceHierarchy {
readonly id: string
readonly type: string
readonly parentId?: string | undefined
readonly parentType?: string | undefined
readonly scope: Record<string, unknown>
readonly data: unknown
readonly createdAt: number
}
export interface ApiCommand {
readonly route: RouteContract
readonly params: Record<string, unknown>
readonly headers: Record<string, string>
readonly category: OperationCategory
}
export interface ModelState {
readonly resources: ReadonlyMap<string, ReadonlyMap<string, ResourceHierarchy>>
readonly counters: ReadonlyMap<string, number>
}
+406
View File
@@ -0,0 +1,406 @@
/**
* Triple-Boundary Property-Based Testing
*
* Generates variations on THREE boundaries simultaneously:
* 1. Inbound request
* 2. Outbound dependency responses
* 3. Chaos events (inbound + outbound corruption)
*
* When a test fails, fast-check shrinks ALL THREE together, producing
* a minimal counterexample like:
* "Request: {amount: 100}
* Dependency: Stripe returns 200 {id: 'pi_123'}
* Chaos: Outbound corruption truncates response body after 'id' field"
*
* This is the most powerful testing mode: it finds bugs where the handler
* fails to handle specific combinations of input, dependency behavior, AND
* network/serialization failures.
*/
import type { Arbitrary } from 'fast-check'
import * as fc from 'fast-check'
import type { ChaosEvent } from '../quality/chaos-v3.js'
import { convertSchema } from './schema-to-arbitrary.js'
import { SeededRng } from '../infrastructure/seeded-rng.js'
// ============================================================================
// Types
import type { ChaosConfig, ResolvedOutboundContract, RouteContract } from '../types.js'
// ============================================================================
export interface DependencyResponseSample {
readonly contractName: string
readonly statusCode: number
readonly body: unknown
}
// Re-export ChaosEvent from chaos-v3 for use in triple-boundary testing
export type ChaosEventSample = ChaosEvent
export interface TripleBoundaryCommand {
readonly route: RouteContract
readonly request: Record<string, unknown>
readonly dependencyResponses: ReadonlyArray<DependencyResponseSample>
readonly chaosEvents: ReadonlyArray<ChaosEventSample>
}
export interface TripleBoundaryResult {
readonly command: TripleBoundaryCommand
readonly success: boolean
readonly error?: string
/** Which boundary(ies) caused the failure */
readonly failureBoundary?: 'request' | 'dependency' | 'chaos' | 'combination'
/** Human-readable description of the failure */
readonly failureDescription?: string
}
// ============================================================================
// Chaos Arbitrary
// ============================================================================
/**
* Create an arbitrary that generates chaos events for a test scenario.
* The chaos is *conditional* on the route and its dependencies we only
* generate chaos events that are relevant to the current test context.
*/
function createChaosEventArbitrary(
route: RouteContract,
contracts: ResolvedOutboundContract[],
chaosConfig: ChaosConfig
): Arbitrary<ReadonlyArray<ChaosEventSample>> {
const events: Arbitrary<ChaosEventSample>[] = []
// Inbound chaos (always possible)
if (chaosConfig.delay) {
events.push(
fc.record({
type: fc.constant('inbound-delay' as const),
target: fc.constant('inbound' as const),
delayMs: fc.integer({ min: chaosConfig.delay.minMs, max: chaosConfig.delay.maxMs }),
})
)
}
if (chaosConfig.error) {
events.push(
fc.record({
type: fc.constant('inbound-error' as const),
target: fc.constant('inbound' as const),
statusCode: fc.constant(chaosConfig.error.statusCode),
body: fc.constant(chaosConfig.error.body),
})
)
}
if (chaosConfig.dropout) {
events.push(
fc.record({
type: fc.constant('inbound-dropout' as const),
target: fc.constant('inbound' as const),
statusCode: fc.constant(chaosConfig.dropout.statusCode ?? 504),
})
)
}
if (chaosConfig.corruption) {
events.push(
fc.record({
type: fc.constant('inbound-corruption' as const),
target: fc.constant('inbound' as const),
corruptionStrategy: fc.oneof(
fc.constant('truncate' as const),
fc.constant('malformed' as const),
fc.constant('field-corrupt' as const)
),
})
)
}
// Outbound chaos (one per dependency contract)
for (const contract of contracts) {
const contractChaos = chaosConfig.outbound?.find(
(o) => o.target === contract.target || contract.target.includes(o.target)
)
if (contractChaos?.delay) {
events.push(
fc.record({
type: fc.constant('outbound-delay' as const),
target: fc.constant('outbound' as const),
contractName: fc.constant(contract.name),
delayMs: fc.integer({ min: contractChaos.delay.minMs, max: contractChaos.delay.maxMs }),
})
)
}
if (contractChaos?.error) {
events.push(
fc.record({
type: fc.constant('outbound-error' as const),
target: fc.constant('outbound' as const),
contractName: fc.constant(contract.name),
statusCode: fc.constant(contractChaos.error.responses[0]?.statusCode ?? 503),
body: fc.constant(contractChaos.error.responses[0]?.body ?? { error: 'Service unavailable' }),
})
)
}
if (contractChaos?.dropout) {
events.push(
fc.record({
type: fc.constant('outbound-dropout' as const),
target: fc.constant('outbound' as const),
contractName: fc.constant(contract.name),
statusCode: fc.constant(contractChaos.dropout.statusCode ?? 504),
})
)
}
// Outbound corruption: corrupt the dependency response body
if (contractChaos?.corruption || chaosConfig.corruption) {
events.push(
fc.record({
type: fc.constant('outbound-corruption' as const),
target: fc.constant('outbound' as const),
contractName: fc.constant(contract.name),
corruptionStrategy: fc.oneof(
fc.constant('truncate' as const),
fc.constant('malformed' as const),
fc.constant('field-corrupt' as const)
),
corruptionField: fc.string({ minLength: 1, maxLength: 20 }),
})
)
}
}
// Always include "no chaos" as an option (most common case)
events.unshift(fc.constant({ type: 'none' as const, target: 'inbound' as const }))
// Pick 0-N events per test (weighted toward fewer events)
return fc.array(fc.oneof(...events), { minLength: 0, maxLength: Math.min(3, events.length) })
}
// ============================================================================
// Request + Dependency Arbitraries
// ============================================================================
function applyEnsuresToResponse(
contract: ResolvedOutboundContract,
request: Record<string, unknown>,
responseBody: unknown
): unknown {
if (!contract.ensures || contract.ensures.length === 0) return responseBody
if (typeof responseBody !== 'object' || responseBody === null) return responseBody
const result = { ...(responseBody as Record<string, unknown>) }
for (const formula of contract.ensures) {
const fieldMatch = formula.match(
/^response_body\.([a-zA-Z_][\w.]*)\s*==\s*request_body\.([a-zA-Z_][\w.]*)$/
)
if (fieldMatch) {
const responseField = fieldMatch[1]!
const requestField = fieldMatch[2]!
const value = getNestedValue(request, requestField)
if (value !== undefined) {
setNestedValue(result, responseField, value)
}
continue
}
const literalMatch = formula.match(
/^response_body\.([a-zA-Z_][\w.]*)\s*==\s*"([^"]*)"$/
)
if (literalMatch) {
const responseField = literalMatch[1]!
const value = literalMatch[2]!
setNestedValue(result, responseField, value)
}
}
return result
}
function getNestedValue(obj: Record<string, unknown>, path: string): unknown {
const parts = path.split('.')
let current: unknown = obj
for (const part of parts) {
if (typeof current !== 'object' || current === null) return undefined
current = (current as Record<string, unknown>)[part]
}
return current
}
function setNestedValue(obj: Record<string, unknown>, path: string, value: unknown): void {
const parts = path.split('.')
let current: Record<string, unknown> = obj
for (let i = 0; i < parts.length - 1; i++) {
const part = parts[i]!
if (typeof current[part] !== 'object' || current[part] === null) {
current[part] = {}
}
current = current[part] as Record<string, unknown>
}
current[parts[parts.length - 1]!] = value
}
function createConditionalDependencyArbitrary(
contract: ResolvedOutboundContract,
request: Record<string, unknown>,
generationProfile: 'quick' | 'standard' | 'thorough',
): Arbitrary<DependencyResponseSample> {
const statuses = Object.keys(contract.response).map(Number)
if (statuses.length === 0) {
return fc.constant({ contractName: contract.name, statusCode: 200, body: null })
}
return fc.integer({ min: 0, max: statuses.length - 1 }).chain((statusIndex) => {
const statusCode = statuses[statusIndex]!
const schema = contract.response[statusCode]
const bodyArb = convertSchema(schema ?? {}, { context: 'response', generationProfile })
return bodyArb.map((rawBody) => ({
contractName: contract.name,
statusCode,
body: applyEnsuresToResponse(contract, request, rawBody),
}))
})
}
function createRequestArbitrary(
route: RouteContract,
generationProfile: 'quick' | 'standard' | 'thorough',
): Arbitrary<Record<string, unknown>> {
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
const bodyArb = bodySchema !== undefined
? convertSchema(bodySchema, { context: 'request', generationProfile })
: fc.constant({})
const pathParams = route.path.match(/:([a-zA-Z_][a-zA-Z0-9_]*)/g) ?? []
const pathParamArbs: Record<string, fc.Arbitrary<string>> = {}
for (const param of pathParams) {
const paramName = param.slice(1)
pathParamArbs[paramName] = fc.string({ minLength: 1, maxLength: 20 }).filter((s) =>
/^[a-zA-Z0-9_-]+$/.test(s)
)
}
const pathParamArb = Object.keys(pathParamArbs).length > 0
? fc.record(pathParamArbs)
: fc.constant({})
return fc.tuple(bodyArb, pathParamArb).map(([body, pathParams]) => ({
...(typeof body === 'object' && body !== null ? body : {}),
...pathParams,
})) as Arbitrary<Record<string, unknown>>
}
function createConditionalDependenciesArbitrary(
contracts: ResolvedOutboundContract[],
request: Record<string, unknown>,
generationProfile: 'quick' | 'standard' | 'thorough',
): Arbitrary<ReadonlyArray<DependencyResponseSample>> {
if (contracts.length === 0) return fc.constant([])
const arbs = contracts.map((contract) =>
createConditionalDependencyArbitrary(contract, request, generationProfile)
)
return fc.tuple(...arbs)
}
// ============================================================================
// Triple-Boundary Arbitrary
// ============================================================================
/**
* Create an arbitrary that generates request + dependencies + chaos events.
*
* The three dimensions are generated together so fast-check can shrink
* them simultaneously. When a failure is found, the shrinker produces
* a minimal counterexample across all three boundaries.
*
* Example minimal counterexample:
* "Request: {amount: 1}
* Dependency: Stripe returns 200 {id: 'pi_1'}
* Chaos: Outbound corruption truncates response after 'id' field
* Result: Handler crashes trying to access response.amount (undefined)"
*/
export function createTripleBoundaryArbitrary(
route: RouteContract,
contracts: ResolvedOutboundContract[],
chaosConfig: ChaosConfig,
generationProfile: 'quick' | 'standard' | 'thorough' = 'standard',
): Arbitrary<TripleBoundaryCommand> {
const requestArb = createRequestArbitrary(route, generationProfile)
return requestArb.chain((request) => {
const depArb = createConditionalDependenciesArbitrary(contracts, request, generationProfile)
const chaosArb = createChaosEventArbitrary(route, contracts, chaosConfig)
return fc.tuple(depArb, chaosArb).map(([dependencyResponses, chaosEvents]) => ({
route,
request,
dependencyResponses,
chaosEvents,
}))
})
}
// ============================================================================
// Chaos Application (apply generated chaos to execution)
// ============================================================================
/**
* Apply a chaos event to a dependency response.
* Returns the corrupted response.
*/
export function applyChaosToDependencyResponse(
response: DependencyResponseSample,
chaos: ChaosEventSample
): DependencyResponseSample {
if (chaos.type !== 'outbound-corruption') return response
if (chaos.contractName && chaos.contractName !== response.contractName) return response
const body = response.body
if (typeof body !== 'object' || body === null) return response
const corrupted = { ...(body as Record<string, unknown>) }
switch (chaos.corruptionStrategy) {
case 'truncate':
// Remove last field from response
const keys = Object.keys(corrupted)
if (keys.length > 0) {
delete corrupted[keys[keys.length - 1]!]
}
return { ...response, body: corrupted }
case 'malformed':
// Replace body with invalid JSON-like structure
return { ...response, body: '{"broken":' }
case 'field-corrupt':
// Corrupt a specific field
if (chaos.corruptionField && chaos.corruptionField in corrupted) {
corrupted[chaos.corruptionField] = null
}
return { ...response, body: corrupted }
default:
return response
}
}
/**
* Apply all chaos events to a set of dependency responses.
*/
export function applyChaosToAllResponses(
responses: ReadonlyArray<DependencyResponseSample>,
chaosEvents: ReadonlyArray<ChaosEventSample>
): ReadonlyArray<DependencyResponseSample> {
return responses.map((response) => {
const relevantChaos = chaosEvents.filter(
(c) => c.target === 'outbound' && c.contractName === response.contractName
)
return relevantChaos.reduce(
(resp, chaos) => applyChaosToDependencyResponse(resp, chaos),
response
)
})
}
// ============================================================================
// Formatting
// ============================================================================
export function formatTripleBoundaryCounterexample(result: TripleBoundaryResult): string {
const lines: string[] = []
lines.push('Triple-boundary counterexample:')
lines.push('')
lines.push(`Route: ${result.command.route.method} ${result.command.route.path}`)
lines.push('')
lines.push('Request:')
lines.push(JSON.stringify(result.command.request, null, 2))
lines.push('')
if (result.command.dependencyResponses.length > 0) {
lines.push('Dependency responses:')
for (const dep of result.command.dependencyResponses) {
lines.push(` ${dep.contractName}: ${dep.statusCode}`)
lines.push(` ${JSON.stringify(dep.body)}`)
}
lines.push('')
}
if (result.command.chaosEvents.length > 0) {
lines.push('Chaos events:')
for (const chaos of result.command.chaosEvents) {
if (chaos.type === 'none') continue
lines.push(` ${chaos.type}`)
if (chaos.contractName) lines.push(` Target: ${chaos.contractName}`)
if (chaos.delayMs) lines.push(` Delay: ${chaos.delayMs}ms`)
if (chaos.statusCode) lines.push(` Status: ${chaos.statusCode}`)
if (chaos.corruptionStrategy) lines.push(` Corruption: ${chaos.corruptionStrategy}`)
if (chaos.corruptionField) lines.push(` Field: ${chaos.corruptionField}`)
}
lines.push('')
}
if (result.failureBoundary) {
lines.push(`Failure boundary: ${result.failureBoundary}`)
}
if (result.failureDescription) {
lines.push(`Description: ${result.failureDescription}`)
}
if (result.error) {
lines.push(`Error: ${result.error}`)
}
return lines.join('\n')
}

Some files were not shown because too many files have changed in this diff Show More