376 lines
9.3 KiB
TypeScript
376 lines
9.3 KiB
TypeScript
|
/**
|
||
|
|
* Tests for stateful-runner.ts
|
||
|
|
*/
|
||
|
|
|
||
|
|
import { test } from 'node:test'
|
||
|
|
import assert from 'node:assert'
|
||
|
|
import Fastify from 'fastify'
|
||
|
|
import { runStatefulTests } from '../test/stateful-runner.js'
|
||
|
|
|
||
|
|
// Helper to create a fastify instance with mock routes for discovery
|
||
|
|
const createMockFastify = (routes: Array<{ method: string; url: string; schema?: Record<string, unknown> }>) => {
|
||
|
|
const fastify = Fastify()
|
||
|
|
// Set mock routes for discovery
|
||
|
|
;(fastify as any).routes = routes
|
||
|
|
return fastify
|
||
|
|
}
|
||
|
|
|
||
|
|
test('stateful runner handles empty routes', async () => {
|
||
|
|
const fastify = createMockFastify([])
|
||
|
|
try {
|
||
|
|
await fastify.ready()
|
||
|
|
|
||
|
|
const result = await runStatefulTests(fastify as any, {
|
||
|
|
|
||
|
|
depth: 'quick',
|
||
|
|
scope: undefined,
|
||
|
|
seed: 42,
|
||
|
|
})
|
||
|
|
|
||
|
|
assert.strictEqual(result.tests.length, 0)
|
||
|
|
assert.strictEqual(result.summary.passed, 0)
|
||
|
|
} finally {
|
||
|
|
await fastify.close()
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
test('stateful runner executes commands', async () => {
|
||
|
|
const mockRoutes = [
|
||
|
|
{
|
||
|
|
method: 'POST',
|
||
|
|
url: '/projects',
|
||
|
|
schema: {
|
||
|
|
body: {
|
||
|
|
type: 'object',
|
||
|
|
properties: { name: { type: 'string' } },
|
||
|
|
},
|
||
|
|
response: {
|
||
|
|
200: {
|
||
|
|
type: 'object',
|
||
|
|
properties: { id: { type: 'string' } },
|
||
|
|
},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
]
|
||
|
|
|
||
|
|
const fastify = createMockFastify(mockRoutes)
|
||
|
|
try {
|
||
|
|
fastify.post('/projects', async (req) => ({ id: 'proj-123', name: (req.body as Record<string, string>).name }))
|
||
|
|
|
||
|
|
await fastify.ready()
|
||
|
|
|
||
|
|
const result = await runStatefulTests(fastify as any, {
|
||
|
|
|
||
|
|
depth: 'quick',
|
||
|
|
scope: undefined,
|
||
|
|
seed: 42,
|
||
|
|
})
|
||
|
|
|
||
|
|
// Should have generated some commands
|
||
|
|
assert.ok(result.tests.length > 0)
|
||
|
|
} finally {
|
||
|
|
await fastify.close()
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
test('stateful runner detects status code violations', async () => {
|
||
|
|
const mockRoutes = [
|
||
|
|
{
|
||
|
|
method: 'POST',
|
||
|
|
url: '/projects',
|
||
|
|
schema: {
|
||
|
|
'x-ensures': ['status:201'],
|
||
|
|
body: {
|
||
|
|
type: 'object',
|
||
|
|
properties: { name: { type: 'string' } },
|
||
|
|
},
|
||
|
|
response: {
|
||
|
|
200: {
|
||
|
|
type: 'object',
|
||
|
|
properties: { id: { type: 'string' } },
|
||
|
|
},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
]
|
||
|
|
|
||
|
|
const fastify = createMockFastify(mockRoutes)
|
||
|
|
try {
|
||
|
|
fastify.post('/projects', async () => ({ id: 'proj-123' }))
|
||
|
|
|
||
|
|
await fastify.ready()
|
||
|
|
|
||
|
|
const result = await runStatefulTests(fastify as any, {
|
||
|
|
|
||
|
|
depth: 'quick',
|
||
|
|
scope: undefined,
|
||
|
|
seed: 42,
|
||
|
|
})
|
||
|
|
|
||
|
|
// Should detect status mismatch
|
||
|
|
const failures = result.tests.filter((t) => !t.ok)
|
||
|
|
assert.ok(failures.length > 0, 'Expected at least one failure due to status mismatch')
|
||
|
|
} finally {
|
||
|
|
await fastify.close()
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
test('stateful runner evaluates APOSTL formulas', async () => {
|
||
|
|
const mockRoutes = [
|
||
|
|
{
|
||
|
|
method: 'POST',
|
||
|
|
url: '/projects',
|
||
|
|
schema: {
|
||
|
|
'x-ensures': ['response_body(this).id != null'],
|
||
|
|
body: {
|
||
|
|
type: 'object',
|
||
|
|
properties: { name: { type: 'string' } },
|
||
|
|
},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
]
|
||
|
|
|
||
|
|
const fastify = createMockFastify(mockRoutes)
|
||
|
|
try {
|
||
|
|
fastify.post('/projects', async () => ({ id: 'proj-123' }))
|
||
|
|
|
||
|
|
await fastify.ready()
|
||
|
|
|
||
|
|
const result = await runStatefulTests(fastify as any, {
|
||
|
|
|
||
|
|
depth: 'quick',
|
||
|
|
scope: undefined,
|
||
|
|
seed: 42,
|
||
|
|
})
|
||
|
|
|
||
|
|
const failures = result.tests.filter((t) =>
|
||
|
|
!t.ok && (
|
||
|
|
(typeof t.diagnostics?.error === 'string' && t.diagnostics.error.includes('Contract violation')) ||
|
||
|
|
t.diagnostics?.formula !== undefined
|
||
|
|
)
|
||
|
|
)
|
||
|
|
// Debug: print failures
|
||
|
|
if (failures.length > 0) {
|
||
|
|
console.log('FAILURES:', JSON.stringify(failures.map(f => ({ name: f.name, error: f.diagnostics?.error, formula: f.diagnostics?.formula })), null, 2))
|
||
|
|
}
|
||
|
|
// Should not have formula violations since id is present
|
||
|
|
assert.strictEqual(failures.length, 0)
|
||
|
|
} finally {
|
||
|
|
await fastify.close()
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
test('stateful runner tracks resource state', async () => {
|
||
|
|
const mockRoutes = [
|
||
|
|
{
|
||
|
|
method: 'POST',
|
||
|
|
url: '/projects',
|
||
|
|
schema: {
|
||
|
|
body: {
|
||
|
|
type: 'object',
|
||
|
|
properties: { name: { type: 'string' } },
|
||
|
|
},
|
||
|
|
response: {
|
||
|
|
200: {
|
||
|
|
type: 'object',
|
||
|
|
properties: { id: { type: 'string' } },
|
||
|
|
},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
]
|
||
|
|
|
||
|
|
const fastify = createMockFastify(mockRoutes)
|
||
|
|
try {
|
||
|
|
fastify.post('/projects', async () => ({ id: 'proj-123' }))
|
||
|
|
|
||
|
|
await fastify.ready()
|
||
|
|
|
||
|
|
const result = await runStatefulTests(fastify as any, {
|
||
|
|
|
||
|
|
depth: 'quick',
|
||
|
|
scope: undefined,
|
||
|
|
seed: 42,
|
||
|
|
})
|
||
|
|
|
||
|
|
// Should have run multiple commands
|
||
|
|
assert.ok(result.tests.length > 0)
|
||
|
|
} finally {
|
||
|
|
await fastify.close()
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
test('stateful runner substitutes path params from resource state', async () => {
|
||
|
|
const mockRoutes = [
|
||
|
|
{
|
||
|
|
method: 'POST',
|
||
|
|
url: '/projects',
|
||
|
|
schema: {
|
||
|
|
'x-category': 'constructor',
|
||
|
|
body: {
|
||
|
|
type: 'object',
|
||
|
|
properties: { name: { type: 'string' } },
|
||
|
|
},
|
||
|
|
response: {
|
||
|
|
201: {
|
||
|
|
type: 'object',
|
||
|
|
properties: { id: { type: 'string' } },
|
||
|
|
},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
{
|
||
|
|
method: 'GET',
|
||
|
|
url: '/projects/:projectId',
|
||
|
|
schema: {
|
||
|
|
'x-category': 'observer',
|
||
|
|
params: {
|
||
|
|
type: 'object',
|
||
|
|
properties: {
|
||
|
|
projectId: { type: 'string' },
|
||
|
|
},
|
||
|
|
required: ['projectId'],
|
||
|
|
},
|
||
|
|
response: {
|
||
|
|
200: {
|
||
|
|
type: 'object',
|
||
|
|
properties: { id: { type: 'string' } },
|
||
|
|
},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
]
|
||
|
|
|
||
|
|
const fastify = createMockFastify(mockRoutes)
|
||
|
|
try {
|
||
|
|
const createdProjects = new Map<string, { id: string }>()
|
||
|
|
|
||
|
|
fastify.post('/projects', async (req) => {
|
||
|
|
const id = `proj-${Date.now()}-${Math.random().toString(36).slice(2)}`
|
||
|
|
const project = { id, name: (req.body as Record<string, string>).name }
|
||
|
|
createdProjects.set(id, project)
|
||
|
|
return project
|
||
|
|
})
|
||
|
|
|
||
|
|
fastify.get('/projects/:projectId', async (req) => {
|
||
|
|
const project = createdProjects.get((req.params as { projectId: string }).projectId)
|
||
|
|
if (!project) {
|
||
|
|
return { error: 'not found' }
|
||
|
|
}
|
||
|
|
return project
|
||
|
|
})
|
||
|
|
|
||
|
|
await fastify.ready()
|
||
|
|
|
||
|
|
const result = await runStatefulTests(fastify as any, {
|
||
|
|
depth: 'quick',
|
||
|
|
scope: undefined,
|
||
|
|
seed: 42,
|
||
|
|
})
|
||
|
|
|
||
|
|
// Should have some GET commands that successfully used created project IDs
|
||
|
|
const getCommands = result.tests.filter((t) => t.name.includes('GET /projects/:projectId'))
|
||
|
|
const successfulGets = getCommands.filter((t) => t.ok)
|
||
|
|
|
||
|
|
// With path substitution, some GETs should succeed by using created project IDs
|
||
|
|
assert.ok(successfulGets.length > 0, 'Expected at least one successful GET with path substitution')
|
||
|
|
} finally {
|
||
|
|
await fastify.close()
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
test('stateful runner supports config-level variants', async () => {
|
||
|
|
const mockRoutes = [
|
||
|
|
{
|
||
|
|
method: 'POST',
|
||
|
|
url: '/items',
|
||
|
|
schema: {
|
||
|
|
'x-category': 'constructor',
|
||
|
|
body: {
|
||
|
|
type: 'object',
|
||
|
|
properties: { name: { type: 'string' } },
|
||
|
|
},
|
||
|
|
response: {
|
||
|
|
201: {
|
||
|
|
type: 'object',
|
||
|
|
properties: { id: { type: 'string' } },
|
||
|
|
},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
]
|
||
|
|
|
||
|
|
const fastify = createMockFastify(mockRoutes)
|
||
|
|
try {
|
||
|
|
fastify.post('/items', async () => ({ id: 'item-123' }))
|
||
|
|
|
||
|
|
await fastify.ready()
|
||
|
|
|
||
|
|
const result = await runStatefulTests(fastify as any, {
|
||
|
|
depth: 'quick',
|
||
|
|
scope: undefined,
|
||
|
|
seed: 42,
|
||
|
|
variants: [
|
||
|
|
{ name: 'json', headers: { accept: 'application/json' } },
|
||
|
|
{ name: 'xml', headers: { accept: 'application/xml' } },
|
||
|
|
],
|
||
|
|
})
|
||
|
|
|
||
|
|
const jsonTests = result.tests.filter((t) => t.name.startsWith('[variant:json]'))
|
||
|
|
const xmlTests = result.tests.filter((t) => t.name.startsWith('[variant:xml]'))
|
||
|
|
|
||
|
|
assert.ok(jsonTests.length > 0, 'expected json variant tests')
|
||
|
|
assert.ok(xmlTests.length > 0, 'expected xml variant tests')
|
||
|
|
} finally {
|
||
|
|
await fastify.close()
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
test('stateful runner supports route-level x-variants', async () => {
|
||
|
|
const mockRoutes = [
|
||
|
|
{
|
||
|
|
method: 'POST',
|
||
|
|
url: '/items',
|
||
|
|
schema: {
|
||
|
|
'x-category': 'constructor',
|
||
|
|
'x-variants': [
|
||
|
|
{ name: 'v1', headers: { 'x-api-version': '1' } },
|
||
|
|
{ name: 'v2', headers: { 'x-api-version': '2' } },
|
||
|
|
],
|
||
|
|
body: {
|
||
|
|
type: 'object',
|
||
|
|
properties: { name: { type: 'string' } },
|
||
|
|
},
|
||
|
|
response: {
|
||
|
|
201: {
|
||
|
|
type: 'object',
|
||
|
|
properties: { id: { type: 'string' } },
|
||
|
|
},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
]
|
||
|
|
|
||
|
|
const fastify = createMockFastify(mockRoutes)
|
||
|
|
try {
|
||
|
|
fastify.post('/items', async () => ({ id: 'item-123' }))
|
||
|
|
|
||
|
|
await fastify.ready()
|
||
|
|
|
||
|
|
const result = await runStatefulTests(fastify as any, {
|
||
|
|
depth: 'quick',
|
||
|
|
scope: undefined,
|
||
|
|
seed: 42,
|
||
|
|
})
|
||
|
|
|
||
|
|
const v1Tests = result.tests.filter((t) => t.name.startsWith('[variant:v1]'))
|
||
|
|
const v2Tests = result.tests.filter((t) => t.name.startsWith('[variant:v2]'))
|
||
|
|
|
||
|
|
assert.ok(v1Tests.length > 0, 'expected v1 variant tests from route-level x-variants')
|
||
|
|
assert.ok(v2Tests.length > 0, 'expected v2 variant tests from route-level x-variants')
|
||
|
|
} finally {
|
||
|
|
await fastify.close()
|
||
|
|
}
|
||
|
|
})
|