From f4df959d040ae1cd59c5af5e199bef146c800349 Mon Sep 17 00:00:00 2001 From: Yao Date: Sun, 27 Sep 2026 00:55:38 +0800 Subject: [PATCH] test(conformance): assert the unserved-path JSON envelope (#563) The status was asserted; the envelope the fallback comment says clients depend on was not, nor the documented refusal to reflect the requested path. The non-reflection case reads the raw body so it cannot fail for a content-type reason. --- CHANGELOG.md | 1 + .../unserved-route-envelope.test.ts | 103 ++++++++++++++++++ tests/fixtures/error-code-coverage.json | 3 +- 3 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 tests/conformance/unserved-route-envelope.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 977ee4b..6dd5a33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ - `GET /v1/sessions/{id}/events` now answers the published collection envelope `{data, prev_page, next_page}` instead of `{data, has_more, first_id, last_id}`. Three of those four field names occur **nowhere** in the published contract — a search of the published documentation finds zero occurrences of `has_more` and `first_id` against 34 of `next_page` — so a client built against the published contract read `next_page` from this listing, found nothing, and could not page at all: the listing cut its page at `limit` and reported the cut in a vocabulary the caller does not speak. `next_page` is now a followable cursor carrying `{order, filter: {session_id}, after_id}`, which is what `contracts/anthropic-cma/pagination.md` had already described this route as doing; `prev_page` is always `null` because the scan is forward-only and an invented predecessor would not resolve. The local `after_id` and `limit` are unchanged. A `page` the route cannot have issued — malformed, issued for a different session, replayed under a different ordering, or naming an event that is not in the log — is a `400` rather than a silent restart from the first event, because a restart is answered as though it were the page after the caller's position and the caller cannot detect it. The SDK's `events()` return type and its accepted options follow the route. The one canonical `/v1` collection still on the local envelope is `/v1/sessions/{id}/artifacts`, filed separately so each conversion is verified on its own. ### Added +- Added a conformance suite asserting that a path this runtime does not serve answers in the JSON error envelope, with a JSON content type, on and off the `/v1` prefix, and that the body does not reflect the requested path. The status was already asserted; the envelope the fallback comment says clients depend on was not. No runtime behaviour changes. - Added the first `tests/conformance/` suite: the CMA beta-header contract is now asserted by executing requests, including that memory-store routes require their own beta, that a canonical resource refuses that beta, that combining both is refused with its own code, and that the two-beta listing exception is admitted under either. No runtime behaviour changes. - Refuses an unimplemented query parameter on the vault and memory-store listings, the two routes that could not adopt admission until they honoured `limit`/`page`. A parameter neither listing read was ignored, so `?include_archived=true&status=archived` answered a page as if the second half of the request had been understood — the failure mode `src/api/routes/query-params.ts` exists to name, and the one thing those two routes did differently from the thirteen that already refuse by name. The admission list is derived from the parameter-name constants the readings use (`COLLECTION_LISTING_QUERY_PARAMS`) rather than repeating the strings in both handlers, because the strings and the readings are one fact and a second copy is the one that gets forgotten; the tests assert the two collections advertise the **identical** list, and that the published `/v1/vaults` mount refuses identically to the local `/v1/credential-vaults` one, since one router serves both. `beta` is still accepted and still not advertised, because its compatibility semantics are not modelled. This also consolidates `memory-stores.ts`'s two separate imports from `query-params.ts` into one. Docs updated in `docs/api.md`, `contracts/anthropic-cma/credentials.md` and `contracts/anthropic-cma/memory-stores.md`. diff --git a/tests/conformance/unserved-route-envelope.test.ts b/tests/conformance/unserved-route-envelope.test.ts new file mode 100644 index 0000000..37a4a47 --- /dev/null +++ b/tests/conformance/unserved-route-envelope.test.ts @@ -0,0 +1,103 @@ +/** + * A path this runtime does not serve answers in the documented JSON envelope. + * + * The second suite under `tests/conformance/`, and provider-free like the first. + * + * `src/api/server.ts:212-220` installs a `notFound` fallback with a stated + * reason: Hono's default fallback is `text/plain` "404 Not Found", which "makes a + * client's error decoder fail while parsing rather than branch on `error.type` — + * the caller sees a transport-shaped failure for what is really a 404". The same + * comment records a second decision that is only observable in a response: the + * message deliberately does **not** echo the requested path, "there is no reason + * to reflect caller input". + * + * The existing coverage asserts `res.status === 404` for a list of retired + * endpoints (`tests/integration/api.test.ts:1572-1574`) and nothing about the + * body, so the envelope, the `error.type`, the JSON content type and the + * non-reflection were all implemented and unasserted. + * + * Not covered here: the fallback's behaviour for non-GET verbs, and whether a + * client decoder is satisfied by the envelope in practice, which is what the + * official SDK slice will settle. + */ + +import { afterEach, describe, expect, it } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createServer } from '@/api/server.js'; +import { Database } from '@/core/db/database.js'; +import { SessionManager } from '@/core/session/session-manager.js'; + +function makeApp() { + const tmpDir = mkdtempSync(join(tmpdir(), 'ma-conformance-404-')); + const db = new Database(join(tmpDir, 'test.db')); + db.runMigrations(); + db.exec(`INSERT INTO environments (id, name, config) VALUES ('env_default', 'local', '{}')`); + const app = createServer({ + db, + sessionManager: new SessionManager(db), + agents: [], + reloadAgents: () => ({ agents: [], errors: [] }), + }); + return { app, db, tmpDir }; +} + +type TestContext = ReturnType; +const contexts: TestContext[] = []; + +function app() { + const ctx = makeApp(); + contexts.push(ctx); + return ctx.app; +} + +async function bodyOf(res: Response) { + return (await res.json()) as { error?: { type?: string; message?: string } }; +} + +describe('unserved route envelope', () => { + afterEach(() => { + for (const ctx of contexts.splice(0)) { + ctx.db.close(); + rmSync(ctx.tmpDir, { recursive: true, force: true }); + } + }); + + it('answers an unserved /v1 path in the JSON error envelope, not Hono text', async () => { + const res = await app().request('/v1/definitely_not_a_route'); + + expect(res.status).toBe(404); + // The content type is the half that fails a client's decoder: a `text/plain` + // body makes the failure look like a transport fault instead of a 404. + expect(res.headers.get('content-type') ?? '').toContain('application/json'); + const body = await bodyOf(res); + expect(body.error?.type).toBe('not_found'); + expect(body.error?.message).toBe('No route matches this request'); + }); + + it('answers an unserved path outside /v1 the same way', async () => { + // The fallback is installed on the app, not on the `/v1/*` mount, so the + // consistency claim covers the whole surface rather than the API prefix. + const res = await app().request('/definitely_not_a_route'); + + expect(res.status).toBe(404); + expect(res.headers.get('content-type') ?? '').toContain('application/json'); + expect((await bodyOf(res)).error?.type).toBe('not_found'); + }); + + it('does not reflect the requested path back in the body', async () => { + // A recorded decision, not an accident: the status and `error.type` carry the + // meaning, so caller input has no reason to appear in the response. + // + // This reads the raw body rather than parsing it, and that is deliberate: the + // claim is about what the body contains, so it must not be able to pass or + // fail because of the content type. Parsing here made the case fail for a + // different reason than the one it names the moment the envelope was broken. + const marker = 'reflect_me_if_you_dare'; + const res = await app().request(`/v1/${marker}`); + + expect(res.status).toBe(404); + expect(await res.text()).not.toContain(marker); + }); +}); diff --git a/tests/fixtures/error-code-coverage.json b/tests/fixtures/error-code-coverage.json index 380549b..c82b659 100644 --- a/tests/fixtures/error-code-coverage.json +++ b/tests/fixtures/error-code-coverage.json @@ -27,7 +27,8 @@ "model_config_invalid": ["tests/unit/model-config-invalid.test.ts"], "model_not_found": ["tests/integration/model-qualified-reference.test.ts", "tests/unit/model-auth-failed.test.ts"], "model_provider_not_configured": ["tests/integration/model-qualified-reference.test.ts", "tests/unit/model-error.test.ts"], - "not_found": ["tests/integration/agent-versions-pagination.test.ts", "tests/integration/deployment-archived-event.test.ts", "tests/integration/deployment-path-aliases.test.ts", "tests/integration/deployment-runs-collection.test.ts", "tests/integration/sdk-error-envelope.test.ts", "tests/integration/session-artifacts-query-admission.test.ts", "tests/integration/unrouted-path-404.test.ts", "tests/integration/vault-path-aliases.test.ts", "tests/unit/orchestrator.test.ts", "tests/unit/session-resource-instances.test.ts"], + "not_found": ["tests/conformance/unserved-route-envelope.test.ts", + "tests/integration/agent-versions-pagination.test.ts", "tests/integration/deployment-archived-event.test.ts", "tests/integration/deployment-path-aliases.test.ts", "tests/integration/deployment-runs-collection.test.ts", "tests/integration/sdk-error-envelope.test.ts", "tests/integration/session-artifacts-query-admission.test.ts", "tests/integration/unrouted-path-404.test.ts", "tests/integration/vault-path-aliases.test.ts", "tests/unit/orchestrator.test.ts", "tests/unit/session-resource-instances.test.ts"], "outcome_evaluator_unavailable": ["tests/integration/outcome-grading.test.ts", "tests/unit/outcome-evaluation.test.ts"], "outcome_grader_unavailable": ["tests/integration/outcome-loop.test.ts"], "outcome_rubric_file_not_found": [],