openclaw/test/scripts/openclaw-performance-source-summary.test.ts

import { spawnSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { buildMarkdown, parseArgs } from "../../scripts/openclaw-performance-source-summary.mts";

const tmpRoots: string[] = [];

const sqliteRun = {
  integrity: { agent: ["ok"], state: "ok" },
  profile: "smoke",
  rows: {
    agentCacheEntries: 1000,
    agentDatabases: 2,
    channelIngressEvents: 1000,
    cronJobs: 100,
    cronTaskRuns: 1000,
    deliveryQueueEntries: 1000,
    pluginStateEntries: 1000,
    stateRows: 4100,
  },
  timingsMs: { checkpoint: 1, seed: 100, total: 150 },
  walBytes: { agentAfter: [0], agentBefore: [1024], stateAfter: 0, stateBefore: 4096 },
};

function mkTmpRoot() {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-source-summary-"));
  tmpRoots.push(root);
  return root;
}

function writeJson(filePath: string, value: unknown) {
  fs.mkdirSync(path.dirname(filePath), { recursive: true });
  fs.writeFileSync(filePath, JSON.stringify(value), "utf8");
}

function runCli(...args: string[]) {
  return spawnSync(
    process.execPath,
    ["--import", "tsx", "scripts/openclaw-performance-source-summary.mts", ...args],
    {
      cwd: path.resolve("."),
      encoding: "utf8",
    },
  );
}

function expectNoNodeStack(stderr: string) {
  expect(stderr).not.toContain("Node.js");
  expect(stderr).not.toContain("\n    at ");
}

function writeSourceFixture(sourceDir: string) {
  writeJson(path.join(sourceDir, "gateway-cpu", "gateway-startup-bench.json"), {
    results: [
      {
        id: "default",
        name: "default",
        summary: {
          readyzMs: { p50: 12, p95: 18 },
          healthzMs: { p50: 5 },
          httpListenLogMs: { p50: 8 },
          gatewayReadyLogMs: { p50: 9 },
          firstOutputMs: { p50: 30 },
          maxRssMb: { p95: 120 },
          cpuCoreRatio: { p95: 0.25 },
          startupTrace: {
            "memory.ready.heapUsedMb": { p50: 30, p95: 32 },
            "phase.load.total": { p50: 70, p95: 80 },
            "phase.load.itemCount": { p50: 40, p95: 50 },
            "phase.load": { p50: 7, p95: 8 },
          },
        },
      },
    ],
  });
  writeJson(path.join(sourceDir, "gateway-cpu", "summary.json"), {
    observations: [],
  });
  writeJson(path.join(sourceDir, "cli-startup.json"), {
    primary: {
      cases: [
        {
          id: "gatewayHealthJson",
          name: "gateway health json",
          summary: {
            durationMs: { p50: 10, p95: 14 },
            maxRssMb: { p95: 90 },
            exitSummary: "code:0x3",
          },
        },
      ],
    },
  });
  writeJson(path.join(sourceDir, "extension-memory.json"), {
    baseline: { maxRssMb: 50, status: "ok" },
    combined: { maxRssMb: 180, status: "ok" },
    counts: { totalEntries: 12 },
    topByDeltaMb: [
      { dir: "extensions/browser", maxRssMb: 80, deltaFromBaselineMb: 12, status: "ok" },
    ],
  });
  writeJson(path.join(sourceDir, "sqlite-perf-smoke.json"), {
    ...sqliteRun,
    queries: [{ p50Ms: 0.1, p95Ms: 0.2, query: "SELECT 1", rows: 1 }],
  });
  writeJson(path.join(sourceDir, "mock-hello", "run-001", "qa-suite-summary.json"), {
    counts: { failed: 0, passed: 1, total: 1 },
    metrics: {
      gatewayCpuCoreRatio: 0.15,
      gatewayProcessRssDeltaBytes: 1024 * 1024,
      gatewayProcessRssEndBytes: 91 * 1024 * 1024,
      gatewayProcessRssStartBytes: 90 * 1024 * 1024,
      wallMs: 250,
    },
    run: { primaryModel: "mock-openai/perf" },
    scenarios: [{ id: "mock-hello", status: "pass" }],
  });
}

it("labels CLI RSS semantics and rejects mixed-metric memory trends", () => {
  const sourceDir = mkTmpRoot();
  const baselineDir = mkTmpRoot();
  writeSourceFixture(sourceDir);
  writeSourceFixture(baselineDir);
  expect(buildMarkdown(sourceDir, baselineDir)).toContain("RSS metric: legacy-last-marker");
  const cliPath = path.join(sourceDir, "cli-startup.json");
  const cli = JSON.parse(fs.readFileSync(cliPath, "utf8"));
  cli.primary.memoryMetric = "cli-runtime-max-rss-v1";
  writeJson(cliPath, cli);
  expect(() => buildMarkdown(sourceDir, baselineDir)).toThrow("Incompatible CLI RSS metrics");
  writeJson(path.join(baselineDir, "cli-startup.json"), cli);
  expect(buildMarkdown(sourceDir, baselineDir)).toContain("RSS metric: cli-runtime-max-rss-v1");
  cli.primary.memoryMetric = "unknown-metric";
  writeJson(cliPath, cli);
  expect(() => buildMarkdown(sourceDir, baselineDir)).toThrow("Unknown CLI RSS metric");

  delete cli.primary.memoryMetric;
  for (const [before, after, error] of [
    [undefined, "native", null],
    ["native", undefined, null],
    ["native", "native", null],
    ["transport", "transport", null],
    [undefined, "transport", "Incompatible CLI execution modes"],
    ["transport", "native", "Incompatible CLI execution modes"],
    ["unknown", "unknown", "Unknown CLI execution mode"],
    [null, "native", "Unknown CLI execution mode"],
    ["native", 1, "Unknown CLI execution mode"],
  ] satisfies Array<[unknown, unknown, string | null]>) {
    writeJson(path.join(baselineDir, "cli-startup.json"), {
      primary: { ...cli.primary, executionMode: before },
    });
    writeJson(cliPath, { primary: { ...cli.primary, executionMode: after } });
    if (error) {
      expect(() => buildMarkdown(sourceDir, baselineDir)).toThrow(error);
    } else {
      expect(buildMarkdown(sourceDir, baselineDir)).toContain("RSS metric: legacy-last-marker");
    }
  }
  writeJson(cliPath, { primary: { ...cli.primary, executionMode: null } });
  expect(() => buildMarkdown(sourceDir, null)).toThrow("Unknown CLI execution mode");
});

function sqliteQuery(overrides: Record<string, unknown> = {}) {
  return {
    database: "state",
    id: "delivery.pending.load",
    p50Ms: 10,
    p95Ms: 12,
    plan: {
      fullTableScans: [],
      indexes: ["idx_delivery_queue_pending"],
      raw: ["SEARCH delivery_queue_entries USING INDEX idx_delivery_queue_pending"],
      tempSorts: [],
    },
    rows: 1000,
    runs: 12,
    sql: "SELECT id FROM delivery_queue_entries WHERE queue_name = ? AND status = ?",
    ...overrides,
  };
}

function writeSqliteV2Fixture(
  sourceDir: string,
  queries: Array<Record<string, unknown>> = [sqliteQuery()],
) {
  writeJson(path.join(sourceDir, "sqlite-perf-smoke.json"), {
    ...sqliteRun,
    queries,
    schemaVersion: 2,
    versions: { agentSchema: 16, sqlite: "3.53.4", stateSchema: 13 },
  });
}

afterEach(() => {
  for (const root of tmpRoots.splice(0)) {
    fs.rmSync(root, { recursive: true, force: true });
  }
});

describe("parseArgs", () => {
  it("parses source summary paths", () => {
    expect(
      parseArgs([
        "--source-dir",
        "reports/current",
        "--baseline-source-dir",
        "reports/baseline",
        "--output",
        "summary.md",
      ]),
    ).toEqual({
      sourceDir: path.resolve("reports/current"),
      baselineSourceDir: path.resolve("reports/baseline"),
      output: path.resolve("summary.md"),
    });
  });

  it("rejects missing path values", () => {
    for (const flag of ["--source-dir", "--baseline-source-dir", "--output"]) {
      expect(() => parseArgs([flag])).toThrow(`${flag} requires a value`);
      expect(() => parseArgs([flag, ""])).toThrow(`${flag} requires a value`);
      expect(() => parseArgs([flag, "-h"])).toThrow(`${flag} requires a value`);
      expect(() => parseArgs([flag, "--source-dir", "reports/current"])).toThrow(
        `${flag} requires a value`,
      );
    }
  });

  it("reports CLI argument errors without a Node stack trace", () => {
    const missingSource = runCli();
    expect(missingSource.status).toBe(1);
    expect(missingSource.stdout).toBe("");
    expect(missingSource.stderr.trim()).toBe("--source-dir is required");
    expectNoNodeStack(missingSource.stderr);

    const unknownArg = runCli("--wat");
    expect(unknownArg.status).toBe(1);
    expect(unknownArg.stdout).toBe("");
    expect(unknownArg.stderr.trim()).toBe("Unknown argument: --wat");
    expectNoNodeStack(unknownArg.stderr);
  });
});

describe("buildMarkdown", () => {
  it("renders source performance fixtures with required artifacts", () => {
    const sourceDir = mkTmpRoot();
    writeSourceFixture(sourceDir);

    const markdown = buildMarkdown(sourceDir, null);
    expect(markdown).toContain("run-001");
    expect(markdown).toContain("gateway health json");
    expect(markdown).toContain("## SQLite State Smoke");
    expect(markdown).toContain("4100");
    expect(markdown).toContain("| default | phase.load | 7.0ms | 8.0ms |");
    expect(markdown).not.toContain("phase.load.total");
    expect(markdown).not.toContain("phase.load.itemCount");
    expect(markdown).not.toContain("memory.ready.heapUsedMb");
    expect(markdown).toContain("Per-plugin rows are isolated cold imports and are not additive.");
    expect(markdown).toContain("| all 12 bundled plugins | 180.0MB | 130.0MB | ok |");
    expect(markdown).toContain("isolated delta from empty process");
  });

  it("compares reordered v2 SQLite scenarios only by shared scenario ID", () => {
    const sourceDir = mkTmpRoot();
    const baselineDir = mkTmpRoot();
    writeSourceFixture(sourceDir);
    writeSourceFixture(baselineDir);
    const agentQuery = sqliteQuery({
      database: "agent",
      id: "agent-cache.plugin-model-catalog.list",
      p50Ms: 1,
      p95Ms: 2,
      plan: {
        fullTableScans: [],
        indexes: ["sqlite_autoindex_cache_entries_1"],
        raw: ["SEARCH cache_entries USING INDEX sqlite_autoindex_cache_entries_1"],
        tempSorts: [],
      },
      rows: 100,
      sql: "SELECT key FROM cache_entries WHERE scope = ?",
    });
    const currentQuery = sqliteQuery({
      p95Ms: 15,
      sql: "SELECT id FROM delivery_queue_entries WHERE status = ?",
    });
    writeSqliteV2Fixture(sourceDir, [currentQuery, agentQuery]);
    writeSqliteV2Fixture(baselineDir, [
      { ...agentQuery, id: "baseline-only", p50Ms: 3, p95Ms: 4 },
      { ...currentQuery, p50Ms: 18, p95Ms: 20 },
    ]);

    const markdown = buildMarkdown(sourceDir, baselineDir);

    expect(markdown).toContain("| current | v2 | smoke | 3.53.4 | 13 | 16 |");
    expect(markdown).toContain(
      "| delivery.pending.load | state | 1000 | 12 | 10.0ms | 15.0ms | 1000 | 12 | 20.0ms | -25.0% |",
    );
    expect(markdown).toContain(
      "| agent-cache.plugin-model-catalog.list | agent | 100 | 12 | 1.0ms | 2.0ms | n/a | n/a | n/a | n/a |",
    );
    expect(markdown).not.toContain("| baseline-only |");
  });

  it("does not compare v2 SQLite scenarios with different workloads", () => {
    for (const baselineQuery of [
      {
        database: "state",
        rows: 999,
        runs: 20,
        sql: "SELECT id FROM delivery_queue_entries WHERE queue_name = ? AND status = ?",
      },
      {
        database: "agent",
        rows: 1000,
        runs: 12,
        sql: "SELECT id FROM delivery_queue_entries WHERE queue_name = ? AND status = ?",
      },
      {
        database: "state",
        rows: 1000,
        runs: 12,
        sql: "SELECT id FROM delivery_queue_entries WHERE status = ?",
      },
    ]) {
      const sourceDir = mkTmpRoot();
      const baselineDir = mkTmpRoot();
      writeSourceFixture(sourceDir);
      writeSourceFixture(baselineDir);
      writeSqliteV2Fixture(sourceDir);
      writeSqliteV2Fixture(baselineDir, [sqliteQuery({ ...baselineQuery, p50Ms: 18, p95Ms: 20 })]);

      expect(buildMarkdown(sourceDir, baselineDir)).toContain(
        `| delivery.pending.load | state | 1000 | 12 | 10.0ms | 12.0ms | ${baselineQuery.rows} | ${baselineQuery.runs} | 20.0ms | n/a (workload differs) |`,
      );
    }
  });

  it("rejects duplicate and empty v2 SQLite scenario IDs", () => {
    for (const ids of [
      ["delivery.pending.load", "delivery.pending.load"],
      ["delivery.pending.load", "   "],
    ]) {
      const sourceDir = mkTmpRoot();
      writeSourceFixture(sourceDir);
      writeSqliteV2Fixture(
        sourceDir,
        ids.map((id) =>
          sqliteQuery({
            id,
            p50Ms: 1,
            p95Ms: 2,
            plan: {
              fullTableScans: ["SCAN delivery_queue_entries"],
              indexes: [],
              raw: ["SCAN delivery_queue_entries"],
              tempSorts: [],
            },
            sql: "SELECT id FROM delivery_queue_entries",
          }),
        ),
      );

      expect(() => buildMarkdown(sourceDir, null)).toThrow(
        "[source-performance] invalid SQLite scenario ID:",
      );
    }
  });

  it("rejects malformed v2 SQLite metrics and normalized plans", () => {
    const plan = {
      fullTableScans: ["SCAN delivery_queue_entries"],
      indexes: [],
      raw: ["SCAN delivery_queue_entries"],
      tempSorts: [],
    };
    const query = sqliteQuery({ plan, sql: "SELECT id FROM delivery_queue_entries" });
    const invalidQueries = [
      { ...query, p95Ms: null },
      { ...query, plan: { ...plan, fullTableScans: [42] } },
      { ...query, plan: { ...plan, fullTableScans: [], indexes: ["idx_fake"] } },
    ];

    for (const invalidQuery of invalidQueries) {
      const sourceDir = mkTmpRoot();
      writeSourceFixture(sourceDir);
      writeSqliteV2Fixture(sourceDir, [invalidQuery]);

      expect(() => buildMarkdown(sourceDir, null)).toThrow(
        /\[source-performance\] invalid SQLite scenario (metrics|plan):/,
      );
    }
  });

  it("rejects control characters in v2 SQLite display fields", () => {
    const sourceDir = mkTmpRoot();
    writeSourceFixture(sourceDir);
    writeSqliteV2Fixture(sourceDir, [
      sqliteQuery({
        id: "delivery.pending\nload",
        sql: "SELECT id FROM delivery_queue_entries",
      }),
    ]);

    expect(() => buildMarkdown(sourceDir, null)).toThrow(
      "[source-performance] invalid SQLite scenario ID:",
    );
  });

  it("renders legacy SQLite artifacts without manufacturing baseline matches", () => {
    const sourceDir = mkTmpRoot();
    const baselineDir = mkTmpRoot();
    writeSourceFixture(sourceDir);
    writeSourceFixture(baselineDir);

    const markdown = buildMarkdown(sourceDir, baselineDir);

    expect(markdown).toContain("| current | legacy | smoke | n/a | n/a | n/a |");
    expect(markdown).toContain(
      "| legacy query 1 | unknown | 1 | n/a | 0.1ms | 0.2ms | n/a | n/a | n/a | n/a |",
    );
  });

  it("rejects a missing source directory", () => {
    expect(() => buildMarkdown(path.join(mkTmpRoot(), "missing"), null)).toThrow(
      "[source-performance] missing required source dir:",
    );
  });

  it("rejects missing source performance artifacts", () => {
    const sourceDir = mkTmpRoot();

    expect(() => buildMarkdown(sourceDir, null)).toThrow(
      "[source-performance] missing required gateway startup artifact:",
    );
  });

  it("rejects malformed mock hello summaries", () => {
    const sourceDir = mkTmpRoot();
    writeSourceFixture(sourceDir);
    writeJson(path.join(sourceDir, "mock-hello", "run-001", "qa-suite-summary.json"), {});

    expect(() => buildMarkdown(sourceDir, null)).toThrow(
      "[source-performance] invalid mock hello summary counts:",
    );
  });

  it("rejects mock hello summaries without matching scenario evidence", () => {
    const sourceDir = mkTmpRoot();
    writeSourceFixture(sourceDir);
    writeJson(path.join(sourceDir, "mock-hello", "run-001", "qa-suite-summary.json"), {
      counts: { failed: 0, passed: 1, total: 1 },
      metrics: {
        gatewayCpuCoreRatio: 0.15,
        gatewayProcessRssDeltaBytes: 1024 * 1024,
        gatewayProcessRssEndBytes: 91 * 1024 * 1024,
        gatewayProcessRssStartBytes: 90 * 1024 * 1024,
        wallMs: 250,
      },
      run: { primaryModel: "mock-openai/perf" },
      scenarios: [{ id: "mock-hello", status: "fail" }],
    });

    expect(() => buildMarkdown(sourceDir, null)).toThrow(
      "[source-performance] invalid mock hello scenario evidence:",
    );
  });

  it("rejects gateway startup artifacts without resource metrics", () => {
    const sourceDir = mkTmpRoot();
    writeSourceFixture(sourceDir);
    writeJson(path.join(sourceDir, "gateway-cpu", "gateway-startup-bench.json"), {
      results: [{ id: "default", summary: { readyzMs: { p50: 12 } } }],
    });

    expect(() => buildMarkdown(sourceDir, null)).toThrow(
      "[source-performance] incomplete gateway startup metrics for default:",
    );
  });

  it("rejects extension memory artifacts without combined-process context", () => {
    const sourceDir = mkTmpRoot();
    writeSourceFixture(sourceDir);
    writeJson(path.join(sourceDir, "extension-memory.json"), {
      topByDeltaMb: [
        { dir: "extensions/browser", maxRssMb: 80, deltaFromBaselineMb: 12, status: "ok" },
      ],
    });

    expect(() => buildMarkdown(sourceDir, null)).toThrow(
      "[source-performance] incomplete extension memory context:",
    );
  });

  it("allows source performance fixtures without older-ref SQLite smoke artifacts", () => {
    const sourceDir = mkTmpRoot();
    writeSourceFixture(sourceDir);
    fs.rmSync(path.join(sourceDir, "sqlite-perf-smoke.json"));

    expect(buildMarkdown(sourceDir, null)).toContain("## SQLite State Smoke");
    expect(buildMarkdown(sourceDir, null)).toContain("No data.");
  });

  it("rejects malformed SQLite perf smoke artifacts", () => {
    const sourceDir = mkTmpRoot();
    writeSourceFixture(sourceDir);
    writeJson(path.join(sourceDir, "sqlite-perf-smoke.json"), {
      integrity: { agent: ["ok"], state: "ok" },
      profile: "smoke",
      rows: { stateRows: 4100 },
      walBytes: { stateAfter: 1 },
    });

    expect(() => buildMarkdown(sourceDir, null)).toThrow(
      "[source-performance] incomplete SQLite perf metrics:",
    );
  });

  it("rejects SQLite perf smoke artifacts with failing agent integrity", () => {
    const sourceDir = mkTmpRoot();
    writeSourceFixture(sourceDir);
    writeJson(path.join(sourceDir, "sqlite-perf-smoke.json"), {
      integrity: { agent: ["ok", "database disk image is malformed"], state: "ok" },
      profile: "smoke",
      queries: [{ p50Ms: 0.1, p95Ms: 0.2, query: "SELECT 1", rows: 1 }],
      rows: { agentCacheEntries: 1000, stateRows: 4100 },
      timingsMs: { total: 150 },
      walBytes: { stateAfter: 0, stateBefore: 4096 },
    });

    expect(() => buildMarkdown(sourceDir, null)).toThrow(
      "[source-performance] SQLite agent integrity check did not pass:",
    );
  });
});