dsh-web/scripts/coverage-gate.mjs

#!/usr/bin/env node
/**
 * Coverage ratchet for the dsh-web monorepo.
 *
 * The repository runs two vitest majors (3.x in the older plugin packages,
 * 4.x everywhere else), so the v8 coverage provider is declared per major: the
 * 3.x packages carry @vitest/coverage-v8@^3.2.7, and the root devDependency
 * serves every 4.x package through Node resolution. Bumping a package's vitest
 * major therefore means bumping its provider too; a missing or mismatched
 * provider fails this gate loudly instead of silently reporting nothing.
 *
 * This is a Tier-2 gate: it runs in the nightly workflow and on demand, not on
 * every pull request, because instrumenting ~4,400 tests costs minutes and the
 * pull-request lane already runs the full suite. The ratchet is what makes it
 * useful: scripts/coverage-baseline.json records the measured percentage of
 * every package and metric, and a change may not lower any of them. Branch
 * coverage is the metric worth reading, because it is the one that exposes an
 * untested failure path.
 *
 * Usage:
 *   node scripts/coverage-gate.mjs                     # compare against the baseline
 *   node scripts/coverage-gate.mjs --report            # print the current table, exit 0
 *   node scripts/coverage-gate.mjs --write-baseline    # re-record the baseline
 *   node scripts/coverage-gate.mjs [names...]          # scope to packages by name
 */
import { spawnSync } from 'node:child_process'
import { existsSync, readFileSync, readdirSync, rmSync, mkdirSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { dirname, join, relative } from 'node:path'
import { fileURLToPath } from 'node:url'

const SCRIPT_PATH = fileURLToPath(import.meta.url)
const ROOT = join(dirname(SCRIPT_PATH), '..')
const BASELINE_PATH = join(dirname(SCRIPT_PATH), 'coverage-baseline.json')

/** Metrics recorded per package, in report order. */
export const METRICS = ['lines', 'statements', 'functions', 'branches']

/**
 * A metric more than this many percentage points below its recorded value is a
 * regression. Coverage instrumentation is not bit-stable: two identical runs
 * of the fleet differed by 0.04 points on dsh-ssh branches, so a zero-tolerance
 * ratchet would go red on noise. Half a point absorbs that while still catching
 * a deleted test, whose cost is far larger.
 */
const EPSILON = 0.5

/** Plugin packages that run vitest, sorted by directory name. */
export function discoverPackages(readManifest, listDirs) {
  const packages = []
  const base = join(ROOT, 'packages')
  for (const name of listDirs(base)) {
    const dir = join(base, name)
    const manifestPath = join(dir, 'package.json')
    if (!existsSync(manifestPath)) continue
    let manifest
    try {
      manifest = readManifest(manifestPath)
    } catch {
      continue
    }
    if (!/vitest/.test(manifest.scripts?.test ?? '')) continue
    packages.push({ name, dir, rel: relative(ROOT, dir).split('\\').join('/') })
  }
  return packages.sort((a, b) => a.name.localeCompare(b.name))
}

/** Package directories under a base directory. */
export function listPackageDirs(base) {
  try {
    return readdirSync(base, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name)
  } catch {
    return []
  }
}

const readManifest = (path) => JSON.parse(readFileSync(path, 'utf8'))

/** Istanbul summary -> the four percentages this gate records. */
export function metricsFromSummary(summary) {
  const out = {}
  for (const metric of METRICS) out[metric] = round(summary?.total?.[metric]?.pct)
  return out
}

function round(value) {
  return typeof value === 'number' ? Math.round(value * 100) / 100 : 0
}

/** Run one package's suite with coverage and return its metrics. */
export function coverPackage(pkg) {
  const outDir = join(tmpdir(), 'dsh-coverage', pkg.name)
  rmSync(outDir, { recursive: true, force: true })
  mkdirSync(outDir, { recursive: true })
  const bin = join(pkg.dir, 'node_modules', '.bin', 'vitest')
  if (!existsSync(bin)) return { ok: false, error: 'vitest is not installed in ' + pkg.rel }
  const result = spawnSync(bin, [
    'run',
    '--coverage',
    '--coverage.reporter=json-summary',
    '--coverage.reportsDirectory=' + outDir,
  ], { cwd: pkg.dir, encoding: 'utf8', maxBuffer: 128 * 1024 * 1024 })
  const output = (result.stdout ?? '') + (result.stderr ?? '')
  if (result.status !== 0) return { ok: false, error: 'vitest exited ' + result.status, output: tail(output) }
  const summaryPath = join(outDir, 'coverage-summary.json')
  if (!existsSync(summaryPath)) return { ok: false, error: 'no coverage summary was written', output: tail(output) }
  let summary
  try {
    summary = JSON.parse(readFileSync(summaryPath, 'utf8'))
  } catch (error) {
    return { ok: false, error: 'unreadable coverage summary: ' + error.message }
  }
  return { ok: true, metrics: metricsFromSummary(summary), total: summary.total }
}

function tail(text, lines = 25) {
  const all = String(text).trimEnd().split('\n')
  return all.slice(Math.max(0, all.length - lines)).join('\n')
}

/** Compare measured metrics against a recorded baseline. */
export function compare(baseline, measured) {
  const regressions = []
  const improvements = []
  const missing = []
  for (const [name, metrics] of Object.entries(measured)) {
    const base = baseline.packages?.[name]
    if (!base) {
      missing.push(name)
      continue
    }
    for (const metric of METRICS) {
      const before = base[metric] ?? 0
      const after = metrics[metric] ?? 0
      if (after + EPSILON < before) regressions.push({ name, metric, before, after })
      else if (after > before + EPSILON) improvements.push({ name, metric, before, after })
    }
  }
  return { regressions, improvements, missing }
}

/** Weighted repository total across the packages that were measured. */
export function aggregate(measured, totals) {
  const sums = {}
  for (const metric of METRICS) sums[metric] = { covered: 0, total: 0 }
  for (const name of Object.keys(measured)) {
    const total = totals[name]
    if (!total) continue
    for (const metric of METRICS) {
      sums[metric].covered += total[metric]?.covered ?? 0
      sums[metric].total += total[metric]?.total ?? 0
    }
  }
  const out = {}
  for (const metric of METRICS) {
    const entry = sums[metric]
    out[metric] = entry.total === 0 ? 0 : round((entry.covered / entry.total) * 100)
  }
  return out
}

/**
 * One package per line: the baseline is reviewed in diffs, and a nested JSON
 * dump would turn a one-package ratchet into a five-line hunk.
 */
export function serializeBaseline(measured) {
  const packages = {}
  for (const name of Object.keys(measured).sort()) packages[name] = measured[name]
  const lines = ['{', '  "version": 1,', '  "metrics": ' + JSON.stringify(METRICS) + ',', '  "packages": {']
  const names = Object.keys(packages)
  names.forEach((name, index) => {
    const metrics = packages[name]
    const body = METRICS.map((metric) => JSON.stringify(metric) + ': ' + metrics[metric]).join(', ')
    lines.push('    ' + JSON.stringify(name) + ': { ' + body + ' }' + (index === names.length - 1 ? '' : ','))
  })
  lines.push('  }', '}')
  return lines.join('\n') + '\n'
}

/** Rewrite the baseline from a fresh measurement; dropped packages are pruned. */
export function writeBaseline(measured) {
  const packages = {}
  for (const name of Object.keys(measured).sort()) packages[name] = measured[name]
  writeFileSync(BASELINE_PATH, serializeBaseline(measured))
  return { version: 1, metrics: METRICS, packages }
}

function readBaseline() {
  if (!existsSync(BASELINE_PATH)) return { version: 1, metrics: METRICS, packages: {} }
  return JSON.parse(readFileSync(BASELINE_PATH, 'utf8'))
}

function formatTable(measured) {
  const width = Math.max(...Object.keys(measured).map((name) => name.length), 7)
  const header = 'package'.padEnd(width) + METRICS.map((metric) => metric.padStart(11)).join('')
  const rows = [header]
  for (const name of Object.keys(measured)) {
    rows.push(name.padEnd(width) + METRICS.map((metric) => String(measured[name][metric]).padStart(11)).join(''))
  }
  return rows.join('\n')
}

function main() {
  const args = process.argv.slice(2)
  if (args.includes('--help') || args.includes('-h')) {
    console.log('Usage: node scripts/coverage-gate.mjs [--report] [--write-baseline] [names...]\n\nRuns vitest coverage for every plugin package and compares the four metrics\n(lines, statements, functions, branches) against scripts/coverage-baseline.json.')
    return 0
  }
  const filters = args.filter((arg) => !arg.startsWith('-'))
  const all = discoverPackages(readManifest, listPackageDirs)
  const packages = filters.length === 0 ? all : all.filter((pkg) => filters.some((filter) => pkg.name.includes(filter)))
  if (packages.length === 0) {
    console.log('[coverage] no package matched')
    return 1
  }

  const measured = {}
  const totals = {}
  const failures = []
  for (const pkg of packages) {
    process.stdout.write('[coverage] ' + pkg.name + ' ... ')
    const result = coverPackage(pkg)
    if (!result.ok) {
      console.log('FAIL (' + result.error + ')')
      failures.push({ pkg, result })
      continue
    }
    console.log(METRICS.map((metric) => metric + ' ' + result.metrics[metric] + '%').join(' '))
    measured[pkg.name] = result.metrics
    totals[pkg.name] = result.total
  }

  for (const failure of failures) {
    console.log('')
    console.log('[coverage] ' + failure.pkg.rel + ': ' + failure.result.error)
    if (failure.result.output) console.log(failure.result.output)
  }
  if (failures.length > 0) {
    console.log('[coverage] FAIL: ' + failures.length + ' package(s) could not produce coverage')
    return 1
  }

  const repo = aggregate(measured, totals)
  if (args.includes('--write-baseline')) {
    const payload = writeBaseline(measured)
    console.log('')
    console.log('[coverage] baseline written for ' + Object.keys(payload.packages).length + ' package(s)')
    console.log('[coverage] repository totals: ' + METRICS.map((metric) => metric + ' ' + repo[metric] + '%').join(' '))
    return 0
  }

  if (args.includes('--report')) {
    console.log('')
    console.log(formatTable(measured))
    console.log('')
    console.log('[coverage] repository totals: ' + METRICS.map((metric) => metric + ' ' + repo[metric] + '%').join(' '))
    return 0
  }

  const baseline = readBaseline()
  const { regressions, improvements, missing } = compare(baseline, measured)
  console.log('')
  console.log('[coverage] repository totals: ' + METRICS.map((metric) => metric + ' ' + repo[metric] + '%').join(' '))

  if (missing.length > 0) {
    console.log('[coverage] not in the baseline (' + missing.length + '); record with --write-baseline: ' + missing.join(', '))
  }
  if (improvements.length > 0) {
    console.log('[coverage] ratchet-up available (' + improvements.length + '); record it with --write-baseline:')
    for (const item of improvements.slice(0, 10)) {
      console.log('  ' + item.name + ' ' + item.metric + ' ' + item.before + '% -> ' + item.after + '%')
    }
    if (improvements.length > 10) console.log('  ... ' + (improvements.length - 10) + ' more')
  }

  if (regressions.length > 0) {
    console.log('[coverage] FAIL: ' + regressions.length + ' metric(s) regressed:')
    for (const item of regressions) {
      console.log('  ' + item.name + ' ' + item.metric + ' ' + item.before + '% -> ' + item.after + '%')
    }
  }
  if (regressions.length > 0 || missing.length > 0) return 1
  console.log('[coverage] OK: no package regressed')
  return 0
}

const invokedDirectly = process.argv[1] ? join(process.argv[1]) === SCRIPT_PATH : false
if (invokedDirectly) process.exit(main())