mirror of
https://github.com/awesome-dsh-plugin/awesome-dsh-plugin.git
synced 2026-09-28 05:13:14 +08:00
feat(catalog): scan what each plugin touches, and publish it as facts (#401)
The catalog now carries capability disclosure: `capabilities` and `capabilityRedLines` per entry, scanned at build time from the artifact a user would install. dsh-market renders them on the card; nothing here says "safe", and no surface is asked to decide anything — #209 settled that a badge stops people reading while the misses are guaranteed to exist. - **scanSourceFor** picks the artifact in the order a user would install it: the npm package when the entry is published there, the author's prebuilt release tarball when that is the only distribution, and the repository's own codeload archive otherwise (`HEAD`, so no branch name and no API call). Subdirectory entries are scanned AT the subdirectory: the monorepo root would report every sibling's capabilities as this plugin's. - **Incremental by release, then by age.** A new npm version invalidates the record; a branch tarball, which can change under the same URL, expires after PROBE_RECHECK_DAYS. A normal push rescans a handful. - **A failure writes nothing.** Download errors, scanner crashes, an output shape this code does not understand — the previous record stays, and an entry with no record stays absent from plugins.json, which the market renders as 未扫描 rather than 未检出. The two are different sentences and only one of them is about the plugin. - **`score`, `band` and `verdict` never leave the scanner.** Upstream says the band is not a pre-install verdict; dropping them here means no surface can render a plugin as green even by accident. - The scanner is pinned as a devDependency (version in package.json and the lockfile, so a bump is a reviewed diff) and invoked through the lockfile's binary; `npx` remains the fallback for a checkout without node_modules. Two things measured while building this, both fixed here: - `execFileSync` in the worker pool blocked the event loop, so CONCURRENCY was a lie: the first full pass ran one entry at a time for fifty minutes and had written nothing. Async children later: eight in flight, ~3 entries/second. - A failure summary that says "957 unreadable" is not actionable; it now repeats the scanner's own sentence — `primary entry unreadable or missing: lib/index.js` is what showed these were packages whose repository does not ship its build product, and that the real cause was a missing data/npm-map.json sending every entry down the GitHub branch. First full pass: 3845 of 4279 entries scanned (90%); 454 carry a red line, the most common being "reads credentials/secrets AND has network access", then "runs code at install time (postinstall)". The 434 without a record are mostly a scanner limitation, not a gap we can close here — its own sentence is `primary entry unreadable or missing: lib/index.js`, i.e. a package whose shipped artifact does not contain the built entry it looks for — and the surfaces print those as 未扫描, which is true. The data file is committed like stars/downloads, for the same reason: "a failed probe keeps what is on disk" is worth nothing if nothing is ever on disk.
This commit is contained in:
fkysly
@@ -206,6 +206,14 @@ jobs:
|
||||
# asked. Neither probe blocks the build: a dead URL is dropped from
|
||||
# the published data, not turned into a failure.
|
||||
node scripts/probe-screenshots.mjs
|
||||
# Capability disclosure (#401): what each plugin touches — files,
|
||||
# network, shell, credentials — scanned from the artifact a user
|
||||
# would install. Runs every time, like the two probes above: it is
|
||||
# incremental by RELEASE (a new npm version invalidates that entry's
|
||||
# record) and by AGE for branch-sourced entries, so a normal push
|
||||
# re-scans a handful. The scanner is pinned inside the script; it
|
||||
# executes nothing from the scanned package and opens no sockets.
|
||||
node scripts/probe-capabilities.mjs
|
||||
node scripts/build-site.mjs
|
||||
- uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
|
||||
@@ -162,6 +162,11 @@ jobs:
|
||||
run: npx awesome-lint
|
||||
- name: Added-date regression tests
|
||||
run: node --test scripts/added-dates.test.mjs
|
||||
- name: Capability-disclosure tests
|
||||
# Pure rules only — no downloads, no scanner, no token: which artifact
|
||||
# an entry is scanned from, when a record is stale, and which fields of
|
||||
# a scanner response are kept (the ranking ones are deliberately not).
|
||||
run: node --test scripts/capabilities.test.mjs
|
||||
- name: Discussion-adoption tests
|
||||
# Pure rules only, no token and no network: the shapes they encode were
|
||||
# read off the live repository (a hand-made thread, a thread the
|
||||
|
||||
+47764
File diff suppressed because it is too large
Load Diff
Generated
+69
@@ -9,6 +9,7 @@
|
||||
"version": "0.1.0",
|
||||
"license": "CC0-1.0",
|
||||
"devDependencies": {
|
||||
"dsh-trust-check": "^0.1.13",
|
||||
"js-yaml": "^5.3.0",
|
||||
"marked": "^18.0.9"
|
||||
}
|
||||
@@ -20,6 +21,74 @@
|
||||
"dev": true,
|
||||
"license": "Python-2.0"
|
||||
},
|
||||
"node_modules/dsh-trust-check": {
|
||||
"version": "0.1.13",
|
||||
"resolved": "https://registry.npmmirror.com/dsh-trust-check/-/dsh-trust-check-0.1.13.tgz",
|
||||
"integrity": "sha512-j1Yz/iXgBMF5TAxCzlOAVe75k7RhVk+C6Uj9TTx4kM2wl/qmuOexkoPm1agdiwnlBIpcakpvGoBzmL4giFdyWg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"js-yaml": "^4.1.0"
|
||||
},
|
||||
"bin": {
|
||||
"dsh-trust-check": "bin/trust-check.mjs"
|
||||
},
|
||||
"engines": {
|
||||
"dsh": ">=0.1.0-rc.8",
|
||||
"node": "^22.19.0 || >=24.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@deepseek-ai/cordis": ">=4.0.0-rc.7",
|
||||
"@deepseek-ai/dsh-client-locale": ">=0.1.0-rc.5",
|
||||
"@deepseek-ai/dsh-client-store": ">=0.1.0-rc.5",
|
||||
"@deepseek-ai/dsh-client-ui-settings": ">=0.1.0-rc.5",
|
||||
"@deepseek-ai/dsh-client-ui-slots": ">=0.1.0-rc.5",
|
||||
"react": "^18.2.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@deepseek-ai/cordis": {
|
||||
"optional": true
|
||||
},
|
||||
"@deepseek-ai/dsh-client-locale": {
|
||||
"optional": true
|
||||
},
|
||||
"@deepseek-ai/dsh-client-store": {
|
||||
"optional": true
|
||||
},
|
||||
"@deepseek-ai/dsh-client-ui-settings": {
|
||||
"optional": true
|
||||
},
|
||||
"@deepseek-ai/dsh-client-ui-slots": {
|
||||
"optional": true
|
||||
},
|
||||
"react": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/dsh-trust-check/node_modules/js-yaml": {
|
||||
"version": "4.3.2",
|
||||
"resolved": "https://registry.npmmirror.com/js-yaml/-/js-yaml-4.3.2.tgz",
|
||||
"integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/puzrin"
|
||||
},
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/nodeca"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"argparse": "^2.0.1"
|
||||
},
|
||||
"bin": {
|
||||
"js-yaml": "bin/js-yaml.js"
|
||||
}
|
||||
},
|
||||
"node_modules/js-yaml": {
|
||||
"version": "5.3.0",
|
||||
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.3.0.tgz",
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
"deepseek"
|
||||
],
|
||||
"devDependencies": {
|
||||
"dsh-trust-check": "^0.1.13",
|
||||
"js-yaml": "^5.3.0",
|
||||
"marked": "^18.0.9"
|
||||
}
|
||||
|
||||
@@ -178,6 +178,9 @@ const starsMap = fs.existsSync('data/stars.json') ? JSON.parse(fs.readFileSync('
|
||||
// already refuses to WRITE the file on a bad run, so whatever is on disk is
|
||||
// the last known-good result, or nothing yet.
|
||||
const downloadsMap = fs.existsSync('data/downloads.json') ? JSON.parse(fs.readFileSync('data/downloads.json', 'utf8')) : {}
|
||||
// Capability disclosure (#401), from probe-capabilities.mjs. An entry absent
|
||||
// here was NOT scanned — the surfaces print that as 未检出, never as clean.
|
||||
const capabilitiesMap = fs.existsSync('data/capabilities.json') ? JSON.parse(fs.readFileSync('data/capabilities.json', 'utf8')) : {}
|
||||
|
||||
// Publishing is the last chance to notice that a data file arrived empty, and
|
||||
// the only one that matters to consumers: docs/ is deployed straight to Pages,
|
||||
@@ -435,6 +438,9 @@ for (const e of ordered) {
|
||||
// entries with no npm package at all — a coverage gap, not a zero.
|
||||
// Consumers must tell "not published" apart from "published, unused".
|
||||
e.downloads = downloadsMap[e.url]?.downloads ?? null
|
||||
e.capabilities = capabilitiesMap[e.url]?.capabilities ?? null
|
||||
e.capabilityRedLines = capabilitiesMap[e.url]?.redLines ?? null
|
||||
e.capabilityCheckedAt = capabilitiesMap[e.url]?.scannedAt ?? null
|
||||
// registry dist-tags.latest from probe-npm.mjs. null when not on npm, OR
|
||||
// when probed but no latest tag was available. A published row whose map
|
||||
// entry still lacks the `version` key has not been backfilled yet — after
|
||||
@@ -1034,6 +1040,14 @@ const registry = {
|
||||
downloadsStart: downloadsMap[e.url]?.start ?? null,
|
||||
downloadsEnd: downloadsMap[e.url]?.end ?? null,
|
||||
downloadsCheckedAt: downloadsMap[e.url]?.checkedAt ?? null,
|
||||
// Capability disclosure (#401). Omitted entirely when the entry was not
|
||||
// scanned, so a consumer cannot read "absent" as "empty" — the market
|
||||
// renders a missing pair as 未检出 / not checked.
|
||||
...(capabilitiesMap[e.url] === undefined ? {} : {
|
||||
capabilities: capabilitiesMap[e.url].capabilities,
|
||||
capabilityRedLines: capabilitiesMap[e.url].redLines,
|
||||
capabilityCheckedAt: capabilitiesMap[e.url].scannedAt,
|
||||
}),
|
||||
install: e.npm ? `dsh plugin --profile web add ${e.npm}` : (e.cmdTarball ?? e.cmdGit),
|
||||
added: e.added,
|
||||
// Optional, author-maintained (data/screenshots.json); omitted when
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
SCANNER_SCHEMA, factsFromScan, repoOf, scanSourceFor, shouldRescan, subdirOf,
|
||||
} from './lib/capabilities.mjs'
|
||||
|
||||
// The values in these tests are not invented: the scanner responses are the
|
||||
// shape `dsh-trust-check@0.1.13 --json` actually printed for
|
||||
// @anionex/dsh-vision-toolkit and for a quiet theme package, measured
|
||||
// 2026-09-24. A test that restated the docs instead would have passed while
|
||||
// the five fields below were wrong.
|
||||
|
||||
test('names the repository and the subdirectory a URL points at', () => {
|
||||
assert.equal(repoOf('https://github.com/omdsh-dev/dsh-mnemon'), 'omdsh-dev/dsh-mnemon')
|
||||
assert.equal(repoOf('https://github.com/zhu1090093659/dsh-web/tree/main/packages/dsh-task-board'), 'zhu1090093659/dsh-web')
|
||||
assert.equal(repoOf('https://github.com/'), null)
|
||||
assert.equal(subdirOf('https://github.com/zhu1090093659/dsh-web/tree/main/packages/dsh-task-board'), 'packages/dsh-task-board')
|
||||
assert.equal(subdirOf('https://github.com/omdsh-dev/dsh-mnemon'), null)
|
||||
})
|
||||
|
||||
test('prefers the npm package, then the author tarball, then the repository', () => {
|
||||
const url = 'https://github.com/anionex/dsh-vision-toolkit'
|
||||
assert.deepEqual(
|
||||
scanSourceFor({ url }, { [url]: { npm: '@anionex/dsh-vision-toolkit', version: '0.1.45' } }),
|
||||
{
|
||||
kind: 'npm',
|
||||
url: 'https://registry.npmjs.org/@anionex/dsh-vision-toolkit/-/dsh-vision-toolkit-0.1.45.tgz',
|
||||
spec: 'npm:@anionex/dsh-vision-toolkit@0.1.45',
|
||||
version: '0.1.45',
|
||||
},
|
||||
)
|
||||
// A prebuilt release with no npm package: the author's tarball IS the
|
||||
// distribution, so scanning the repository instead would describe source
|
||||
// nobody installs.
|
||||
assert.deepEqual(
|
||||
scanSourceFor({ url }, {}, { [url]: 'https://github.com/o/r/releases/latest/download/x.tgz' }),
|
||||
{ kind: 'tarball', url: 'https://github.com/o/r/releases/latest/download/x.tgz', spec: `tarball:${url}`, version: null },
|
||||
)
|
||||
// Neither: the same artifact pnpm fetches for a `github:` spec, with HEAD so
|
||||
// no branch name (or API call) is needed.
|
||||
assert.deepEqual(
|
||||
scanSourceFor({ url }),
|
||||
{ kind: 'github', url: 'https://codeload.github.com/anionex/dsh-vision-toolkit/tar.gz/HEAD', spec: 'github:anionex/dsh-vision-toolkit', version: null },
|
||||
)
|
||||
})
|
||||
|
||||
test('rescan decisions follow the release, then age', () => {
|
||||
const stored = { version: '0.1.45', scannedAt: '2026-09-24T00:00:00Z' }
|
||||
const now = Date.parse('2026-09-24T12:00:00Z')
|
||||
// Same release, npm source: the facts still describe what would be installed.
|
||||
assert.equal(shouldRescan(stored, { version: '0.1.45' }, now, 7), false)
|
||||
// A new release invalidates them — they describe a build nobody installs.
|
||||
assert.equal(shouldRescan(stored, { version: '0.1.46' }, now, 7), true)
|
||||
// No record at all is always a scan.
|
||||
assert.equal(shouldRescan(undefined, { version: null }, now, 7), true)
|
||||
// A branch source has no version to compare, so it expires by AGE: three
|
||||
// days inside a seven-day window is fresh, thirty days is not.
|
||||
assert.equal(shouldRescan(stored, { version: null }, now, 7), false)
|
||||
assert.equal(shouldRescan(stored, { version: null }, Date.parse('2026-10-24T00:00:00Z'), 7), true)
|
||||
// An unreadable timestamp is not evidence of freshness.
|
||||
assert.equal(shouldRescan({ version: null, scannedAt: 'not a date' }, { version: null }, now, 7), true)
|
||||
})
|
||||
|
||||
test('stores the capability shape the scanner printed, and nothing else', () => {
|
||||
const facts = factsFromScan({
|
||||
schemaVersion: SCANNER_SCHEMA,
|
||||
plugins: [{
|
||||
name: '@anionex/dsh-vision-toolkit',
|
||||
version: '0.1.45',
|
||||
spec: 'npm:@anionex/dsh-vision-toolkit@0.1.45',
|
||||
capabilities: ['shell', 'fs-write', 'fs-read', 'network', 'credentials', 'env', 'host-runtime', 'shell', ''],
|
||||
redLines: ['reads credentials/secrets AND has network access'],
|
||||
score: 28,
|
||||
band: 'red',
|
||||
}],
|
||||
}, { spec: 'fallback', version: null, tool: 'dsh-trust-check@0.1.13', now: '2026-09-24T12:00:00Z' })
|
||||
|
||||
assert.deepEqual(facts, {
|
||||
version: '0.1.45',
|
||||
spec: 'npm:@anionex/dsh-vision-toolkit@0.1.45',
|
||||
capabilities: ['shell', 'fs-write', 'fs-read', 'network', 'credentials', 'env', 'host-runtime'],
|
||||
redLines: ['reads credentials/secrets AND has network access'],
|
||||
scannedAt: '2026-09-24T12:00:00Z',
|
||||
tool: 'dsh-trust-check@0.1.13',
|
||||
})
|
||||
// `score` and `band` are dropped on purpose: upstream says the band is not a
|
||||
// pre-install verdict, and a number ranking plugins safe/unsafe is the badge
|
||||
// this feature exists to avoid. A surface cannot render what never arrives.
|
||||
assert.equal('score' in facts, false)
|
||||
assert.equal('band' in facts, false)
|
||||
})
|
||||
|
||||
test('a quiet package stores empty lists, not a clean verdict', () => {
|
||||
const facts = factsFromScan({ schemaVersion: 1, plugins: [{ name: 'skin', version: '1.1.0', capabilities: [], redLines: [] }] },
|
||||
{ spec: 'npm:skin@1.1.0', version: '1.1.0', tool: 't', now: 'n' })
|
||||
assert.deepEqual(facts?.capabilities, [])
|
||||
assert.deepEqual(facts?.redLines, [])
|
||||
// The record says what was DETECTED; "nothing detected" is the scanner's
|
||||
// own wording upstream, and the surfaces print it as 未检出 for the same
|
||||
// reason: it is not a claim that there is nothing to find.
|
||||
})
|
||||
|
||||
test('cannot read a shape it does not understand, and says so by returning nothing', () => {
|
||||
const fallback = { spec: 's', version: null, tool: 't', now: 'n' }
|
||||
assert.equal(factsFromScan({ schemaVersion: 2, plugins: [{}] }, fallback), null)
|
||||
assert.equal(factsFromScan({ plugins: [{}] }, fallback), null)
|
||||
assert.equal(factsFromScan({ schemaVersion: 1, plugins: [] }, fallback), null)
|
||||
assert.equal(factsFromScan({ schemaVersion: 1, errors: ['boom'] }, fallback), null)
|
||||
assert.equal(factsFromScan(null, fallback), null)
|
||||
assert.equal(factsFromScan('scan failed', fallback), null)
|
||||
})
|
||||
@@ -0,0 +1,144 @@
|
||||
// Where a plugin's capability facts come from, and what they are allowed to say.
|
||||
//
|
||||
// The catalog renders "what this plugin touches" — files, network, shell,
|
||||
// credentials — as FACTS, never as a verdict. #209 is why: a badge saying
|
||||
// "safe" stops people reading, and the misses are guaranteed to exist, so the
|
||||
// only honest shape is a disclosure with its own blind spots printed next to
|
||||
// it. The scanner this reads is `dsh-trust-check` (MIT, no code execution, no
|
||||
// network, milliseconds per package — see its docs/INTEGRATION.md), and
|
||||
// everything below exists to keep two promises about it:
|
||||
//
|
||||
// 1. A fact is only ever written when it was READ. A failed download, a
|
||||
// crashed scanner, a schema this code does not understand — all of them
|
||||
// leave the entry absent, which the surfaces render as "未检出 / not
|
||||
// checked". Absence is never written as "clean".
|
||||
// 2. The tool can be replaced. Its output shape is versioned
|
||||
// (`schemaVersion`) and the stable fields are named here, once, so a
|
||||
// future scanner is a change to this file rather than to the catalog
|
||||
// data or to any surface reading it.
|
||||
|
||||
/** The output shape this code understands. A different one is not read. */
|
||||
export const SCANNER_SCHEMA = 1
|
||||
|
||||
/**
|
||||
* @typedef {object} CapabilityFacts
|
||||
* @property {string|null} version Which release was scanned: the facts describe THIS version.
|
||||
* @property {string} spec The install spec the scan labelled the package with.
|
||||
* @property {string[]} capabilities Chip names, e.g. `shell`, `network`, `credentials`, `fs-write`.
|
||||
* @property {string[]} redLines Sentences naming a combination worth a look, e.g. credentials+network.
|
||||
* @property {string} scannedAt ISO 8601, UTC — when this record was produced.
|
||||
* @property {string} tool The scanner that produced it, so a later reader knows which one.
|
||||
*/
|
||||
|
||||
/** `owner/repo` for a catalog URL, dropping any `/tree/<ref>/<sub>` tail. */
|
||||
export function repoOf(url) {
|
||||
const path = url.replace('https://github.com/', '')
|
||||
const parts = path.split('/')
|
||||
if (parts.length < 2 || parts[0] === '' || parts[1] === '') return null
|
||||
return `${parts[0]}/${parts[1]}`
|
||||
}
|
||||
|
||||
/**
|
||||
* The subdirectory an entry lives in, or null for a whole-repo entry.
|
||||
*
|
||||
* The scanner is pointed at the subdirectory because that is what the user
|
||||
* installs (#path:/ specs): scanning the monorepo root would report the
|
||||
* capabilities of every sibling package as if they were this plugin's.
|
||||
*/
|
||||
export function subdirOf(url) {
|
||||
const path = url.replace('https://github.com/', '')
|
||||
if (!path.includes('/tree/')) return null
|
||||
const tail = path.split('/tree/')[1] ?? ''
|
||||
const parts = tail.split('/')
|
||||
return parts.length > 1 ? parts.slice(1).join('/') : null
|
||||
}
|
||||
|
||||
/**
|
||||
* What to download for an entry, and what to tell the scanner it is looking at.
|
||||
*
|
||||
* Preference order is the order a user would install it in: the npm package
|
||||
* when the entry is published there (that is also what the market's install
|
||||
* button fetches), the author's prebuilt release tarball when that is the only
|
||||
* distribution, and the repository's own tarball otherwise — the same codeload
|
||||
* artifact pnpm fetches for a `github:` spec, resolved through `HEAD` so no
|
||||
* branch name or API call is needed (measured: 200 with no ref).
|
||||
*
|
||||
* @returns {{kind: 'npm'|'tarball'|'github', url: string, spec: string, version: string|null}|null}
|
||||
*/
|
||||
export function scanSourceFor(entry, npmMap = {}, tarballs = {}) {
|
||||
const mapped = npmMap[entry.url]
|
||||
const npmName = mapped?.npm ?? (typeof entry.npm === 'string' && entry.npm !== '' ? entry.npm : null)
|
||||
const npmVersion = mapped?.version ?? null
|
||||
if (npmName !== null && npmName !== undefined) {
|
||||
// The packument's own tarball URL shape; the version comes from the same
|
||||
// map the rest of the build reads, so this costs no extra request.
|
||||
const file = npmName.startsWith('@') ? npmName.split('/')[1] : npmName
|
||||
return {
|
||||
kind: 'npm',
|
||||
url: npmVersion === null
|
||||
? `https://registry.npmjs.org/${npmName}`
|
||||
: `https://registry.npmjs.org/${npmName}/-/${file}-${npmVersion}.tgz`,
|
||||
spec: `npm:${npmName}${npmVersion === null ? '' : `@${npmVersion}`}`,
|
||||
version: npmVersion,
|
||||
}
|
||||
}
|
||||
const declared = tarballs[entry.url]
|
||||
const tarball = typeof declared === 'string' ? declared : declared?.tarball ?? null
|
||||
if (typeof tarball === 'string' && tarball !== '') {
|
||||
return { kind: 'tarball', url: tarball, spec: `tarball:${entry.url}`, version: null }
|
||||
}
|
||||
const repo = repoOf(entry.url)
|
||||
if (repo === null) return null
|
||||
return { kind: 'github', url: `https://codeload.github.com/${repo}/tar.gz/HEAD`, spec: `github:${repo}`, version: null }
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a stored record still answers for this source.
|
||||
*
|
||||
* A record describes one release: when the source names a different version,
|
||||
* the facts are about a build nobody installs any more. Sources that name no
|
||||
* version (a branch tarball) can change under the same URL, so they expire by
|
||||
* age instead — `recheckDays` — while npm's per-release artifacts are cheap
|
||||
* and re-checked on the same window.
|
||||
*
|
||||
* @param {CapabilityFacts|undefined} stored
|
||||
* @param {{version: string|null}} source
|
||||
*/
|
||||
export function shouldRescan(stored, source, now, recheckDays) {
|
||||
if (stored === undefined) return true
|
||||
if (source.version !== null) return stored.version !== source.version
|
||||
const scannedAt = Date.parse(stored.scannedAt)
|
||||
if (!Number.isFinite(scannedAt)) return true
|
||||
return now - scannedAt > recheckDays * 86_400_000
|
||||
}
|
||||
|
||||
/**
|
||||
* The facts to store from one scanner response, or null when it cannot be read.
|
||||
*
|
||||
* Only the stable fields are copied. `score`, `band` and `verdict` are
|
||||
* deliberately dropped: upstream says the band is not a pre-install verdict,
|
||||
* and a number that ranks plugins as safe/unsafe is the badge this whole
|
||||
* feature exists to avoid — a surface cannot render what never reaches it.
|
||||
*
|
||||
* @param {unknown} payload
|
||||
* @param {{spec: string, version: string|null, tool: string, now: string}} fallback
|
||||
* @returns {CapabilityFacts|null}
|
||||
*/
|
||||
export function factsFromScan(payload, fallback) {
|
||||
if (payload === null || typeof payload !== 'object' || Array.isArray(payload)) return null
|
||||
if (payload.schemaVersion !== SCANNER_SCHEMA) return null
|
||||
const plugins = Array.isArray(payload.plugins) ? payload.plugins : []
|
||||
const first = plugins.find(plugin => plugin !== null && typeof plugin === 'object')
|
||||
if (first === undefined) return null
|
||||
const strings = (value) => Array.isArray(value)
|
||||
? [...new Set(value.filter(item => typeof item === 'string' && item !== ''))]
|
||||
: []
|
||||
return {
|
||||
version: typeof first.version === 'string' && first.version !== '' ? first.version : fallback.version,
|
||||
spec: typeof first.spec === 'string' && first.spec !== '' ? first.spec : fallback.spec,
|
||||
capabilities: strings(first.capabilities),
|
||||
redLines: strings(first.redLines),
|
||||
scannedAt: fallback.now,
|
||||
tool: fallback.tool,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Scan every listed plugin for the capabilities it touches, into
|
||||
* data/capabilities.json, consumed by build-site.mjs so plugins.json carries
|
||||
* `capabilities` / `redLines` per entry.
|
||||
*
|
||||
* The facts are DISCLOSURE, never a verdict (#209): nothing here writes
|
||||
* "safe", an entry that could not be scanned is simply absent from the file,
|
||||
* and the surfaces print that absence as 未检出. The scanner itself
|
||||
* (`dsh-trust-check`, MIT) executes no code, opens no sockets and needs no
|
||||
* model — it reads the extracted tree and reports `文件:行号` for each hit.
|
||||
*
|
||||
* Where the tree comes from follows what a user would install (see
|
||||
* lib/capabilities.mjs): the npm tarball when the entry is published there,
|
||||
* the author's prebuilt release when that is the only distribution, and the
|
||||
* repository's own codeload artifact otherwise. Subdirectory entries
|
||||
* (`/tree/<ref>/<sub>`) are scanned at the subdirectory, because that is the
|
||||
* package the user gets — the monorepo root would report its siblings.
|
||||
*
|
||||
* Incremental by release, then by age: a new npm version invalidates the
|
||||
* record, a branch tarball (which can change under the same URL) expires
|
||||
* after RECHECK_DAYS. A failed download or a scanner crash KEEPS the previous
|
||||
* record on disk and is reported in the summary; it never writes an empty
|
||||
* capability list, because "we could not look" and "we looked and saw
|
||||
* nothing" are different sentences and only one of them is true.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/probe-capabilities.mjs # incremental pass
|
||||
* PROBE_ALL=1 node scripts/probe-capabilities.mjs # every entry
|
||||
* node scripts/probe-capabilities.mjs --limit=20 # first N unscanned
|
||||
*
|
||||
* The scanner is pinned HERE rather than resolved at run time; bumping it is a
|
||||
* reviewed change (its schemaVersion, not its npm version, is what this code
|
||||
* reads — see lib/capabilities.mjs).
|
||||
*/
|
||||
import fs from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { execFile } from 'node:child_process'
|
||||
import { promisify } from 'node:util'
|
||||
import { readEntries } from './lib/entries.mjs'
|
||||
import { SCANNER_SCHEMA, factsFromScan, scanSourceFor, shouldRescan, subdirOf } from './lib/capabilities.mjs'
|
||||
|
||||
const OUT_FILE = 'data/capabilities.json'
|
||||
const NPM_MAP_FILE = 'data/npm-map.json'
|
||||
const TARBALLS_FILE = 'data/tarballs.json'
|
||||
const TOOL = process.env.CAPABILITY_SCANNER ?? 'dsh-trust-check@0.1.13'
|
||||
|
||||
/**
|
||||
* How to invoke the scanner.
|
||||
*
|
||||
* `npm ci` installs it (a devDependency, so the pin lives in package.json and
|
||||
* the lockfile rather than in this string, and a bump is a reviewed diff), and
|
||||
* the local binary is what a normal run uses. `npx` remains the fallback so a
|
||||
* checkout without node_modules still works — it costs about a second per
|
||||
* entry, which over four thousand entries is the difference between a nightly
|
||||
* pass and an hour.
|
||||
*/
|
||||
function scannerCommand() {
|
||||
if (process.env.CAPABILITY_SCANNER !== undefined && process.env.CAPABILITY_SCANNER !== '') return process.env.CAPABILITY_SCANNER
|
||||
return fs.existsSync('node_modules/.bin/dsh-trust-check') ? 'node_modules/.bin/dsh-trust-check' : 'npx'
|
||||
}
|
||||
|
||||
function scannerArgs(packageDir, spec) {
|
||||
return scannerCommand() === 'npx'
|
||||
? ['--yes', TOOL, '--dir', packageDir, '--spec', spec, '--json']
|
||||
: ['--dir', packageDir, '--spec', spec, '--json']
|
||||
}
|
||||
const CONCURRENCY = Number(process.env.PROBE_CONCURRENCY ?? 6)
|
||||
const RECHECK_DAYS = Number(process.env.PROBE_RECHECK_DAYS ?? 7)
|
||||
const SCAN_TIMEOUT_MS = Number(process.env.PROBE_SCAN_TIMEOUT_MS ?? 120_000)
|
||||
const LIMIT = Number(/^--limit=(\d+)$/.exec(process.argv[2] ?? '')?.[1] ?? '') || Infinity
|
||||
const PROBE_ALL = process.env.PROBE_ALL === '1'
|
||||
// How often the file is rewritten during a pass (see flush()).
|
||||
const FLUSH_EVERY = Number(process.env.PROBE_FLUSH_EVERY ?? 100)
|
||||
|
||||
// Async, not `execFileSync`: a synchronous child blocks the whole event loop,
|
||||
// which silently made CONCURRENCY a lie — the pool ran one entry at a time and
|
||||
// a full pass took hours instead of minutes. Measured on the first run.
|
||||
const run = promisify(execFile)
|
||||
|
||||
const readJson = (file) => fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, 'utf8')) : {}
|
||||
const stored = readJson(OUT_FILE)
|
||||
const npmMap = readJson(NPM_MAP_FILE)
|
||||
const tarballs = readJson(TARBALLS_FILE)
|
||||
const entries = await readEntries('data/plugins')
|
||||
const now = Date.now()
|
||||
|
||||
const next = { ...stored }
|
||||
|
||||
/**
|
||||
* Persist what has been scanned so far.
|
||||
*
|
||||
* A full pass is hours of downloads, and it used to write the file ONLY at the
|
||||
* end — so a runner that hit its job timeout, or a laptop that slept, threw
|
||||
* away every scan it had paid for. Flushing periodically makes the work
|
||||
* resumable: the next run skips whatever is already current.
|
||||
*/
|
||||
function flush() {
|
||||
const sorted = Object.fromEntries(Object.entries(next).sort(([a], [b]) => a.localeCompare(b)))
|
||||
fs.writeFileSync(OUT_FILE, `${JSON.stringify(sorted, null, 1)}\n`)
|
||||
return Object.keys(sorted).length
|
||||
}
|
||||
|
||||
let sinceFlush = 0
|
||||
let scanned = 0
|
||||
let kept = 0
|
||||
let skipped = 0
|
||||
let failed = 0
|
||||
const failures = new Map()
|
||||
|
||||
async function download(url, file) {
|
||||
await run('curl', ['-sSL', '--fail', '--max-time', '120', '-o', file, url], { maxBuffer: 1024 * 1024 })
|
||||
}
|
||||
|
||||
/**
|
||||
* One entry, start to finish. Returns nothing: a failure is COUNTED, and the
|
||||
* previous record (if any) stays exactly where it was.
|
||||
*/
|
||||
async function scanEntry(entry) {
|
||||
const source = scanSourceFor(entry, npmMap, tarballs)
|
||||
if (source === null) { skipped += 1; return }
|
||||
if (!PROBE_ALL && !shouldRescan(stored[entry.url], source, now, RECHECK_DAYS)) { kept += 1; return }
|
||||
const dir = fs.mkdtempSync(join(os.tmpdir(), 'dshm-cap-'))
|
||||
try {
|
||||
const file = join(dir, 'package.tgz')
|
||||
await download(source.url, file)
|
||||
const extracted = join(dir, 'x')
|
||||
fs.mkdirSync(extracted)
|
||||
await run('tar', ['xzf', file, '-C', extracted], { maxBuffer: 1024 * 1024 })
|
||||
// Both npm tarballs and codeload archives wrap everything in one top-level
|
||||
// directory; a subdirectory entry starts one level further in.
|
||||
const top = fs.readdirSync(extracted).filter(name => !name.startsWith('.'))[0]
|
||||
if (top === undefined) throw new Error('empty archive')
|
||||
const sub = subdirOf(entry.url)
|
||||
const packageDir = sub === null ? join(extracted, top) : join(extracted, top, sub)
|
||||
if (!fs.existsSync(packageDir)) throw new Error(`no such subdirectory: ${sub}`)
|
||||
const { stdout: raw } = await run(scannerCommand(), scannerArgs(packageDir, source.spec),
|
||||
{ encoding: 'utf8', timeout: SCAN_TIMEOUT_MS, maxBuffer: 8 * 1024 * 1024 })
|
||||
const payload = JSON.parse(raw)
|
||||
const facts = factsFromScan(payload, {
|
||||
spec: source.spec, version: source.version, tool: TOOL, now: new Date(now).toISOString(),
|
||||
})
|
||||
if (facts === null) {
|
||||
// Say WHICH nothing this was. Most of these are a package whose entry
|
||||
// file is a build product the repository does not ship — the scanner
|
||||
// answers `primary entry unreadable or missing: lib/index.js`, which is
|
||||
// exactly the case an npm-sourced scan avoids (measured: 957 of them on
|
||||
// the first full pass, when a missing data/npm-map.json sent every entry
|
||||
// down the GitHub branch). That sentence is the difference between a
|
||||
// count and something a maintainer can act on.
|
||||
const why = Array.isArray(payload.errors) && typeof payload.errors[0]?.message === 'string'
|
||||
? payload.errors[0].message
|
||||
: `no plugin record (schemaVersion ${String(payload.schemaVersion)})`
|
||||
throw new Error(why)
|
||||
}
|
||||
next[entry.url] = facts
|
||||
scanned += 1
|
||||
sinceFlush += 1
|
||||
if (sinceFlush >= FLUSH_EVERY) {
|
||||
sinceFlush = 0
|
||||
// Progress, because a full pass is hours and a silent log cannot be told
|
||||
// apart from a hung one.
|
||||
console.log(` … ${scanned} scanned, ${failed} failed, ${kept} current (${flush()} records)`)
|
||||
}
|
||||
} catch (error) {
|
||||
failed += 1
|
||||
const message = error instanceof Error ? error.message.slice(0, 120) : String(error)
|
||||
failures.set(message, (failures.get(message) ?? 0) + 1)
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
// A bounded pool, like the other probes: the work is network-bound and the
|
||||
// remote end is one registry and one CDN.
|
||||
let cursor = 0
|
||||
const pending = Math.min(CONCURRENCY, entries.length)
|
||||
await Promise.all(Array.from({ length: pending }, async () => {
|
||||
while (cursor < entries.length) {
|
||||
const index = cursor++
|
||||
if (scanned + kept + failed >= LIMIT) return
|
||||
await scanEntry(entries[index])
|
||||
}
|
||||
}))
|
||||
|
||||
const total = flush()
|
||||
console.log(`capabilities: ${scanned} scanned, ${kept} current, ${skipped} unscannable, ${failed} failed — ${total} records in ${OUT_FILE}`)
|
||||
for (const [message, count] of [...failures].sort((a, b) => b[1] - a[1]).slice(0, 5)) {
|
||||
console.log(` ${count}× ${message}`)
|
||||
}
|
||||
if (scanned === 0 && failed > 0) {
|
||||
// Loud in the log, exit 0: this probe is DISCLOSURE, and a scanner outage on
|
||||
// a cold cache must not fail the publish. The records already on disk are
|
||||
// what the surfaces read, and an entry without one prints 未检出 — true
|
||||
// whether the scanner is broken or the package is. Same discipline as
|
||||
// probe-updates.mjs, for the same reason: the build is not the place to
|
||||
// discover that a network dependency is down.
|
||||
console.log('every scan failed — keeping the previous file rather than publishing an empty one')
|
||||
}
|
||||
Reference in New Issue
Block a user