fix: 收紧 version 背填条件并去掉按日刷新

- 仅当缺少 version 键时背填,显式 null 不再触发 push 死循环
- 背填只跑 probe-npm,不连带 downloads/stars 等
- 版本新鲜度交给 nightly PROBE_ALL,去掉按日 registry refresh

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Liu Wenjie
2026-09-05 01:59:56 +08:00
co-authored by Cursor
parent 0edee31710
commit 9a5bb725ec
3 changed files with 44 additions and 45 deletions
+12 -8
View File
@@ -138,21 +138,23 @@ jobs:
# every entry with npm and downloads blank. Probing is slow; that is
# cheaper than a wrong plugins.json.
#
# Also re-probe on push when the restored npm-map still has published
# rows without a string `version` (schema backfill after that field
# shipped). Skipping would publish plugins.json with npm set and
# version null for every package until the next nightly.
# Schema backfill: if the restored npm-map still has published rows
# that predate the `version` key, run probe-npm only (not the whole
# suite). Check for a missing key — not `typeof !== "string"` — so
# an explicit `version: null` does not re-trigger on every push.
NEED_PROBES=0
NEED_NPM_VERSION_BACKFILL=0
if [ "${{ github.event_name }}" != "push" ] || [ ! -s data/npm-map.json ]; then
NEED_PROBES=1
elif node -e '
const map = JSON.parse(require("fs").readFileSync("data/npm-map.json", "utf8"));
const published = Object.values(map).filter((e) => e && e.npm);
const missing = published.some((e) => typeof e.version !== "string");
const missing = Object.values(map).some(
(e) => e && e.npm && !Object.prototype.hasOwnProperty.call(e, "version"),
);
process.exit(missing ? 0 : 1);
'; then
echo "cached npm-map has published entries without version — probing to backfill"
NEED_PROBES=1
echo "cached npm-map has published entries without a version key — backfilling via probe-npm"
NEED_NPM_VERSION_BACKFILL=1
fi
if [ "$NEED_PROBES" = 1 ]; then
node scripts/probe-npm.mjs
@@ -174,6 +176,8 @@ jobs:
# downloads), and absence of notes is a normal state downstream,
# so this probe never blocks the build.
node scripts/probe-updates.mjs
elif [ "$NEED_NPM_VERSION_BACKFILL" = 1 ]; then
node scripts/probe-npm.mjs
else
echo "push build with cached probe data — skipping probes"
fi
+7 -6
View File
@@ -356,9 +356,11 @@ for (const e of ordered) {
// Consumers must tell "not published" apart from "published, unused".
e.downloads = downloadsMap[e.url]?.downloads ?? null
// registry dist-tags.latest from probe-npm.mjs. null when not on npm, OR
// when the package is on npm but the map row has not been backfilled /
// refreshed yet (consumers: use `npm != null && version == null` for that
// gap — do not treat version null alone as "github-only").
// when probed but no latest tag was available. A published row whose map
// entry still lacks the `version` key has not been backfilled yet — after
// backfill the key is always present (string or null). Consumers: prefer
// `npm` for "on the registry"; treat missing/null version as "unknown",
// not as "github-only".
// Surfaced for dsh-market's discover list (dsh-market#348).
e.version = e.npm ? (npmMap[e.url]?.version ?? null) : null
e.slug = e.sub ? `${e.repo}--${e.sub.replaceAll('/', '-')}` : e.repo
@@ -940,9 +942,8 @@ const registry = {
// by parsing the command string is not a contract worth offering, so the
// field is published directly. Omitted when absent, like `screenshots`.
tarball: e.tarball ?? undefined,
// Current npm `latest` when known. null means either github-only
// (`npm` is also null) or published but not yet version-backfilled
// (`npm` set, `version` null) — not interchangeable with `downloads`.
// Current npm `latest` when known. null = github-only (`npm` null) or
// probed with no latest tag. Not the same signal as `downloads`.
version: e.version,
stars: e.stars,
downloads: e.downloads,
+25 -31
View File
@@ -13,18 +13,18 @@
*
* Results are cached in data/npm-map.json:
* { "<github url>": { npm, version, checkedAt } }
* `version` is the registry `dist-tags.latest` when `npm` is set.
* dsh-market's "discover" list reads it from plugins.json (dsh-market#348)
* so clients do not page the registry. The full probe already fetched the
* packument; recording `latest` there adds no extra request.
* `version` is the registry `dist-tags.latest` when known, else null.
* The key is always written once a published (or not) verdict is recorded so
* consumers can tell "not backfilled yet" (key absent) from "probed, no
* latest tag" (`version: null`). dsh-market reads it from plugins.json
* (dsh-market#348). Recording `latest` on the full probe adds no extra
* request; the packument is already fetched.
*
* Published package *names* stay cached. `version` is refreshed on the same
* daily cadence when this script actually runs (registry-only). In CI, push
* builds usually skip probes when the cache hits; the nightly `PROBE_ALL`
* full pass is what keeps versions fresh there. Unpublished verdicts are
* fully re-probed daily. Network failures leave the existing entry untouched.
* A packument that is missing `dist-tags.latest` does not wipe a previously
* recorded version.
* Published package *names* stay cached. Version freshness in CI comes from
* the nightly `PROBE_ALL` full pass. Rows that predate the `version` field
* get a one-shot registry-only backfill (key absent → fetch once, write the
* key). Network failures leave the existing entry untouched. A packument
* missing `dist-tags.latest` does not wipe a previously recorded string.
*
* Usage: node scripts/probe-npm.mjs
*/
@@ -55,21 +55,21 @@ const priorVersion = (url) => {
return typeof v === 'string' ? v : null
}
const hasVersionKey = (entry) => Object.prototype.hasOwnProperty.call(entry ?? {}, 'version')
// Full probe: unknown, forced, or an expired "not on npm" verdict.
const needsFullProbe = (entry) =>
PROBE_ALL
|| entry === undefined
|| (entry.npm === null && ageDays(entry) > RECHECK_DAYS)
// Published name is sticky; version still moves and must be refreshed, and
// older map rows predate the field entirely (backfill on first run).
const needsVersionRefresh = (entry) =>
Boolean(entry?.npm)
&& (PROBE_ALL || typeof entry.version !== 'string' || ageDays(entry) > RECHECK_DAYS)
// One-shot backfill for map rows that predate the version field. Not an
// ongoing refresh — nightly PROBE_ALL already renews versions via full probe.
const needsVersionBackfill = (entry) => Boolean(entry?.npm) && !hasVersionKey(entry)
const pendingFull = urls.filter((url) => needsFullProbe(map[url]))
const pendingVersion = urls.filter((url) => !needsFullProbe(map[url]) && needsVersionRefresh(map[url]))
console.log(`${urls.length} listed, ${pendingFull.length} full probe(s), ${pendingVersion.length} version refresh(es)`)
const pendingVersion = urls.filter((url) => !needsFullProbe(map[url]) && needsVersionBackfill(map[url]))
console.log(`${urls.length} listed, ${pendingFull.length} full probe(s), ${pendingVersion.length} version backfill(s)`)
async function fetchJson(url) {
const res = await fetch(url, {
@@ -97,8 +97,7 @@ async function probe(url) {
const repoField = typeof repository === 'string' ? repository : repository?.url ?? ''
const linked = repoField.toLowerCase().includes(repo.toLowerCase())
if (!linked) return { npm: null, version: null, checkedAt: today }
// Keep a previously recorded version when the packument has no latest tag
// rather than publishing a false "unknown" over known data.
// Always write the version key. Keep a prior string when latest is absent.
return {
npm: name,
version: latest ?? priorVersion(url),
@@ -109,21 +108,16 @@ async function probe(url) {
}
}
/** Refresh dist-tags.latest for a package already confirmed published. */
async function refreshVersion(url) {
/** One-shot dist-tags.latest backfill for a package already confirmed published. */
async function backfillVersion(url) {
const name = map[url]?.npm
if (typeof name !== 'string') return null
try {
const meta = await fetchJson(`https://registry.npmjs.org/${encodeURIComponent(name)}`)
const latest = typeof meta['dist-tags']?.latest === 'string' ? meta['dist-tags'].latest : null
if (latest === null) {
const kept = priorVersion(url)
// No latest tag: leave the whole entry untouched if we never had a
// version; otherwise bump checkedAt but keep the old version string.
if (kept === null) return null
return { npm: name, version: kept, checkedAt: today }
}
return { npm: name, version: latest, checkedAt: today }
// Write the key even when latest is missing (null), so push-time backfill
// does not re-trigger forever on typeof-null checks.
return { npm: name, version: latest ?? priorVersion(url), checkedAt: today }
} catch {
return null
}
@@ -144,7 +138,7 @@ async function runBatch(label, pending, worker) {
}
await runBatch('full', pendingFull, probe)
await runBatch('version', pendingVersion, refreshVersion)
await runBatch('version-backfill', pendingVersion, backfillVersion)
const listed = new Set(urls)
for (const k of Object.keys(map)) if (!listed.has(k)) delete map[k]