mirror of
https://github.com/awesome-dsh-plugin/awesome-dsh-plugin.git
synced 2026-09-28 05:13:14 +08:00
fix: 收紧 version 背填条件并去掉按日刷新
- 仅当缺少 version 键时背填,显式 null 不再触发 push 死循环 - 背填只跑 probe-npm,不连带 downloads/stars 等 - 版本新鲜度交给 nightly PROBE_ALL,去掉按日 registry refresh Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -138,21 +138,23 @@ jobs:
|
||||
# every entry with npm and downloads blank. Probing is slow; that is
|
||||
# cheaper than a wrong plugins.json.
|
||||
#
|
||||
# Also re-probe on push when the restored npm-map still has published
|
||||
# rows without a string `version` (schema backfill after that field
|
||||
# shipped). Skipping would publish plugins.json with npm set and
|
||||
# version null for every package until the next nightly.
|
||||
# Schema backfill: if the restored npm-map still has published rows
|
||||
# that predate the `version` key, run probe-npm only (not the whole
|
||||
# suite). Check for a missing key — not `typeof !== "string"` — so
|
||||
# an explicit `version: null` does not re-trigger on every push.
|
||||
NEED_PROBES=0
|
||||
NEED_NPM_VERSION_BACKFILL=0
|
||||
if [ "${{ github.event_name }}" != "push" ] || [ ! -s data/npm-map.json ]; then
|
||||
NEED_PROBES=1
|
||||
elif node -e '
|
||||
const map = JSON.parse(require("fs").readFileSync("data/npm-map.json", "utf8"));
|
||||
const published = Object.values(map).filter((e) => e && e.npm);
|
||||
const missing = published.some((e) => typeof e.version !== "string");
|
||||
const missing = Object.values(map).some(
|
||||
(e) => e && e.npm && !Object.prototype.hasOwnProperty.call(e, "version"),
|
||||
);
|
||||
process.exit(missing ? 0 : 1);
|
||||
'; then
|
||||
echo "cached npm-map has published entries without version — probing to backfill"
|
||||
NEED_PROBES=1
|
||||
echo "cached npm-map has published entries without a version key — backfilling via probe-npm"
|
||||
NEED_NPM_VERSION_BACKFILL=1
|
||||
fi
|
||||
if [ "$NEED_PROBES" = 1 ]; then
|
||||
node scripts/probe-npm.mjs
|
||||
@@ -174,6 +176,8 @@ jobs:
|
||||
# downloads), and absence of notes is a normal state downstream,
|
||||
# so this probe never blocks the build.
|
||||
node scripts/probe-updates.mjs
|
||||
elif [ "$NEED_NPM_VERSION_BACKFILL" = 1 ]; then
|
||||
node scripts/probe-npm.mjs
|
||||
else
|
||||
echo "push build with cached probe data — skipping probes"
|
||||
fi
|
||||
|
||||
@@ -356,9 +356,11 @@ for (const e of ordered) {
|
||||
// Consumers must tell "not published" apart from "published, unused".
|
||||
e.downloads = downloadsMap[e.url]?.downloads ?? null
|
||||
// registry dist-tags.latest from probe-npm.mjs. null when not on npm, OR
|
||||
// when the package is on npm but the map row has not been backfilled /
|
||||
// refreshed yet (consumers: use `npm != null && version == null` for that
|
||||
// gap — do not treat version null alone as "github-only").
|
||||
// when probed but no latest tag was available. A published row whose map
|
||||
// entry still lacks the `version` key has not been backfilled yet — after
|
||||
// backfill the key is always present (string or null). Consumers: prefer
|
||||
// `npm` for "on the registry"; treat missing/null version as "unknown",
|
||||
// not as "github-only".
|
||||
// Surfaced for dsh-market's discover list (dsh-market#348).
|
||||
e.version = e.npm ? (npmMap[e.url]?.version ?? null) : null
|
||||
e.slug = e.sub ? `${e.repo}--${e.sub.replaceAll('/', '-')}` : e.repo
|
||||
@@ -940,9 +942,8 @@ const registry = {
|
||||
// by parsing the command string is not a contract worth offering, so the
|
||||
// field is published directly. Omitted when absent, like `screenshots`.
|
||||
tarball: e.tarball ?? undefined,
|
||||
// Current npm `latest` when known. null means either github-only
|
||||
// (`npm` is also null) or published but not yet version-backfilled
|
||||
// (`npm` set, `version` null) — not interchangeable with `downloads`.
|
||||
// Current npm `latest` when known. null = github-only (`npm` null) or
|
||||
// probed with no latest tag. Not the same signal as `downloads`.
|
||||
version: e.version,
|
||||
stars: e.stars,
|
||||
downloads: e.downloads,
|
||||
|
||||
+25
-31
@@ -13,18 +13,18 @@
|
||||
*
|
||||
* Results are cached in data/npm-map.json:
|
||||
* { "<github url>": { npm, version, checkedAt } }
|
||||
* `version` is the registry `dist-tags.latest` when `npm` is set.
|
||||
* dsh-market's "discover" list reads it from plugins.json (dsh-market#348)
|
||||
* so clients do not page the registry. The full probe already fetched the
|
||||
* packument; recording `latest` there adds no extra request.
|
||||
* `version` is the registry `dist-tags.latest` when known, else null.
|
||||
* The key is always written once a published (or not) verdict is recorded so
|
||||
* consumers can tell "not backfilled yet" (key absent) from "probed, no
|
||||
* latest tag" (`version: null`). dsh-market reads it from plugins.json
|
||||
* (dsh-market#348). Recording `latest` on the full probe adds no extra
|
||||
* request; the packument is already fetched.
|
||||
*
|
||||
* Published package *names* stay cached. `version` is refreshed on the same
|
||||
* daily cadence when this script actually runs (registry-only). In CI, push
|
||||
* builds usually skip probes when the cache hits; the nightly `PROBE_ALL`
|
||||
* full pass is what keeps versions fresh there. Unpublished verdicts are
|
||||
* fully re-probed daily. Network failures leave the existing entry untouched.
|
||||
* A packument that is missing `dist-tags.latest` does not wipe a previously
|
||||
* recorded version.
|
||||
* Published package *names* stay cached. Version freshness in CI comes from
|
||||
* the nightly `PROBE_ALL` full pass. Rows that predate the `version` field
|
||||
* get a one-shot registry-only backfill (key absent → fetch once, write the
|
||||
* key). Network failures leave the existing entry untouched. A packument
|
||||
* missing `dist-tags.latest` does not wipe a previously recorded string.
|
||||
*
|
||||
* Usage: node scripts/probe-npm.mjs
|
||||
*/
|
||||
@@ -55,21 +55,21 @@ const priorVersion = (url) => {
|
||||
return typeof v === 'string' ? v : null
|
||||
}
|
||||
|
||||
const hasVersionKey = (entry) => Object.prototype.hasOwnProperty.call(entry ?? {}, 'version')
|
||||
|
||||
// Full probe: unknown, forced, or an expired "not on npm" verdict.
|
||||
const needsFullProbe = (entry) =>
|
||||
PROBE_ALL
|
||||
|| entry === undefined
|
||||
|| (entry.npm === null && ageDays(entry) > RECHECK_DAYS)
|
||||
|
||||
// Published name is sticky; version still moves and must be refreshed, and
|
||||
// older map rows predate the field entirely (backfill on first run).
|
||||
const needsVersionRefresh = (entry) =>
|
||||
Boolean(entry?.npm)
|
||||
&& (PROBE_ALL || typeof entry.version !== 'string' || ageDays(entry) > RECHECK_DAYS)
|
||||
// One-shot backfill for map rows that predate the version field. Not an
|
||||
// ongoing refresh — nightly PROBE_ALL already renews versions via full probe.
|
||||
const needsVersionBackfill = (entry) => Boolean(entry?.npm) && !hasVersionKey(entry)
|
||||
|
||||
const pendingFull = urls.filter((url) => needsFullProbe(map[url]))
|
||||
const pendingVersion = urls.filter((url) => !needsFullProbe(map[url]) && needsVersionRefresh(map[url]))
|
||||
console.log(`${urls.length} listed, ${pendingFull.length} full probe(s), ${pendingVersion.length} version refresh(es)`)
|
||||
const pendingVersion = urls.filter((url) => !needsFullProbe(map[url]) && needsVersionBackfill(map[url]))
|
||||
console.log(`${urls.length} listed, ${pendingFull.length} full probe(s), ${pendingVersion.length} version backfill(s)`)
|
||||
|
||||
async function fetchJson(url) {
|
||||
const res = await fetch(url, {
|
||||
@@ -97,8 +97,7 @@ async function probe(url) {
|
||||
const repoField = typeof repository === 'string' ? repository : repository?.url ?? ''
|
||||
const linked = repoField.toLowerCase().includes(repo.toLowerCase())
|
||||
if (!linked) return { npm: null, version: null, checkedAt: today }
|
||||
// Keep a previously recorded version when the packument has no latest tag
|
||||
// rather than publishing a false "unknown" over known data.
|
||||
// Always write the version key. Keep a prior string when latest is absent.
|
||||
return {
|
||||
npm: name,
|
||||
version: latest ?? priorVersion(url),
|
||||
@@ -109,21 +108,16 @@ async function probe(url) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Refresh dist-tags.latest for a package already confirmed published. */
|
||||
async function refreshVersion(url) {
|
||||
/** One-shot dist-tags.latest backfill for a package already confirmed published. */
|
||||
async function backfillVersion(url) {
|
||||
const name = map[url]?.npm
|
||||
if (typeof name !== 'string') return null
|
||||
try {
|
||||
const meta = await fetchJson(`https://registry.npmjs.org/${encodeURIComponent(name)}`)
|
||||
const latest = typeof meta['dist-tags']?.latest === 'string' ? meta['dist-tags'].latest : null
|
||||
if (latest === null) {
|
||||
const kept = priorVersion(url)
|
||||
// No latest tag: leave the whole entry untouched if we never had a
|
||||
// version; otherwise bump checkedAt but keep the old version string.
|
||||
if (kept === null) return null
|
||||
return { npm: name, version: kept, checkedAt: today }
|
||||
}
|
||||
return { npm: name, version: latest, checkedAt: today }
|
||||
// Write the key even when latest is missing (null), so push-time backfill
|
||||
// does not re-trigger forever on typeof-null checks.
|
||||
return { npm: name, version: latest ?? priorVersion(url), checkedAt: today }
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
@@ -144,7 +138,7 @@ async function runBatch(label, pending, worker) {
|
||||
}
|
||||
|
||||
await runBatch('full', pendingFull, probe)
|
||||
await runBatch('version', pendingVersion, refreshVersion)
|
||||
await runBatch('version-backfill', pendingVersion, backfillVersion)
|
||||
|
||||
const listed = new Set(urls)
|
||||
for (const k of Object.keys(map)) if (!listed.has(k)) delete map[k]
|
||||
|
||||
Reference in New Issue
Block a user