feat(toplist): 新增 npm Top200 周榜缓存功能

- db.js: 新增 top_list 表 + replaceTopList/getTopListRows/getTopListCount
- npm-toplist.js: 核心模块(Search+Downloads 复核+缓存比对+经私服拉取)
- server.js: /api/toplist* 端点 + scheduleWeeklyTopList(周一 00:00 定时抓榜)
- 前端: TopListView 页面、路由、顶/底栏菜单、store/api 扩展

实现: 22 关键词 Search 粗排 350 → Downloads 复核截 200 → 与 rawCache 三态比对(待拉取/需更新/已就绪)→ 一键拉取待缓存/需更新包
复用 sync-latest 的 triggerVerdaccio 落盘机制,限流 1.5s 间隔 + 429 指数退避
This commit is contained in:
yinjun.chen1
2026-09-22 10:50:43 +08:00
parent b7be2ee14e
commit b0f8a60d6f
9 changed files with 860 additions and 2 deletions
+96 -1
View File
@@ -19,7 +19,7 @@ import { randomUUID } from 'node:crypto'
import { fileURLToPath } from 'node:url'
import {
getSetting, setSetting, getIgnored, getIgnoredRows, addIgnored, removeIgnored,
addLog, getLogs,
addLog, getLogs, nowIso,
} from './db.js'
import {
resolveStorageDir, isValidStorage, validateStorage, scanStorage, scanStorageAsync,
@@ -28,6 +28,7 @@ import {
import { cleanPackage, deleteSingleVersion } from './clean.js'
import semverCompare from './semver.js'
import { syncState, runSyncLatest } from './sync-latest.js'
import { toplistState, runTopListFetch, runTopListPull, getTopList } from './npm-toplist.js'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const app = express()
@@ -481,6 +482,78 @@ app.get('/api/sync/progress', (req, res) => {
})
})
// —— API:Top200 榜单 ——
app.get('/api/toplist', (req, res) => {
const { filter } = req.query
const data = getTopList(getRawPackages)
if (filter === 'missing') {
data.list = data.list.filter(e => !e.cached)
} else if (filter === 'outdated') {
data.list = data.list.filter(e => e.cached && !e.latestCached)
} else if (filter === 'ready') {
data.list = data.list.filter(e => e.cached && e.latestCached)
}
res.json(data)
})
app.get('/api/toplist/progress', (req, res) => {
res.json(toplistState)
})
app.post('/api/toplist/fetch', (req, res) => {
if (!toplistState.running) {
runTopListFetch({
addLog,
getRawPackages,
invalidateScan,
setSetting,
})
}
res.json({
ok: true,
running: toplistState.running,
phase: toplistState.phase,
startedAt: toplistState.startedAt,
})
})
app.post('/api/toplist/pull', (req, res) => {
if (isDemoMode()) {
return res.json({ ok: true, demo: true, running: false, message: '演示模式:未配置 Verdaccio 缓存目录,不支持拉取' })
}
if (!toplistState.pullRunning) {
runTopListPull({
addLog,
getRawPackages,
invalidateScan,
})
}
res.json({
ok: true,
demo: false,
running: toplistState.pullRunning,
total: toplistState.pullTotal,
startedAt: toplistState.pullStartedAt,
})
})
app.get('/api/toplist/pull/progress', (req, res) => {
res.json({
running: toplistState.pullRunning,
done: toplistState.pullDone,
total: toplistState.pullTotal,
processed: toplistState.pullProcessed,
success: toplistState.pullSuccess,
skipped: toplistState.pullSkipped,
failed: toplistState.pullFailed,
failedList: toplistState.pullFailedList,
current: toplistState.pullCurrent,
error: toplistState.pullError,
startedAt: toplistState.pullStartedAt,
finishedAt: toplistState.pullFinishedAt,
})
})
// —— API:清理 ——
app.post('/api/clean', (req, res) => {
const { pkgName, keep } = req.body || {}
@@ -593,6 +666,7 @@ app.listen(PORT, () => {
if (mode === 'storage') console.log(`[verdaccio-manager] storage: ${storage}`)
else console.log('[verdaccio-manager] 未配置 VERDACCIO_STORAGE,运行于演示模式(设置 VCM_DEMO=0 + 真实目录可关闭)')
scheduleDailyScan()
scheduleWeeklyTopList()
})
// 清理登录限速表:超过窗口 + 锁定期的记录不再保留(防 Map 无限增长)
@@ -633,6 +707,27 @@ function scheduleDailyScan() {
}, wait)
}
// —— 每周一 00:00(服务器本地时区)自动抓取 Top200 榜单 ——
// 与每日扫描(03:00)错峰,避免并发冲突
function scheduleWeeklyTopList() {
const now = new Date()
const next = new Date(now)
next.setHours(0, 0, 0, 0)
const dow = next.getDay() // 0=Sun, 1=Mon
let daysUntilMonday = (1 - dow + 7) % 7
if (daysUntilMonday === 0 && now.getTime() > next.getTime()) daysUntilMonday = 7
next.setDate(next.getDate() + daysUntilMonday)
setSetting('toplist_next_run', nowIso(next))
setTimeout(async () => {
try {
await runTopListFetch({ addLog, getRawPackages, invalidateScan, setSetting })
} catch (e) {
console.error('[toplist] 定时抓榜失败:', e.message)
}
scheduleWeeklyTopList()
}, next.getTime() - now.getTime())
}
function clampInt(v, min, max) {
const n = Number(v)
if (Number.isNaN(n)) return min