feat(toplist): 新增 npm Top200 周榜缓存功能
- db.js: 新增 top_list 表 + replaceTopList/getTopListRows/getTopListCount - npm-toplist.js: 核心模块(Search+Downloads 复核+缓存比对+经私服拉取) - server.js: /api/toplist* 端点 + scheduleWeeklyTopList(周一 00:00 定时抓榜) - 前端: TopListView 页面、路由、顶/底栏菜单、store/api 扩展 实现: 22 关键词 Search 粗排 350 → Downloads 复核截 200 → 与 rawCache 三态比对(待拉取/需更新/已就绪)→ 一键拉取待缓存/需更新包 复用 sync-latest 的 triggerVerdaccio 落盘机制,限流 1.5s 间隔 + 429 指数退避
This commit is contained in:
+96
-1
@@ -19,7 +19,7 @@ import { randomUUID } from 'node:crypto'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import {
|
||||
getSetting, setSetting, getIgnored, getIgnoredRows, addIgnored, removeIgnored,
|
||||
addLog, getLogs,
|
||||
addLog, getLogs, nowIso,
|
||||
} from './db.js'
|
||||
import {
|
||||
resolveStorageDir, isValidStorage, validateStorage, scanStorage, scanStorageAsync,
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
import { cleanPackage, deleteSingleVersion } from './clean.js'
|
||||
import semverCompare from './semver.js'
|
||||
import { syncState, runSyncLatest } from './sync-latest.js'
|
||||
import { toplistState, runTopListFetch, runTopListPull, getTopList } from './npm-toplist.js'
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
const app = express()
|
||||
@@ -481,6 +482,78 @@ app.get('/api/sync/progress', (req, res) => {
|
||||
})
|
||||
})
|
||||
|
||||
// —— API:Top200 榜单 ——
|
||||
app.get('/api/toplist', (req, res) => {
|
||||
const { filter } = req.query
|
||||
const data = getTopList(getRawPackages)
|
||||
if (filter === 'missing') {
|
||||
data.list = data.list.filter(e => !e.cached)
|
||||
} else if (filter === 'outdated') {
|
||||
data.list = data.list.filter(e => e.cached && !e.latestCached)
|
||||
} else if (filter === 'ready') {
|
||||
data.list = data.list.filter(e => e.cached && e.latestCached)
|
||||
}
|
||||
res.json(data)
|
||||
})
|
||||
|
||||
app.get('/api/toplist/progress', (req, res) => {
|
||||
res.json(toplistState)
|
||||
})
|
||||
|
||||
app.post('/api/toplist/fetch', (req, res) => {
|
||||
if (!toplistState.running) {
|
||||
runTopListFetch({
|
||||
addLog,
|
||||
getRawPackages,
|
||||
invalidateScan,
|
||||
setSetting,
|
||||
})
|
||||
}
|
||||
res.json({
|
||||
ok: true,
|
||||
running: toplistState.running,
|
||||
phase: toplistState.phase,
|
||||
startedAt: toplistState.startedAt,
|
||||
})
|
||||
})
|
||||
|
||||
app.post('/api/toplist/pull', (req, res) => {
|
||||
if (isDemoMode()) {
|
||||
return res.json({ ok: true, demo: true, running: false, message: '演示模式:未配置 Verdaccio 缓存目录,不支持拉取' })
|
||||
}
|
||||
if (!toplistState.pullRunning) {
|
||||
runTopListPull({
|
||||
addLog,
|
||||
getRawPackages,
|
||||
invalidateScan,
|
||||
})
|
||||
}
|
||||
res.json({
|
||||
ok: true,
|
||||
demo: false,
|
||||
running: toplistState.pullRunning,
|
||||
total: toplistState.pullTotal,
|
||||
startedAt: toplistState.pullStartedAt,
|
||||
})
|
||||
})
|
||||
|
||||
app.get('/api/toplist/pull/progress', (req, res) => {
|
||||
res.json({
|
||||
running: toplistState.pullRunning,
|
||||
done: toplistState.pullDone,
|
||||
total: toplistState.pullTotal,
|
||||
processed: toplistState.pullProcessed,
|
||||
success: toplistState.pullSuccess,
|
||||
skipped: toplistState.pullSkipped,
|
||||
failed: toplistState.pullFailed,
|
||||
failedList: toplistState.pullFailedList,
|
||||
current: toplistState.pullCurrent,
|
||||
error: toplistState.pullError,
|
||||
startedAt: toplistState.pullStartedAt,
|
||||
finishedAt: toplistState.pullFinishedAt,
|
||||
})
|
||||
})
|
||||
|
||||
// —— API:清理 ——
|
||||
app.post('/api/clean', (req, res) => {
|
||||
const { pkgName, keep } = req.body || {}
|
||||
@@ -593,6 +666,7 @@ app.listen(PORT, () => {
|
||||
if (mode === 'storage') console.log(`[verdaccio-manager] storage: ${storage}`)
|
||||
else console.log('[verdaccio-manager] 未配置 VERDACCIO_STORAGE,运行于演示模式(设置 VCM_DEMO=0 + 真实目录可关闭)')
|
||||
scheduleDailyScan()
|
||||
scheduleWeeklyTopList()
|
||||
})
|
||||
|
||||
// 清理登录限速表:超过窗口 + 锁定期的记录不再保留(防 Map 无限增长)
|
||||
@@ -633,6 +707,27 @@ function scheduleDailyScan() {
|
||||
}, wait)
|
||||
}
|
||||
|
||||
// —— 每周一 00:00(服务器本地时区)自动抓取 Top200 榜单 ——
|
||||
// 与每日扫描(03:00)错峰,避免并发冲突
|
||||
function scheduleWeeklyTopList() {
|
||||
const now = new Date()
|
||||
const next = new Date(now)
|
||||
next.setHours(0, 0, 0, 0)
|
||||
const dow = next.getDay() // 0=Sun, 1=Mon
|
||||
let daysUntilMonday = (1 - dow + 7) % 7
|
||||
if (daysUntilMonday === 0 && now.getTime() > next.getTime()) daysUntilMonday = 7
|
||||
next.setDate(next.getDate() + daysUntilMonday)
|
||||
setSetting('toplist_next_run', nowIso(next))
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
await runTopListFetch({ addLog, getRawPackages, invalidateScan, setSetting })
|
||||
} catch (e) {
|
||||
console.error('[toplist] 定时抓榜失败:', e.message)
|
||||
}
|
||||
scheduleWeeklyTopList()
|
||||
}, next.getTime() - now.getTime())
|
||||
}
|
||||
|
||||
function clampInt(v, min, max) {
|
||||
const n = Number(v)
|
||||
if (Number.isNaN(n)) return min
|
||||
|
||||
Reference in New Issue
Block a user