feat(toplist): 新增 npm Top200 周榜缓存功能
- db.js: 新增 top_list 表 + replaceTopList/getTopListRows/getTopListCount - npm-toplist.js: 核心模块(Search+Downloads 复核+缓存比对+经私服拉取) - server.js: /api/toplist* 端点 + scheduleWeeklyTopList(周一 00:00 定时抓榜) - 前端: TopListView 页面、路由、顶/底栏菜单、store/api 扩展 实现: 22 关键词 Search 粗排 350 → Downloads 复核截 200 → 与 rawCache 三态比对(待拉取/需更新/已就绪)→ 一键拉取待缓存/需更新包 复用 sync-latest 的 triggerVerdaccio 落盘机制,限流 1.5s 间隔 + 429 指数退避
This commit is contained in:
@@ -33,6 +33,17 @@ CREATE TABLE IF NOT EXISTS logs (
|
||||
freed_space TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS top_list (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
rank INTEGER NOT NULL,
|
||||
pkg_name TEXT NOT NULL UNIQUE,
|
||||
latest_version TEXT NOT NULL,
|
||||
weekly_downloads INTEGER NOT NULL DEFAULT 0,
|
||||
search_weekly INTEGER DEFAULT 0,
|
||||
keyword TEXT,
|
||||
fetched_at DATETIME
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_top_list_rank ON top_list(rank);
|
||||
`)
|
||||
|
||||
// 北京时间(Asia/Shanghai 固定 UTC+8,无夏令时):SQLite DEFAULT CURRENT_TIMESTAMP 是 UTC,
|
||||
@@ -163,6 +174,28 @@ export function getLogs({ action = null, limit = 50 } = {}) {
|
||||
return db.prepare('SELECT * FROM logs ORDER BY id DESC LIMIT ?').all(limit)
|
||||
}
|
||||
|
||||
// —— top_list ——
|
||||
const insertTop = db.prepare(
|
||||
`INSERT OR REPLACE INTO top_list (rank, pkg_name, latest_version, weekly_downloads, search_weekly, keyword, fetched_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
)
|
||||
export function replaceTopList(rows) {
|
||||
db.transaction(rows => {
|
||||
db.prepare('DELETE FROM top_list').run()
|
||||
for (const r of rows || []) {
|
||||
insertTop.run(r.rank, r.pkg_name, r.latest_version, r.weekly_downloads || 0, r.search_weekly || 0, r.keyword || null, nowIso())
|
||||
}
|
||||
})(rows)
|
||||
}
|
||||
|
||||
export function getTopListRows() {
|
||||
return db.prepare('SELECT * FROM top_list ORDER BY rank').all()
|
||||
}
|
||||
|
||||
export function getTopListCount() {
|
||||
return db.prepare('SELECT COUNT(*) AS c FROM top_list').get().c
|
||||
}
|
||||
|
||||
// 模块加载时执行时间迁移 + logs v2 字段迁移(均幂等)
|
||||
migrateTimezone()
|
||||
migrateLogsV2()
|
||||
|
||||
@@ -0,0 +1,355 @@
|
||||
/**
|
||||
* npm Top200 周榜缓存 —— 抓取 npm 周下载量榜单,与私服 storage 比对标记三态缓存状态
|
||||
*
|
||||
* 实现原理:
|
||||
* 1. Search 阶段:22 个关键词调用 npm Search API 各取 size=250,粗排取前 350 候选
|
||||
* 2. Recheck 阶段:Downloads API 复核真实周下载量(非 @ 包批量 70/批,@ 包逐个 4 并发),
|
||||
* 用复核值覆盖快照值,降序截取前 200 入榜
|
||||
* 3. Compare 阶段:榜单与私服 rawCache 实时比对,标记 cached / latestCached 三态
|
||||
* 4. Pull 阶段:拉取 cached=false(待拉取)与 latestCached=false(需更新)的包,
|
||||
* 经 Verdaccio ⚙ 代理落盘,与 sync-latest 同机制
|
||||
*
|
||||
* 环境变量:VERDACCIO_BASE_URL(复用 sync-latest)
|
||||
* 不引入新依赖:网络复用 Node 内置 fetch;落盘复用 sync-latest 的 triggerVerdaccio
|
||||
*/
|
||||
import { getSetting, getTopListRows, replaceTopList } from './db.js'
|
||||
import { triggerVerdaccio } from './sync-latest.js'
|
||||
|
||||
// —— 抓榜状态(前端轮询进度,与 syncState 模式对称) ——
|
||||
export const toplistState = {
|
||||
running: false, done: false,
|
||||
phase: 'idle', // 'idle' | 'search' | 'recheck' | 'compare' | 'done'
|
||||
total: 0, processed: 0,
|
||||
success: 0, skipped: 0, failed: 0,
|
||||
failedList: [], // [{name, reason}]
|
||||
current: null, // 当前处理的关键词或包名
|
||||
error: null,
|
||||
startedAt: null, finishedAt: null,
|
||||
// 拉取子状态(与 sync-latest 对称)
|
||||
pullRunning: false, pullTotal: 0, pullProcessed: 0,
|
||||
pullSuccess: 0, pullSkipped: 0, pullFailed: 0, pullFailedList: [],
|
||||
pullCurrent: null, pullStartedAt: null, pullFinishedAt: null,
|
||||
}
|
||||
|
||||
// npm 公共匿名接口,隐性速率限制(超了返回 429 或临时封 IP);用户明确"不急、接口慢慢调"
|
||||
const KEYWORDS = ['javascript', 'node', 'react', 'vue', 'css', 'util', 'web', 'frontend',
|
||||
'angular', 'typescript', 'api', 'test', 'tool', 'cli', 'server', 'build', 'http',
|
||||
'data', 'component', 'style', 'message', 'theme']
|
||||
const SEARCH_API = 'https://registry.npmjs.org/-/v1/search'
|
||||
const DOWNLOADS_API = 'https://api.npmjs.org/downloads/point/last-week'
|
||||
const SEARCH_SIZE = 250
|
||||
const COARSE_TOP = 350
|
||||
const FINAL_TOP = 200
|
||||
const REQ_INTERVAL_MS = 1500 // 请求间隔:避开 npm 匿名隐性限流(用户允许慢)
|
||||
const RECHECK_CONCURRENCY = 4 // @ 包逐个复核并发
|
||||
const RECHECK_BATCH = 70 // 非 @ 包批量复核每批大小(一次请求复核 70 个)
|
||||
const PULL_CONCURRENT = 3 // 拉取并发(与 sync-latest 一致)
|
||||
const RATE_LIMIT_BACKOFF_MAX = 8 // 429 退避最大重试指数(2^n 秒,最高 256s)
|
||||
const FETCH_TIMEOUT = 20_000
|
||||
const PULL_TIMEOUT = 10 * 60_000
|
||||
|
||||
const sleep = ms => new Promise(r => setTimeout(r, ms))
|
||||
|
||||
function resetFetchState() {
|
||||
toplistState.done = false
|
||||
toplistState.phase = 'idle'
|
||||
toplistState.total = 0
|
||||
toplistState.processed = 0
|
||||
toplistState.success = 0
|
||||
toplistState.skipped = 0
|
||||
toplistState.failed = 0
|
||||
toplistState.failedList = []
|
||||
toplistState.current = null
|
||||
toplistState.error = null
|
||||
toplistState.startedAt = new Date().toISOString()
|
||||
}
|
||||
|
||||
/** GET JSON 带超时;带 429/5xx 指数退避重试(2^n 秒,最多 RATE_LIMIT_BACKOFF_MAX 次) */
|
||||
async function fetchJson(url, timeout) {
|
||||
let attempt = 0
|
||||
for (;;) {
|
||||
const ctrl = new AbortController()
|
||||
const timer = setTimeout(() => ctrl.abort(), timeout)
|
||||
try {
|
||||
const r = await fetch(url, { signal: ctrl.signal, headers: { Accept: 'application/json' } })
|
||||
if (r.status === 404) return null
|
||||
if (r.ok) return await r.json()
|
||||
if (r.status === 429 || r.status >= 500) {
|
||||
// 429 或服务端错误:指数退避重试
|
||||
attempt += 1
|
||||
if (attempt > RATE_LIMIT_BACKOFF_MAX) throw new Error(`接口响应 ${r.status}(已重试 ${attempt - 1} 次)`)
|
||||
const waitMs = Math.min(256, 2 ** attempt) * 1000
|
||||
console.warn(`[toplist] ${r.status},退避 ${waitMs / 1000}s 后重试(第 ${attempt} 次)`)
|
||||
await sleep(waitMs)
|
||||
continue
|
||||
}
|
||||
throw new Error(`接口响应 ${r.status}`)
|
||||
} catch (e) {
|
||||
if (e && e.name === 'AbortError') throw new Error('请求超时')
|
||||
throw e
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Search 阶段:顺序抓 22 个关键词 → 合并去重候选(name 唯一,保留最大周下载) */
|
||||
async function runSearch(pool) {
|
||||
toplistState.phase = 'search'
|
||||
toplistState.total = KEYWORDS.length
|
||||
toplistState.processed = 0
|
||||
for (const kw of KEYWORDS) {
|
||||
toplistState.current = `搜索:${kw}`
|
||||
try {
|
||||
const data = await fetchJson(`${SEARCH_API}?text=${encodeURIComponent(kw)}&size=${SEARCH_SIZE}`, FETCH_TIMEOUT)
|
||||
for (const obj of (data && data.objects) || []) {
|
||||
const p = obj.package
|
||||
if (!p || !p.name || !p.version) continue
|
||||
const weekly = Math.max(0, Number((obj.downloads && obj.downloads.weekly) || 0))
|
||||
const prev = pool.get(p.name)
|
||||
if (!prev || weekly > (prev.search_weekly || 0)) {
|
||||
pool.set(p.name, {
|
||||
pkg_name: p.name,
|
||||
latest_version: p.version,
|
||||
search_weekly: weekly,
|
||||
keyword: kw,
|
||||
})
|
||||
}
|
||||
}
|
||||
toplistState.success += 1
|
||||
} catch (e) {
|
||||
const reason = e && e.message
|
||||
toplistState.failed += 1
|
||||
toplistState.failedList.push({ name: `关键词:${kw}`, reason })
|
||||
} finally {
|
||||
toplistState.processed += 1
|
||||
await sleep(REQ_INTERVAL_MS)
|
||||
}
|
||||
}
|
||||
// 粗排截前 350
|
||||
const cands = [...pool.values()].sort((a, b) => (b.search_weekly || 0) - (a.search_weekly || 0)).slice(0, COARSE_TOP)
|
||||
return cands
|
||||
}
|
||||
|
||||
/** Downloads API 批量复核非 @ 包(70/批) */
|
||||
async function recheckBatch(names, map) {
|
||||
for (let i = 0; i < names.length; i += RECHECK_BATCH) {
|
||||
const chunk = names.slice(i, i + RECHECK_BATCH)
|
||||
try {
|
||||
const data = await fetchJson(`${DOWNLOADS_API}/${chunk.join(',')}`, FETCH_TIMEOUT) || {}
|
||||
for (const name of chunk) {
|
||||
if (data[name] && data[name].downloads != null) map.get(name).weekly_downloads = Number(data[name].downloads) || 0
|
||||
}
|
||||
} catch (e) {
|
||||
toplistState.failed += 1
|
||||
toplistState.failedList.push({ name: `批量复核[${i}-${Math.min(i + RECHECK_BATCH, names.length)}]`, reason: e.message })
|
||||
}
|
||||
await sleep(REQ_INTERVAL_MS)
|
||||
}
|
||||
}
|
||||
|
||||
/** Downloads API 逐个复核 @ 包(4 并发 worker pool) */
|
||||
async function recheckScoped(cands, map) {
|
||||
const items = cands.slice()
|
||||
let next = 0
|
||||
const worker = async () => {
|
||||
while (next < items.length) {
|
||||
const c = items[next++]
|
||||
toplistState.current = `复核:${c.pkg_name}`
|
||||
try {
|
||||
const data = await fetchJson(`${DOWNLOADS_API}/${encodeURIComponent(c.pkg_name)}`, FETCH_TIMEOUT)
|
||||
if (data && data.downloads != null) map.get(c.pkg_name).weekly_downloads = Number(data.downloads) || 0
|
||||
toplistState.processed += 1
|
||||
} catch (e) {
|
||||
toplistState.failed += 1
|
||||
toplistState.failedList.push({ name: c.pkg_name, reason: e.message })
|
||||
toplistState.processed += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: Math.min(RECHECK_CONCURRENCY, items.length) }, worker))
|
||||
}
|
||||
|
||||
/** Recheck 阶段:复核真实下载量 → 降序截取前 200 */
|
||||
async function runRecheck(cands) {
|
||||
toplistState.phase = 'recheck'
|
||||
toplistState.total = cands.length
|
||||
toplistState.processed = 0
|
||||
const map = new Map(cands.map(c => [c.pkg_name, c]))
|
||||
const scoped = cands.filter(c => c.pkg_name.startsWith('@'))
|
||||
const unscoped = cands.filter(c => !c.pkg_name.startsWith('@'))
|
||||
toplistState.current = `复核批量 ${unscoped.length} 个`
|
||||
await recheckBatch(unscoped.map(c => c.pkg_name), map)
|
||||
if (scoped.length) await recheckScoped(scoped, map)
|
||||
return [...map.values()]
|
||||
.sort((a, b) => (b.weekly_downloads || 0) - (a.weekly_downloads || 0))
|
||||
.slice(0, FINAL_TOP)
|
||||
.map((c, i) => ({ ...c, rank: i + 1 }))
|
||||
}
|
||||
|
||||
/** Compare 阶段:榜单与私服 rawCache 实时比对,标记三态 */
|
||||
export function compareWithCache(topList, rawPackages) {
|
||||
const pkgMap = new Map((rawPackages || []).map(p => [p.name, p]))
|
||||
return (topList || []).map(entry => {
|
||||
const pkg = pkgMap.get(entry.pkg_name)
|
||||
const vs = pkg && Array.isArray(pkg.versionsDetail) ? pkg.versionsDetail : []
|
||||
const versionSet = new Set(vs.map(v => v.version))
|
||||
return {
|
||||
...entry,
|
||||
cached: versionSet.size > 0,
|
||||
latestCached: versionSet.has(entry.latest_version),
|
||||
cachedVersions: [...versionSet],
|
||||
// 兼容前端易读字段名
|
||||
name: entry.pkg_name,
|
||||
latestVersion: entry.latest_version,
|
||||
weeklyDownload: entry.weekly_downloads,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/** 拉取子状态重置 */
|
||||
function resetPullState() {
|
||||
toplistState.pullRunning = false
|
||||
toplistState.pullDone = false
|
||||
toplistState.pullTotal = 0
|
||||
toplistState.pullProcessed = 0
|
||||
toplistState.pullSuccess = 0
|
||||
toplistState.pullSkipped = 0
|
||||
toplistState.pullFailed = 0
|
||||
toplistState.pullFailedList = []
|
||||
toplistState.pullCurrent = null
|
||||
toplistState.pullError = null
|
||||
toplistState.pullStartedAt = null
|
||||
toplistState.pullFinishedAt = null
|
||||
}
|
||||
|
||||
/** 同步 GET 返回 DB 当前快照 + 实时比对结果(不写 DB),供 GET /api/toplist */
|
||||
export function getTopList(getRawPackages) {
|
||||
const list = getTopListRows()
|
||||
const compared = compareWithCache(list, (getRawPackages && getRawPackages()) || [])
|
||||
const summary = {
|
||||
total: compared.length,
|
||||
cached: compared.filter(e => e.cached).length,
|
||||
latestCached: compared.filter(e => e.latestCached).length,
|
||||
missing: compared.filter(e => !e.cached).length,
|
||||
outdated: compared.filter(e => e.cached && !e.latestCached).length,
|
||||
}
|
||||
return {
|
||||
list: compared,
|
||||
summary,
|
||||
state: toplistState,
|
||||
lastRun: getSetting('toplist_last_run', null),
|
||||
nextRun: getSetting('toplist_next_run', null),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 抓榜主流程:search → recheck → compare → persist
|
||||
* @param {{ addLog, getRawPackages, invalidateScan, setSetting }} services 依赖注入
|
||||
*/
|
||||
export async function runTopListFetch({ addLog, getRawPackages, invalidateScan, setSetting }) {
|
||||
if (toplistState.running) return { ok: false, running: true }
|
||||
resetFetchState()
|
||||
resetPullState()
|
||||
toplistState.running = true
|
||||
console.log('[toplist] 开始抓取 npm Top200 周榜')
|
||||
try {
|
||||
const pool = new Map()
|
||||
await runSearch(pool)
|
||||
const cands = [...pool.values()].sort((a, b) => (b.search_weekly || 0) - (a.search_weekly || 0)).slice(0, COARSE_TOP)
|
||||
if (!cands.length) throw new Error('Search 阶段未获取到任何候选包')
|
||||
|
||||
toplistState.phase = 'compare'
|
||||
// 复核前先算出真实下载,最后统一截取
|
||||
const ranked = await runRecheck(cands)
|
||||
replaceTopList(ranked)
|
||||
|
||||
const lastRun = new Date().toISOString()
|
||||
setSetting('toplist_last_run', lastRun)
|
||||
addLog({
|
||||
action: 'toplist_fetch',
|
||||
result: 'success',
|
||||
message: `Top200 抓榜完成:${ranked.length} 个包入榜`,
|
||||
durationMs: Date.now() - new Date(toplistState.startedAt).getTime(),
|
||||
meta: { total: ranked.length, failed: toplistState.failedList },
|
||||
deletedCount: ranked.length,
|
||||
})
|
||||
toplistState.phase = 'done'
|
||||
console.log(`[toplist] 抓榜完成:${ranked.length} 个包入榜`)
|
||||
} catch (e) {
|
||||
toplistState.error = e.message
|
||||
addLog({ action: 'toplist_fetch', result: 'fail', message: e.message })
|
||||
console.error('[toplist] 抓榜失败:', e.message)
|
||||
} finally {
|
||||
toplistState.running = false
|
||||
toplistState.done = true
|
||||
toplistState.finishedAt = new Date().toISOString()
|
||||
}
|
||||
return { ok: !toplistState.error }
|
||||
}
|
||||
|
||||
/**
|
||||
* 拉取榜单中待缓存 / 需更新的包(经 Verdaccio ⚙ 代理落盘)
|
||||
* @param {{ addLog, getRawPackages, invalidateScan }} services
|
||||
*/
|
||||
export async function runTopListPull({ addLog, getRawPackages, invalidateScan }) {
|
||||
if (toplistState.pullRunning) return { ok: false, running: true }
|
||||
// 必须存在已入榜数据
|
||||
const snapshot = getTopList(getRawPackages)
|
||||
const targets = snapshot.list.filter(e => !e.cached || !e.latestCached)
|
||||
resetPullState()
|
||||
toplistState.pullRunning = true
|
||||
toplistState.pullStartedAt = new Date().toISOString()
|
||||
toplistState.pullTotal = targets.length
|
||||
console.log(`[toplist] 开始拉取榜单待缓存包:${targets.length} 个`)
|
||||
if (!targets.length) {
|
||||
toplistState.pullFinishedAt = new Date().toISOString()
|
||||
toplistState.pullRunning = false
|
||||
toplistState.pullDone = true
|
||||
return { ok: true, running: false, total: 0 }
|
||||
}
|
||||
let next = 0
|
||||
const worker = async () => {
|
||||
while (next < targets.length) {
|
||||
const t = targets[next++]
|
||||
toplistState.pullCurrent = t.pkg_name
|
||||
try {
|
||||
await triggerVerdaccio(t.pkg_name, t.latest_version)
|
||||
toplistState.pullSuccess += 1
|
||||
addLog({
|
||||
action: 'toplist_pull', pkgName: t.pkg_name, version: t.latest_version,
|
||||
result: 'success', message: `已拉取 ${t.latest_version}`,
|
||||
})
|
||||
} catch (e) {
|
||||
const reason = (e && e.message) || '拉取失败'
|
||||
toplistState.pullFailed += 1
|
||||
toplistState.pullFailedList.push({ name: t.pkg_name, reason })
|
||||
addLog({ action: 'toplist_pull', pkgName: t.pkg_name, version: t.latest_version, result: 'fail', message: reason })
|
||||
} finally {
|
||||
toplistState.pullProcessed += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
await Promise.all(Array.from({ length: Math.min(PULL_CONCURRENT, targets.length) }, worker))
|
||||
} catch (e) {
|
||||
toplistState.pullError = e.message
|
||||
} finally {
|
||||
toplistState.pullRunning = false
|
||||
toplistState.pullDone = true
|
||||
toplistState.pullFinishedAt = new Date().toISOString()
|
||||
{
|
||||
// 汇总日志
|
||||
addLog({
|
||||
action: 'toplist_pull_summary',
|
||||
result: toplistState.pullError ? 'fail' : 'success',
|
||||
message: `Top200 拉取完成:成功 ${toplistState.pullSuccess} · 失败 ${toplistState.pullFailed}`,
|
||||
durationMs: Date.now() - new Date(toplistState.pullStartedAt).getTime(),
|
||||
meta: { success: toplistState.pullSuccess, failed: toplistState.pullFailedList },
|
||||
deletedCount: toplistState.pullSuccess,
|
||||
})
|
||||
invalidateScan() // 有新版本入库 → 废弃扫描缓存,列表下次请求重扫
|
||||
}
|
||||
}
|
||||
return { ok: !toplistState.pullError, running: false, total: targets.length }
|
||||
}
|
||||
+96
-1
@@ -19,7 +19,7 @@ import { randomUUID } from 'node:crypto'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import {
|
||||
getSetting, setSetting, getIgnored, getIgnoredRows, addIgnored, removeIgnored,
|
||||
addLog, getLogs,
|
||||
addLog, getLogs, nowIso,
|
||||
} from './db.js'
|
||||
import {
|
||||
resolveStorageDir, isValidStorage, validateStorage, scanStorage, scanStorageAsync,
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
import { cleanPackage, deleteSingleVersion } from './clean.js'
|
||||
import semverCompare from './semver.js'
|
||||
import { syncState, runSyncLatest } from './sync-latest.js'
|
||||
import { toplistState, runTopListFetch, runTopListPull, getTopList } from './npm-toplist.js'
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
const app = express()
|
||||
@@ -481,6 +482,78 @@ app.get('/api/sync/progress', (req, res) => {
|
||||
})
|
||||
})
|
||||
|
||||
// —— API:Top200 榜单 ——
|
||||
app.get('/api/toplist', (req, res) => {
|
||||
const { filter } = req.query
|
||||
const data = getTopList(getRawPackages)
|
||||
if (filter === 'missing') {
|
||||
data.list = data.list.filter(e => !e.cached)
|
||||
} else if (filter === 'outdated') {
|
||||
data.list = data.list.filter(e => e.cached && !e.latestCached)
|
||||
} else if (filter === 'ready') {
|
||||
data.list = data.list.filter(e => e.cached && e.latestCached)
|
||||
}
|
||||
res.json(data)
|
||||
})
|
||||
|
||||
app.get('/api/toplist/progress', (req, res) => {
|
||||
res.json(toplistState)
|
||||
})
|
||||
|
||||
app.post('/api/toplist/fetch', (req, res) => {
|
||||
if (!toplistState.running) {
|
||||
runTopListFetch({
|
||||
addLog,
|
||||
getRawPackages,
|
||||
invalidateScan,
|
||||
setSetting,
|
||||
})
|
||||
}
|
||||
res.json({
|
||||
ok: true,
|
||||
running: toplistState.running,
|
||||
phase: toplistState.phase,
|
||||
startedAt: toplistState.startedAt,
|
||||
})
|
||||
})
|
||||
|
||||
app.post('/api/toplist/pull', (req, res) => {
|
||||
if (isDemoMode()) {
|
||||
return res.json({ ok: true, demo: true, running: false, message: '演示模式:未配置 Verdaccio 缓存目录,不支持拉取' })
|
||||
}
|
||||
if (!toplistState.pullRunning) {
|
||||
runTopListPull({
|
||||
addLog,
|
||||
getRawPackages,
|
||||
invalidateScan,
|
||||
})
|
||||
}
|
||||
res.json({
|
||||
ok: true,
|
||||
demo: false,
|
||||
running: toplistState.pullRunning,
|
||||
total: toplistState.pullTotal,
|
||||
startedAt: toplistState.pullStartedAt,
|
||||
})
|
||||
})
|
||||
|
||||
app.get('/api/toplist/pull/progress', (req, res) => {
|
||||
res.json({
|
||||
running: toplistState.pullRunning,
|
||||
done: toplistState.pullDone,
|
||||
total: toplistState.pullTotal,
|
||||
processed: toplistState.pullProcessed,
|
||||
success: toplistState.pullSuccess,
|
||||
skipped: toplistState.pullSkipped,
|
||||
failed: toplistState.pullFailed,
|
||||
failedList: toplistState.pullFailedList,
|
||||
current: toplistState.pullCurrent,
|
||||
error: toplistState.pullError,
|
||||
startedAt: toplistState.pullStartedAt,
|
||||
finishedAt: toplistState.pullFinishedAt,
|
||||
})
|
||||
})
|
||||
|
||||
// —— API:清理 ——
|
||||
app.post('/api/clean', (req, res) => {
|
||||
const { pkgName, keep } = req.body || {}
|
||||
@@ -593,6 +666,7 @@ app.listen(PORT, () => {
|
||||
if (mode === 'storage') console.log(`[verdaccio-manager] storage: ${storage}`)
|
||||
else console.log('[verdaccio-manager] 未配置 VERDACCIO_STORAGE,运行于演示模式(设置 VCM_DEMO=0 + 真实目录可关闭)')
|
||||
scheduleDailyScan()
|
||||
scheduleWeeklyTopList()
|
||||
})
|
||||
|
||||
// 清理登录限速表:超过窗口 + 锁定期的记录不再保留(防 Map 无限增长)
|
||||
@@ -633,6 +707,27 @@ function scheduleDailyScan() {
|
||||
}, wait)
|
||||
}
|
||||
|
||||
// —— 每周一 00:00(服务器本地时区)自动抓取 Top200 榜单 ——
|
||||
// 与每日扫描(03:00)错峰,避免并发冲突
|
||||
function scheduleWeeklyTopList() {
|
||||
const now = new Date()
|
||||
const next = new Date(now)
|
||||
next.setHours(0, 0, 0, 0)
|
||||
const dow = next.getDay() // 0=Sun, 1=Mon
|
||||
let daysUntilMonday = (1 - dow + 7) % 7
|
||||
if (daysUntilMonday === 0 && now.getTime() > next.getTime()) daysUntilMonday = 7
|
||||
next.setDate(next.getDate() + daysUntilMonday)
|
||||
setSetting('toplist_next_run', nowIso(next))
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
await runTopListFetch({ addLog, getRawPackages, invalidateScan, setSetting })
|
||||
} catch (e) {
|
||||
console.error('[toplist] 定时抓榜失败:', e.message)
|
||||
}
|
||||
scheduleWeeklyTopList()
|
||||
}, next.getTime() - now.getTime())
|
||||
}
|
||||
|
||||
function clampInt(v, min, max) {
|
||||
const n = Number(v)
|
||||
if (Number.isNaN(n)) return min
|
||||
|
||||
@@ -386,6 +386,33 @@ return r.json()
|
||||
})
|
||||
return r.json()
|
||||
},
|
||||
|
||||
// —— Top200 榜单 ——
|
||||
async getTopList(filter = '') {
|
||||
if (USE_MOCK) {
|
||||
await delay(80)
|
||||
return { list: [], summary: { total: 0 }, state: { running: false, phase: 'idle' }, lastRun: null, nextRun: null }
|
||||
}
|
||||
const qs = filter ? `?filter=${encodeURIComponent(filter)}` : ''
|
||||
const r = await authFetch(`/api/toplist${qs}`)
|
||||
return r.json()
|
||||
},
|
||||
async fetchTopList() {
|
||||
const r = await authFetch('/api/toplist/fetch', { method: 'POST' })
|
||||
return r.json()
|
||||
},
|
||||
async getTopListProgress() {
|
||||
const r = await authFetch('/api/toplist/progress')
|
||||
return r.json()
|
||||
},
|
||||
async pullTopList() {
|
||||
const r = await authFetch('/api/toplist/pull', { method: 'POST' })
|
||||
return r.json()
|
||||
},
|
||||
async getTopListPullProgress() {
|
||||
const r = await authFetch('/api/toplist/pull/progress')
|
||||
return r.json()
|
||||
},
|
||||
}
|
||||
|
||||
export default api
|
||||
|
||||
@@ -19,6 +19,7 @@ const navs = [
|
||||
{ path: '/search', label: '我的缓存', emo: '🔍' },
|
||||
{ path: '/thresholds', label: '预警设置', emo: '⚙️' },
|
||||
{ path: '/logs', label: '操作日志', emo: '🕒' },
|
||||
{ path: '/toplist', label: 'Top榜单', emo: '🏆' },
|
||||
{ path: '/ignored', label: '忽略记录', emo: '📋' },
|
||||
]
|
||||
|
||||
|
||||
@@ -14,7 +14,8 @@ const tabs = [
|
||||
{ path: '/search', label: '我的缓存', emo: '🔍' },
|
||||
{ path: '/thresholds', label: '阈值', emo: '⚙️' },
|
||||
{ path: '/logs', label: '操作日志', emo: '🕒' },
|
||||
{ path: '/ignored', label: '忽略记录', emo: '📋' },
|
||||
{ path: '/toplist', label: 'Top榜', emo: '🏆' },
|
||||
{ path: '/ignored', label: '忽略', emo: '📋' },
|
||||
]
|
||||
|
||||
function go(path) {
|
||||
|
||||
@@ -6,6 +6,7 @@ import SearchView from '../views/SearchView.vue'
|
||||
import ThresholdsView from '../views/ThresholdsView.vue'
|
||||
import IgnoredView from '../views/IgnoredView.vue'
|
||||
import LogsView from '../views/LogsView.vue'
|
||||
import TopListView from '../views/TopListView.vue'
|
||||
import { storage } from '../utils/storage'
|
||||
|
||||
const routes = [
|
||||
@@ -15,6 +16,7 @@ const routes = [
|
||||
{ path: '/thresholds', name: 'thresholds', component: ThresholdsView },
|
||||
{ path: '/ignored', name: 'ignored', component: IgnoredView },
|
||||
{ path: '/logs', name: 'logs', component: LogsView },
|
||||
{ path: '/toplist', name: 'toplist', component: TopListView },
|
||||
{ path: '/:pathMatch(.*)*', redirect: '/' },
|
||||
]
|
||||
|
||||
|
||||
@@ -24,6 +24,19 @@ const state = reactive({
|
||||
running: false, done: false, total: 0, processed: 0,
|
||||
success: 0, skipped: 0, failed: 0, failedList: [], error: null,
|
||||
},
|
||||
// Top200 榜单数据与进度
|
||||
toplist: [],
|
||||
toplistSummary: { total: 0, cached: 0, latestCached: 0, missing: 0, outdated: 0 },
|
||||
toplistProgress: {
|
||||
running: false, done: false, phase: 'idle', total: 0, processed: 0,
|
||||
success: 0, skipped: 0, failed: 0, failedList: [], error: null,
|
||||
},
|
||||
toplistPullProgress: {
|
||||
running: false, done: false, total: 0, processed: 0,
|
||||
success: 0, skipped: 0, failed: 0, failedList: [], error: null,
|
||||
},
|
||||
toplistNextRun: null,
|
||||
toplistLastRun: null,
|
||||
})
|
||||
|
||||
// —— 派生统计 ——
|
||||
@@ -227,6 +240,109 @@ function pollSyncProgress() {
|
||||
tick()
|
||||
}
|
||||
|
||||
// —— Top200 榜单 ——
|
||||
let toplistFetchTimer = null
|
||||
let toplistPullTimer = null
|
||||
|
||||
/** 刷新榜单数据(含实时缓存比对状态),可传 filter 筛选子集 */
|
||||
async function refreshTopList(filter = '') {
|
||||
const data = await api.getTopList(filter)
|
||||
state.toplist = data.list || []
|
||||
state.toplistSummary = data.summary || { total: 0 }
|
||||
state.toplistNextRun = data.nextRun || null
|
||||
state.toplistLastRun = data.lastRun || null
|
||||
state.toplistProgress = data.state || state.toplistProgress
|
||||
return data
|
||||
}
|
||||
|
||||
/** 立即抓榜:触发后端任务并轮询进度 */
|
||||
async function fetchTopList() {
|
||||
try {
|
||||
const res = await api.fetchTopList()
|
||||
if (res.ok === false) return res
|
||||
state.toplistProgress.running = true
|
||||
state.toplistProgress.phase = res.phase || 'search'
|
||||
pollTopListProgress()
|
||||
return res
|
||||
} catch (e) {
|
||||
ElMessage.error(e?.message || '发起抓榜失败,请稍后重试')
|
||||
return { ok: false }
|
||||
}
|
||||
}
|
||||
|
||||
/** 轮询抓榜进度,任务结束后刷新榜单 */
|
||||
function pollTopListProgress() {
|
||||
clearTimeout(toplistFetchTimer)
|
||||
const tick = async () => {
|
||||
let p
|
||||
try {
|
||||
p = await api.getTopListProgress()
|
||||
} catch {
|
||||
state.toplistProgress.running = false
|
||||
state.toplistProgress.error = '抓榜进度查询失败'
|
||||
return
|
||||
}
|
||||
Object.assign(state.toplistProgress, {
|
||||
running: !!p.running, done: !!p.done, phase: p.phase || 'idle',
|
||||
total: p.total || 0, processed: p.processed || 0,
|
||||
success: p.success || 0, skipped: p.skipped || 0, failed: p.failed || 0,
|
||||
failedList: p.failedList || [], error: p.error || null,
|
||||
})
|
||||
if (p.running) {
|
||||
toplistFetchTimer = setTimeout(tick, 800)
|
||||
} else {
|
||||
clearTimeout(toplistFetchTimer)
|
||||
await refreshTopList()
|
||||
}
|
||||
}
|
||||
tick()
|
||||
}
|
||||
|
||||
/** 一键拉取待缓存/需更新的榜单包:触发后端任务并轮询进度 */
|
||||
async function pullTopList() {
|
||||
try {
|
||||
const res = await api.pullTopList()
|
||||
if (!res.ok) return res
|
||||
if (res.demo) return res
|
||||
state.toplistPullProgress.running = true
|
||||
state.toplistPullProgress.done = false
|
||||
state.toplistPullProgress.total = res.total || 0
|
||||
pollTopListPullProgress()
|
||||
return res
|
||||
} catch (e) {
|
||||
ElMessage.error(e?.message || '发起拉取失败,请稍后重试')
|
||||
return { ok: false }
|
||||
}
|
||||
}
|
||||
|
||||
/** 轮询拉取进度,任务结束后刷新榜单与日志 */
|
||||
function pollTopListPullProgress() {
|
||||
clearTimeout(toplistPullTimer)
|
||||
const tick = async () => {
|
||||
let p
|
||||
try {
|
||||
p = await api.getTopListPullProgress()
|
||||
} catch {
|
||||
state.toplistPullProgress.running = false
|
||||
state.toplistPullProgress.error = '拉取进度查询失败'
|
||||
return
|
||||
}
|
||||
Object.assign(state.toplistPullProgress, {
|
||||
running: !!p.running, done: !!p.done,
|
||||
total: p.total || 0, processed: p.processed || 0,
|
||||
success: p.success || 0, skipped: p.skipped || 0, failed: p.failed || 0,
|
||||
failedList: p.failedList || [], error: p.error || null,
|
||||
})
|
||||
if (p.running) {
|
||||
toplistPullTimer = setTimeout(tick, 800)
|
||||
} else {
|
||||
clearTimeout(toplistPullTimer)
|
||||
await Promise.all([refreshTopList(), refreshLogs()])
|
||||
}
|
||||
}
|
||||
tick()
|
||||
}
|
||||
|
||||
async function refreshThresholds() {
|
||||
const t = await api.getThresholds()
|
||||
state.threshold.versionThreshold = t.versionThreshold
|
||||
@@ -293,6 +409,9 @@ export function useStore() {
|
||||
refreshLogs,
|
||||
scanNow,
|
||||
syncLatest,
|
||||
refreshTopList,
|
||||
fetchTopList,
|
||||
pullTopList,
|
||||
saveThresholds,
|
||||
ignorePkg,
|
||||
unignorePkg,
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
<script setup>
|
||||
import { ref, computed, onMounted } from 'vue'
|
||||
import { useStore } from '../store'
|
||||
|
||||
const { state, refreshTopList, fetchTopList, pullTopList } = useStore()
|
||||
|
||||
const filter = ref('') // '' | missing | outdated | ready
|
||||
|
||||
const PHASE_TEXT = {
|
||||
idle: '空闲',
|
||||
search: '搜索关键词中',
|
||||
recheck: '复核下载量中',
|
||||
compare: '比对缓存中',
|
||||
done: '完成',
|
||||
}
|
||||
|
||||
/** 当前筛选下的列表 */
|
||||
const rows = computed(() => {
|
||||
if (!filter.value) return state.toplist
|
||||
return state.toplist.filter(e => {
|
||||
if (filter.value === 'missing') return !e.cached
|
||||
if (filter.value === 'outdated') return e.cached && !e.latestCached
|
||||
if (filter.value === 'ready') return e.cached && e.latestCached
|
||||
return true
|
||||
})
|
||||
})
|
||||
|
||||
const phaseText = computed(() => state.toplistProgress.running
|
||||
? PHASE_TEXT[state.toplistProgress.phase] || '处理中'
|
||||
: '')
|
||||
|
||||
const fetchPercent = computed(() => {
|
||||
const t = state.toplistProgress.total
|
||||
return t ? Math.min(100, Math.round((state.toplistProgress.processed / t) * 100)) : 0
|
||||
})
|
||||
const pullPercent = computed(() => {
|
||||
const t = state.toplistPullProgress.total
|
||||
return t ? Math.min(100, Math.round((state.toplistPullProgress.processed / t) * 100)) : 0
|
||||
})
|
||||
|
||||
function fmt(n) {
|
||||
if (n == null) return '0'
|
||||
return Number(n).toLocaleString('en-US')
|
||||
}
|
||||
|
||||
function fmtDate(iso) {
|
||||
if (!iso) return '—'
|
||||
return String(iso).slice(0, 16)
|
||||
}
|
||||
|
||||
async function doFetch() {
|
||||
await fetchTopList()
|
||||
}
|
||||
|
||||
async function doPull() {
|
||||
await pullTopList()
|
||||
}
|
||||
|
||||
onMounted(() => refreshTopList())
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="page">
|
||||
<div class="head">
|
||||
<div class="title">
|
||||
<h2>🏆 npm Top200 周榜</h2>
|
||||
<span class="sub">每周一 00:00 自动抓取,据此缓存热门包最新版</span>
|
||||
</div>
|
||||
<div class="actions">
|
||||
<button class="btn primary" :disabled="state.toplistProgress.running" @click="doFetch">
|
||||
{{ state.toplistProgress.running ? '⏳ 抓榜中…' : '🔄 立即抓榜' }}
|
||||
</button>
|
||||
<button class="btn" :disabled="state.toplistPullProgress.running" @click="doPull">
|
||||
{{ state.toplistPullProgress.running ? '⏳ 拉取中…' : '📥 一键拉取' }}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 摘要条 -->
|
||||
<div class="summary">
|
||||
<span>共 <b>{{ state.toplistSummary.total || 0 }}</b></span>
|
||||
<span>已缓存 <b class="g">{{ state.toplistSummary.cached || 0 }}</b></span>
|
||||
<span>待拉取 <b class="r">{{ state.toplistSummary.missing || 0 }}</b></span>
|
||||
<span>需更新 <b class="y">{{ state.toplistSummary.outdated || 0 }}</b></span>
|
||||
<span class="next">下次抓榜 <b>{{ fmtDate(state.toplistNextRun) }}</b></span>
|
||||
<span class="last">上次抓榜 <b>{{ fmtDate(state.toplistLastRun) }}</b></span>
|
||||
</div>
|
||||
|
||||
<!-- 抓榜进度 -->
|
||||
<div v-if="state.toplistProgress.running" class="bar">
|
||||
<div class="bar-head">
|
||||
<span>{{ phaseText }} · {{ state.toplistProgress.processed }}/{{ state.toplistProgress.total }}</span>
|
||||
<span>{{ fetchPercent }}%</span>
|
||||
</div>
|
||||
<div class="track"><div class="fill" :style="{ width: fetchPercent + '%' }"></div></div>
|
||||
</div>
|
||||
|
||||
<!-- 拉取进度 -->
|
||||
<div v-if="state.toplistPullProgress.running" class="bar pull">
|
||||
<div class="bar-head">
|
||||
<span>正在拉取 {{ state.toplistPullProgress.current || '' }} · 成功 {{ state.toplistPullProgress.success }} · 失败 {{ state.toplistPullProgress.failed }}</span>
|
||||
<span>{{ state.toplistPullProgress.processed }}/{{ state.toplistPullProgress.total }}</span>
|
||||
</div>
|
||||
<div class="track"><div class="fill" :style="{ width: pullPercent + '%' }"></div></div>
|
||||
</div>
|
||||
|
||||
<!-- 筛选 tabs -->
|
||||
<div class="tabs">
|
||||
<span class="tab" :class="{ active: filter === '' }" @click="filter = ''">全部</span>
|
||||
<span class="tab" :class="{ active: filter === 'missing' }" @click="filter = 'missing'">待拉取</span>
|
||||
<span class="tab" :class="{ active: filter === 'outdated' }" @click="filter = 'outdated'">需更新</span>
|
||||
<span class="tab" :class="{ active: filter === 'ready' }" @click="filter = 'ready'">已就绪</span>
|
||||
</div>
|
||||
|
||||
<!-- 空态 -->
|
||||
<div v-if="!state.toplist.length" class="empty">
|
||||
<div class="empty-emo">🏆</div>
|
||||
<p>尚无 Top200 数据,点击「立即抓榜」获取榜单</p>
|
||||
</div>
|
||||
|
||||
<!-- 表格 -->
|
||||
<div v-else class="table">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>#</th>
|
||||
<th>包名</th>
|
||||
<th>最新版本</th>
|
||||
<th class="num">周下载</th>
|
||||
<th>状态</th>
|
||||
<th>已缓存版本</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="row in rows" :key="row.rank">
|
||||
<td class="dim">{{ row.rank }}</td>
|
||||
<td class="name">{{ row.name }}</td>
|
||||
<td class="ver">{{ row.latestVersion }}</td>
|
||||
<td class="num">{{ fmt(row.weeklyDownload) }}</td>
|
||||
<td>
|
||||
<span v-if="!row.cached" class="badge r">❌ 待拉取</span>
|
||||
<span v-else-if="!row.latestCached" class="badge y">⬆️ 需更新</span>
|
||||
<span v-else class="badge g">✅ 已就绪</span>
|
||||
</td>
|
||||
<td class="dim vs">{{ (row.cachedVersions || []).length ? row.cachedVersions.length + ' 个' : '—' }}</td>
|
||||
</tr>
|
||||
<tr v-if="!rows.length"><td colspan="6" class="no">当前筛选下无条目</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.page { padding: 24px; display: flex; flex-direction: column; gap: 16px; }
|
||||
.head { display: flex; align-items: flex-end; justify-content: space-between; gap: 16px; flex-wrap: wrap; }
|
||||
.title h2 { font-size: 21px; margin: 0; letter-spacing: -.3px; }
|
||||
.sub { font-size: 12.5px; color: var(--ink-faint); margin-top: 4px; display: inline-block; }
|
||||
.actions { display: flex; gap: 10px; }
|
||||
.btn {
|
||||
padding: 9px 20px; border: 1.5px solid var(--line); border-radius: 11px;
|
||||
background: var(--card); color: var(--ink); font-size: 13.5px; font-weight: 700;
|
||||
cursor: pointer; transition: .16s;
|
||||
}
|
||||
.btn:hover { border-color: var(--orange); color: var(--orange); }
|
||||
.btn.primary { border-color: transparent; background: var(--grad-main); color: #fff; box-shadow: 0 8px 18px -6px rgba(255,140,0,.5); }
|
||||
.btn:disabled { opacity: .55; cursor: not-allowed; }
|
||||
|
||||
.summary {
|
||||
display: flex; flex-wrap: wrap; gap: 10px 26px; padding: 14px 18px;
|
||||
background: var(--panel-soft); border: 1px solid var(--line); border-radius: 13px;
|
||||
font-size: 13px; color: var(--ink-dim);
|
||||
}
|
||||
.summary b { font-family: var(--num); color: var(--ink); }
|
||||
.summary b.g { color: var(--green-deep); }
|
||||
.summary b.r { color: var(--red); }
|
||||
.summary b.y { color: var(--orange); }
|
||||
.summary .next, .summary .last { margin-left: auto; }
|
||||
|
||||
.bar .bar-head { display: flex; justify-content: space-between; font-size: 12.5px; color: var(--ink-dim); margin-bottom: 7px; }
|
||||
.track { height: 8px; border-radius: 99px; background: var(--line); overflow: hidden; }
|
||||
.fill { height: 100%; border-radius: 99px; background: var(--grad-main); transition: width .3s; }
|
||||
.bar.pull .fill { background: var(--grad-avatar); }
|
||||
|
||||
.tabs { display: flex; gap: 8px; flex-wrap: wrap; }
|
||||
.tab {
|
||||
padding: 7px 18px; border-radius: 99px; font-size: 13px; font-weight: 600;
|
||||
color: var(--ink-dim); background: var(--card); border: 1.5px solid var(--line); cursor: pointer; transition: .16s;
|
||||
}
|
||||
.tab:hover { color: var(--orange); border-color: var(--orange); }
|
||||
.tab.active { background: var(--grad-main); color: #fff; border-color: transparent; }
|
||||
|
||||
.empty {
|
||||
display: flex; flex-direction: column; align-items: center; gap: 8px;
|
||||
padding: 70px 0; color: var(--ink-faint);
|
||||
}
|
||||
.empty-emo { font-size: 42px; }
|
||||
.empty p { font-size: 14px; }
|
||||
|
||||
.table { overflow: auto; border: 1px solid var(--line); border-radius: 13px; background: var(--card); }
|
||||
table { width: 100%; border-collapse: collapse; font-size: 13.5px; min-width: 640px; }
|
||||
th, td { padding: 11px 16px; text-align: left; border-bottom: 1px solid var(--line); }
|
||||
th { font-size: 12px; color: var(--ink-faint); font-weight: 600; white-space: nowrap; }
|
||||
tbody tr:hover { background: var(--panel-soft); }
|
||||
td.num { text-align: right; font-family: var(--num); }
|
||||
td.dim { color: var(--ink-faint); }
|
||||
td.name { font-weight: 700; font-family: var(--num); }
|
||||
td.ver { color: var(--green-deep); font-family: var(--num); }
|
||||
td.vs { font-size: 12px; }
|
||||
td.no { text-align: center; color: var(--ink-faint); padding: 30px 0; }
|
||||
|
||||
.badge {
|
||||
display: inline-flex; align-items: center; gap: 5px; font-size: 11.5px; font-weight: 700;
|
||||
padding: 4px 11px; border-radius: 99px; white-space: nowrap;
|
||||
}
|
||||
.badge.r { background: var(--red-soft); color: var(--red); }
|
||||
.badge.y { background: var(--orange-soft); color: var(--orange); }
|
||||
.badge.g { background: var(--green-soft); color: var(--green-deep); }
|
||||
|
||||
@media (max-width: 767px) {
|
||||
.page { padding: 16px; }
|
||||
.actions { width: 100%; }
|
||||
.btn { flex: 1; }
|
||||
}
|
||||
</style>
|
||||
Reference in New Issue
Block a user