Files
verdaccio-cache-manager/backend/db.js
T
yinjun.chen1 b0f8a60d6f feat(toplist): 新增 npm Top200 周榜缓存功能
- db.js: 新增 top_list 表 + replaceTopList/getTopListRows/getTopListCount
- npm-toplist.js: 核心模块(Search+Downloads 复核+缓存比对+经私服拉取)
- server.js: /api/toplist* 端点 + scheduleWeeklyTopList(周一 00:00 定时抓榜)
- 前端: TopListView 页面、路由、顶/底栏菜单、store/api 扩展

实现: 22 关键词 Search 粗排 350 → Downloads 复核截 200 → 与 rawCache 三态比对(待拉取/需更新/已就绪)→ 一键拉取待缓存/需更新包
复用 sync-latest 的 triggerVerdaccio 落盘机制,限流 1.5s 间隔 + 429 指数退避
2026-09-22 10:50:43 +08:00

204 lines
7.7 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* SQLite 数据层(§4.1)
* 表:settings / ignored_packages / logs
*/
import Database from 'better-sqlite3'
import fs from 'node:fs'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
// 数据目录:环境变量 DATA_DIR 可覆盖(Docker 挂载 manager-data)
const DATA_DIR = process.env.DATA_DIR || path.join(__dirname, 'data')
fs.mkdirSync(DATA_DIR, { recursive: true })
const db = new Database(path.join(DATA_DIR, 'manager.db'))
db.pragma('journal_mode = WAL')
db.exec(`
CREATE TABLE IF NOT EXISTS settings (
key TEXT PRIMARY KEY,
value TEXT
);
CREATE TABLE IF NOT EXISTS ignored_packages (
pkg_name TEXT PRIMARY KEY,
ignored_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS logs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
action TEXT DEFAULT 'clean',
pkg_name TEXT,
deleted_versions TEXT,
deleted_count INTEGER,
freed_space TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS top_list (
id INTEGER PRIMARY KEY AUTOINCREMENT,
rank INTEGER NOT NULL,
pkg_name TEXT NOT NULL UNIQUE,
latest_version TEXT NOT NULL,
weekly_downloads INTEGER NOT NULL DEFAULT 0,
search_weekly INTEGER DEFAULT 0,
keyword TEXT,
fetched_at DATETIME
);
CREATE INDEX IF NOT EXISTS idx_top_list_rank ON top_list(rank);
`)
// 北京时间(Asia/Shanghai 固定 UTC+8,无夏令时):SQLite DEFAULT CURRENT_TIMESTAMP 是 UTC,
// 与前端按本地时间展示相差 8 小时,故写入与迁移统一显式使用北京时间墙上时间字符串
// (格式与前端 slice 兼容:YYYY-MM-DD HH:MM:SS,不带时区后缀,避免被二次误解)
const pad2 = n => String(n).padStart(2, '0')
export function nowIso() {
const d = new Date(Date.now() + 8 * 3600_000)
return `${d.getUTCFullYear()}-${pad2(d.getUTCMonth() + 1)}-${pad2(d.getUTCDate())} ${pad2(d.getUTCHours())}:${pad2(d.getUTCMinutes())}:${pad2(d.getUTCSeconds())}`
}
function utcToShanghai(iso) {
try {
const ts = Date.parse(iso)
if (!Number.isFinite(ts)) return null
const d = new Date(ts + 8 * 3600_000)
return `${d.getUTCFullYear()}-${pad2(d.getUTCMonth() + 1)}-${pad2(d.getUTCDate())} ${pad2(d.getUTCHours())}:${pad2(d.getUTCMinutes())}:${pad2(d.getUTCSeconds())}`
} catch {
return null
}
}
// 旧库迁移:早期 logs 表无 action 列,运行时补齐(忽略/取消忽略等操作依赖该列区分类型)
const logCols = db.prepare('PRAGMA table_info(logs)').all().map(c => c.name)
if (!logCols.includes('action')) {
db.exec("ALTER TABLE logs ADD COLUMN action TEXT DEFAULT 'clean'")
}
// 旧库迁移 v1:历史日志/忽略记录的时间戳是 UTC,统一转成北京时间;settings 标志位防重复执行
function migrateTimezone() {
if (getSetting('timezone_migrated_v1')) return
db.transaction(() => {
for (const { id, created_at: t } of db.prepare('SELECT id, created_at FROM logs').all()) {
if (t) {
const local = utcToShanghai(t)
if (local) db.prepare('UPDATE logs SET created_at = ? WHERE id = ?').run(local, id)
}
}
for (const { pkg_name: n, ignored_at: t } of db.prepare('SELECT pkg_name, ignored_at FROM ignored_packages').all()) {
if (t) {
const local = utcToShanghai(t)
if (local) db.prepare('UPDATE ignored_packages SET ignored_at = ? WHERE pkg_name = ?').run(local, n)
}
}
setSetting('timezone_migrated_v1', '1')
})()
}
// —— settings ——
export function getSetting(key, def = null) {
const row = db.prepare('SELECT value FROM settings WHERE key = ?').get(key)
return row ? row.value : def
}
export function setSetting(key, value) {
db.prepare(
'INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value',
).run(key, String(value))
}
// —— ignored ——
export function getIgnored() {
return db.prepare('SELECT pkg_name FROM ignored_packages ORDER BY ignored_at DESC').all().map(r => r.pkg_name)
}
export function getIgnoredRows() {
return db.prepare('SELECT pkg_name, ignored_at FROM ignored_packages ORDER BY ignored_at DESC').all()
}
export function isIgnored(pkgName) {
return !!db.prepare('SELECT 1 FROM ignored_packages WHERE pkg_name = ?').get(pkgName)
}
export function addIgnored(pkgName) {
db.prepare(
'INSERT INTO ignored_packages (pkg_name, ignored_at) VALUES (?, ?) ON CONFLICT(pkg_name) DO NOTHING',
).run(pkgName, nowIso())
}
export function removeIgnored(pkgName) {
db.prepare('DELETE FROM ignored_packages WHERE pkg_name = ?').run(pkgName)
}
// 旧库迁移 v2:logs 表扩展 6 个字段承载 6 类操作日志(登录/扫描/批量逐包/手工拉取/榜单拉取),幂等
function migrateLogsV2() {
if (getSetting('logs_v2_migrated')) return
const cols = db.prepare('PRAGMA table_info(logs)').all().map(c => c.name)
const adds = {
version: 'ALTER TABLE logs ADD COLUMN version TEXT', // 单版本操作目标版本
result: 'ALTER TABLE logs ADD COLUMN result TEXT', // 'success' | 'fail' | 'skipped'
message: 'ALTER TABLE logs ADD COLUMN message TEXT', // 详细信息/失败原因
duration_ms: 'ALTER TABLE logs ADD COLUMN duration_ms INTEGER', // 任务耗时(扫描/批量拉取汇总)
meta: 'ALTER TABLE logs ADD COLUMN meta TEXT', // JSON 扩展字段
ip: 'ALTER TABLE logs ADD COLUMN ip TEXT', // 登录 IP
}
for (const [col, sql] of Object.entries(adds)) {
if (!cols.includes(col)) db.exec(sql)
}
setSetting('logs_v2_migrated', '1')
}
// —— logs ——
export function addLog({
action = 'clean', pkgName, deletedVersions = null, deletedCount = 0, freedSpace = null,
version = null, result = null, message = null, durationMs = null, meta = null, ip = null,
} = {}) {
const info = db.prepare(
`INSERT INTO logs (action, pkg_name, deleted_versions, deleted_count, freed_space,
version, result, message, duration_ms, meta, ip, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
).run(action, pkgName, deletedVersions, deletedCount, freedSpace,
version, result, message, durationMs, meta ? JSON.stringify(meta) : null, ip, nowIso())
return info.lastInsertRowid
}
export function getLogActions() {
return db.prepare('SELECT DISTINCT action FROM logs ORDER BY action').all().map(r => r.action)
}
export function getLogs({ action = null, limit = 50 } = {}) {
// 兼容旧调用 getLogs(50):首参为数字时视为旧版 limit
if (typeof action === 'number') {
return db.prepare('SELECT * FROM logs ORDER BY id DESC LIMIT ?').all(action)
}
if (action) {
return db.prepare('SELECT * FROM logs WHERE action = ? ORDER BY id DESC LIMIT ?').all(action, limit)
}
return db.prepare('SELECT * FROM logs ORDER BY id DESC LIMIT ?').all(limit)
}
// —— top_list ——
const insertTop = db.prepare(
`INSERT OR REPLACE INTO top_list (rank, pkg_name, latest_version, weekly_downloads, search_weekly, keyword, fetched_at)
VALUES (?, ?, ?, ?, ?, ?, ?)`,
)
export function replaceTopList(rows) {
db.transaction(rows => {
db.prepare('DELETE FROM top_list').run()
for (const r of rows || []) {
insertTop.run(r.rank, r.pkg_name, r.latest_version, r.weekly_downloads || 0, r.search_weekly || 0, r.keyword || null, nowIso())
}
})(rows)
}
export function getTopListRows() {
return db.prepare('SELECT * FROM top_list ORDER BY rank').all()
}
export function getTopListCount() {
return db.prepare('SELECT COUNT(*) AS c FROM top_list').get().c
}
// 模块加载时执行时间迁移 + logs v2 字段迁移(均幂等)
migrateTimezone()
migrateLogsV2()
export default db