diff --git a/.gitignore b/.gitignore index cbfb2f3..6beb6c5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,8 @@ frontend/node_modules/ frontend/dist/ backend/node_modules/ .CodeCounter - +.dumate +.tmp # 运行时数据（SQLite） backend/data/ manager-data/ diff --git a/backend/server.js b/backend/server.js index 8dbdf85..6998996 100644 --- a/backend/server.js +++ b/backend/server.js @@ -27,6 +27,7 @@ import { } from './scanner.js' import { cleanPackage, deleteSingleVersion } from './clean.js' import semverCompare from './semver.js' +import { syncState, runSyncLatest } from './sync-latest.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) const app = express() @@ -402,6 +403,44 @@ app.get('/api/scan/progress', (req, res) => { }) }) +// —— API：一键拉取存量包最新版本（异步，期间前端轮询进度） —— +app.post('/api/sync/latest', (req, res) => { + if (isDemoMode()) { + return res.json({ ok: true, demo: true, running: false, message: '演示模式：未配置 Verdaccio 缓存目录，不支持拉取' }) + } + if (!syncState.running) { + runSyncLatest({ + addLog, + invalidateScan, + }) + } + res.json({ + ok: true, + demo: false, + running: syncState.running, + total: syncState.total, + startedAt: syncState.startedAt, + }) +}) + +// —— API：拉取进度（拉取期间由前端轮询） —— +app.get('/api/sync/progress', (req, res) => { + res.json({ + running: syncState.running, + done: syncState.done, + total: syncState.total, + processed: syncState.processed, + success: syncState.success, + skipped: syncState.skipped, + failed: syncState.failed, + failedList: syncState.failedList, + current: syncState.current, + error: syncState.error, + startedAt: syncState.startedAt, + finishedAt: syncState.finishedAt, + }) +}) + // —— API：清理 —— app.post('/api/clean', (req, res) => { const { pkgName, keep } = req.body || {} diff --git a/backend/sync-latest.js b/backend/sync-latest.js new file mode 100644 index 0000000..4277f1c --- /dev/null +++ b/backend/sync-latest.js @@ -0,0 +1,176 @@ +/** + * 存量包最新版本拉取 —— 一键把私服缓存中已有的包更新到上游最新版 + * + * 实现原理： + * 1. 查询：请求 Verdaccio 私服上该包的元数据（GET /:pkg），取 dist-tags.latest； + * Verdaccio 按自身 uplinks 配置（npm / 淘宝 / 华为等多源）路由到上游获取； + * 2. 比对：本地 storage 已缓存该精确版本则跳过（不重复拉取）； + * 3. 拉取：未缓存的版本请求 Verdaccio HTTP 端点（GET /:pkg/-/:file.tgz）， + * Verdaccio 作为代理会从 uplinks 拉取并把 tgz + 元数据写入自身存储， + * 元数据由 Verdaccio 自己维护，格式与扫描器读取的完全一致。 + * + * 环境变量： + * VERDACCIO_BASE_URL 私服地址（默认 http://127.0.0.1:4873；查询与代拉均走它，上游由 Verdaccio 自行路由） + */ +import { scanStorageAsync, resolveStorageDir } from './scanner.js' + +const VERDACCIO_BASE = (process.env.VERDACCIO_BASE_URL || 'http://127.0.0.1:4873').replace(/\/+$/, '') +const QUERY_TIMEOUT = 15000 // 元数据查询超时（Verdaccio 本地/uplink 转发通常较快） +const FETCH_TIMEOUT = 10 * 60_000 // tgz 拉取超时：大包（echarts 等）需要时间 +const CONCURRENT = 3 // 同时拉取的包数：网络 IO 温和并发，避免打满 NAS 带宽 + +// —— 拉取状态（前端轮询进度，与 scanState 模式对称） —— +export const syncState = { + running: false, done: false, + total: 0, processed: 0, + success: 0, skipped: 0, failed: 0, + failedList: [], // [{ name, reason }] + current: null, // 正在处理的包名 + error: null, + startedAt: null, finishedAt: null, +} + +function resetState() { + syncState.done = false + syncState.total = 0 + syncState.processed = 0 + syncState.success = 0 + syncState.skipped = 0 + syncState.failed = 0 + syncState.failedList = [] + syncState.current = null + syncState.error = null + syncState.startedAt = new Date().toISOString() +} + +// 编码包名：scoped 包 @scope/name → %40scope%2fname（registry / verdaccio 均接受该编码） +const enc = name => encodeURIComponent(name) +// tarball 文件名：@scope/name → name-version.tgz（Verdaccio 落盘命名不携带 scope 前缀） +const fileOf = (name, version) => `${name.split('/').pop()}-${version}.tgz` + +/** GET JSON，带超时；404 返回 null（包不存在），其余非 2xx 抛可读错误 */ +async function fetchJson(url, timeout) { + const ctrl = new AbortController() + const timer = setTimeout(() => ctrl.abort(), timeout) + try { + const r = await fetch(url, { signal: ctrl.signal, headers: { Accept: 'application/json' } }) + if (r.status === 404) return null + if (!r.ok) throw new Error(`私服响应 ${r.status}`) + return await r.json() + } catch (e) { + if (e && e.name === 'AbortError') throw new Error('请求超时') + throw e + } finally { + clearTimeout(timer) + } +} + +/** 触发 Verdaccio 代理拉取：走到该端点时 Verdaccio 会从 uplink 拉取并落盘缓存 */ +async function triggerVerdaccio(name, version) { + const url = `${VERDACCIO_BASE}/${enc(name)}/-/${enc(fileOf(name, version))}` + const ctrl = new AbortController() + const timer = setTimeout(() => ctrl.abort(), FETCH_TIMEOUT) + try { + const r = await fetch(url, { signal: ctrl.signal, headers: { accept: 'application/octet-stream' } }) + if (!r.ok) throw new Error(`私服响应 ${r.status}（请确认 VERDACCIO_BASE_URL 配置正确）`) + if (r.body) { + // 丢弃响应体：Verdaccio 已把包写进存储，本次只需等待传输完成 + for await (const _ of r.body) { /* drain */ } + } + } catch (e) { + if (e && e.name === 'AbortError') throw new Error('拉取超时') + throw e + } finally { + clearTimeout(timer) + } +} + +/** 本地是否已缓存该精确版本（versionsDetail 缺失时保守视为未缓存，交给 Verdaccio 层幂等判断） */ +function hasLocalVersion(pkg, version) { + const vs = pkg && pkg.versionsDetail + if (!Array.isArray(vs)) return false + return vs.some(v => String(v.version) === version) +} + +async function handleOne(pkg) { + try { + const meta = await fetchJson(`${VERDACCIO_BASE}/${enc(pkg.name)}`, QUERY_TIMEOUT) + const latest = meta && meta['dist-tags'] && meta['dist-tags'].latest + if (!meta) { + syncState.failed += 1 + syncState.failedList.push({ name: pkg.name, reason: '私服无此包（404）' }) + return + } + if (!latest) { + syncState.failed += 1 + syncState.failedList.push({ name: pkg.name, reason: '私服未提供 latest 版本' }) + return + } + if (hasLocalVersion(pkg, latest)) { + syncState.skipped += 1 // 已是最新，不重复拉取 + return + } + await triggerVerdaccio(pkg.name, latest) + syncState.success += 1 + } catch (e) { + syncState.failed += 1 + syncState.failedList.push({ name: pkg.name, reason: (e && e.message) || '拉取失败' }) + } +} + +/** + * 启动拉取任务（异步；running 时重复调用直接返回） + * @param {{ addLog: Function, invalidateScan: Function }} services 依赖注入（避免与 server.js 循环引用） + */ +export async function runSyncLatest({ addLog, invalidateScan }) { + if (syncState.running) return + resetState() + syncState.running = true + console.log(`[sync-latest] 开始拉取存量包最新版本（经私服: ${VERDACCIO_BASE}，上游按 Verdaccio uplinks 路由）`) + try { + // 存量包列表：优先直接扫描 storage（与面板列表同源，且不依赖接口层扫描缓存是否就绪） + let pkgs = [] + try { + pkgs = await scanStorageAsync(resolveStorageDir()) + } catch (e) { + syncState.error = `扫描缓存目录失败：${e.message}` + } + // 跳过异常包（元数据读取失败无法判断版本），其余全部纳入（含忽略中的包，它们同样是存量缓存） + pkgs = (pkgs || []).filter(p => p && p.name && p.status !== 'broken') + syncState.total = pkgs.length + if (syncState.total === 0) { + if (!syncState.error) syncState.error = '缓存目录为空，无可拉取的包' + } else { + let next = 0 + const worker = async () => { + while (next < pkgs.length) { + const pkg = pkgs[next++] + syncState.current = pkg.name + await handleOne(pkg) + syncState.processed += 1 + } + } + await Promise.all(Array.from({ length: Math.min(CONCURRENT, pkgs.length) }, worker)) + } + // 汇总写操作日志（失败包名列表存 deletedVersions，前端日志详情展示） + addLog({ + action: 'sync', + pkgName: null, + deletedCount: syncState.success, + deletedVersions: JSON.stringify({ + success: syncState.success, + skipped: syncState.skipped, + failed: syncState.failedList, + }), + }) + invalidateScan() // 有新版本入库 → 废弃扫描缓存，列表下次请求重扫 + console.log(`[sync-latest] 完成：成功 ${syncState.success} · 跳过 ${syncState.skipped} · 失败 ${syncState.failed}`) + } catch (e) { + syncState.error = e.message + console.error('[sync-latest] 任务异常:', e.message) + } finally { + syncState.running = false + syncState.done = true + syncState.finishedAt = new Date().toISOString() + } +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index ae6daaf..254bf8e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ services: user: "0:0" # root：清理需写 verdaccio 卷；vcm-data 命名卷 root 亦可写 environment: - VERDACCIO_STORAGE=/verdaccio/storage # 容器内路径，对应下方挂载的右侧 + - VERDACCIO_BASE_URL=${VERDACCIO_BASE_URL:-http://218.61.196.156:11691} # 私服地址：查询最新版与代拉均走它（容器内不能用 127.0.0.1；上游由 Verdaccio 自身 uplinks 路由） - PORT=3000 - TZ=Asia/Shanghai # 每日 00:00 自动扫描按北京时间执行 - VCM_USERNAME=${VCM_USERNAME:-admin} # 面板登录账号（可在宿主机 .env 覆盖） diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 22bce1c..29748f6 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -178,7 +178,7 @@ async login(username, password, remember = true) { return r.json() }, - /** 查询扫描进度（包数 / 字节数 / 是否完成） */ +/** 查询扫描进度（包数 / 字节数 / 是否完成） */ async getScanProgress() { if (USE_MOCK) { return { scanning: false, done: true, packages: 33, bytes: 0 } @@ -187,6 +187,33 @@ async login(username, password, remember = true) { return r.json() }, + /** 一键拉取存量包最新版本（异步，触发后轮询进度） */ + async syncLatest() { + if (USE_MOCK) { + await delay(260) + return { ok: true, demo: false, running: false, total: mockState.packages.length, startedAt: new Date().toISOString() } + } + const r = await authFetch('/api/sync/latest', { method: 'POST' }) + return r.json() + }, + + /** 查询拉取进度（成功/跳过/失败汇总 + 失败明细） */ + async getSyncProgress() { + if (USE_MOCK) { + await delay(60) + // mock：模拟一次已完成的拉取（少量包有新版本被拉取，其余已是最新） + return { + running: false, done: true, + total: mockState.packages.length, processed: mockState.packages.length, + success: 2, skipped: Math.max(0, mockState.packages.length - 2), failed: 0, + failedList: [], current: null, error: null, + startedAt: null, finishedAt: new Date().toISOString(), + } + } + const r = await authFetch('/api/sync/progress') + return r.json() + }, + async getThresholds() { if (USE_MOCK) { await delay(80) diff --git a/frontend/src/store/index.js b/frontend/src/store/index.js index 3e03192..e2ef6c3 100644 --- a/frontend/src/store/index.js +++ b/frontend/src/store/index.js @@ -17,8 +17,13 @@ const state = reactive({ storageDir: storage.get('vcm-storage-dir') || '', // 后端目录扫描总字节（所有文件，与 du 同口径；null 表示未知/演示模式） totalBytes: null, - // 扫描进度（异步扫描期间实时更新，用于遮罩展示） +// 扫描进度（异步扫描期间实时更新，用于遮罩展示） scanProgress: { packages: 0, bytes: 0 }, + // 拉取进度（一键拉取存量包最新版本期间实时更新） + syncProgress: { + running: false, done: false, total: 0, processed: 0, + success: 0, skipped: 0, failed: 0, failedList: [], error: null, + }, }) // —— 派生统计 —— @@ -137,6 +142,67 @@ async function scanNow() { } } +let syncPollTimer = null + +/** 重置拉取进度状态（触发新一轮拉取时调用） */ +function resetSyncProgress(res) { + state.syncProgress.running = true + state.syncProgress.done = false + state.syncProgress.total = res.total || 0 + state.syncProgress.processed = 0 + state.syncProgress.success = 0 + state.syncProgress.skipped = 0 + state.syncProgress.failed = 0 + state.syncProgress.failedList = [] + state.syncProgress.error = null +} + +/** 一键拉取存量包最新版本：触发后端任务并轮询进度，完成后刷新列表 */ +async function syncLatest() { + try { + const res = await api.syncLatest() + if (!res.ok) return res + if (res.demo) return res // 演示模式：后端不支持拉取，由调用方直接提示 + resetSyncProgress(res) + pollSyncProgress() + return res + } catch (e) { + ElMessage.error(e?.message || '发起拉取失败，请稍后重试') + return { ok: false } + } +} + +/** 轮询拉取进度，任务结束后刷新包列表与操作日志（新版本已入库） */ +function pollSyncProgress() { + clearTimeout(syncPollTimer) + const tick = async () => { + let p + try { + p = await api.getSyncProgress() + } catch { + state.syncProgress.running = false + state.syncProgress.error = '拉取进度查询失败' + return + } + state.syncProgress.running = !!p.running + state.syncProgress.done = !!p.done + state.syncProgress.total = p.total || 0 + state.syncProgress.processed = p.processed || 0 + state.syncProgress.success = p.success || 0 + state.syncProgress.skipped = p.skipped || 0 + state.syncProgress.failed = p.failed || 0 + state.syncProgress.failedList = p.failedList || [] + state.syncProgress.error = p.error || null + if (p.running) { + syncPollTimer = setTimeout(tick, 800) + } else { + clearTimeout(syncPollTimer) + await Promise.all([refreshPackages(), refreshLogs()]) + } + } + tick() +} + async function refreshThresholds() { const t = await api.getThresholds() state.threshold.versionThreshold = t.versionThreshold @@ -201,7 +267,8 @@ export function useStore() { refreshPackages, refreshThresholds, refreshLogs, - scanNow, +scanNow, + syncLatest, saveThresholds, ignorePkg, unignorePkg, diff --git a/frontend/src/views/LogsView.vue b/frontend/src/views/LogsView.vue index e9b93d9..76371b9 100644 --- a/frontend/src/views/LogsView.vue +++ b/frontend/src/views/LogsView.vue @@ -15,8 +15,18 @@ function fmtTime(t) { function actionMeta(action) { if (action === 'ignore') return { emo: '💤', text: '忽略', cls: 'ignore' } if (action === 'unignore') return { emo: '↩️', text: '取消忽略', cls: 'unignore' } + if (action === 'sync') return { emo: '📥', text: '拉取最新版', cls: 'sync' } return { emo: '🗑', text: '清理', cls: 'clean' } } + +// sync 日志的汇总明细（addLog 时存于 deletedVersions JSON 字段） +function syncMeta(log) { + try { + return JSON.parse(log.deletedVersions || '{}') + } catch { + return {} + } +} +
说明 {{ detail.action === 'ignore' ? '已将包加入忽略列表，不再提示超阈值警告' : '已取消忽略，恢复超阈值提醒' }} @@ -133,6 +155,7 @@ function actionMeta(action) { .act-badge.clean { color: var(--red); background: var(--red-soft); } .act-badge.ignore { color: var(--ink-dim); background: var(--num-bg); } .act-badge.unignore { color: var(--green-deep); background: var(--green-soft); } +.act-badge.sync { color: var(--blue); background: var(--blue-soft); } .chip { font-family: var(--num); font-size: 12px; font-weight: 700; padding: 4px 12px; border-radius: 9px; display: inline-block; } .chip.n { color: var(--red); background: var(--red-soft); } .chip.s { color: var(--green-deep); background: var(--green-soft); } diff --git a/frontend/src/views/SearchView.vue b/frontend/src/views/SearchView.vue index 9add696..2823fac 100644 --- a/frontend/src/views/SearchView.vue +++ b/frontend/src/views/SearchView.vue @@ -8,7 +8,7 @@ import PkgLogo from '../components/PkgLogo.vue' import { useStore, fmtSize } from '../store' import { storage } from '../utils/storage' -const { state, stats, refreshPackages, ignorePkg, unignorePkg, cleanPkg, deleteVersion } = useStore() +const { state, stats, refreshPackages, ignorePkg, unignorePkg, cleanPkg, deleteVersion, syncLatest } = useStore() const route = useRoute() const router = useRouter() @@ -173,6 +173,54 @@ async function onCleanConfirmed(res) { ElMessage.success(`已清理 ${res.deletedCount} 个版本，释放 ${res.freedSpace}`) drawerVisible.value = false } + +// —— 一键拉取存量包最新版本 —— +const summaryShown = ref(false) + +async function doSyncLatest() { + const sp = state.syncProgress + if (sp.running) return + try { + await ElMessageBox.confirm( + '将查询上游 npm registry，为存量包拉取每个包的最新版本（已缓存最新版会自动跳过）。', + '拉取最新版本', + { type: 'info', confirmButtonText: '开始拉取', cancelButtonText: '取消', closeOnClickModal: false }, + ) + } catch { + return // 用户取消 + } + summaryShown.value = false + const res = await syncLatest() + if (!res.ok) return + if (res.demo) { + ElMessage.info(res.message || '演示模式：不支持拉取最新版本') + return + } + // 任务异步执行，完成后由下方 watch 弹出汇总 +} + +watch( + () => state.syncProgress.running, + async (running, prev) => { + if (prev === true && running === false && state.syncProgress.done && !summaryShown.value) { + summaryShown.value = true + const sp = state.syncProgress + const detail = [`本次共检查 ${sp.total} 个存量包：`, `✅ 已拉取最新版本 ${sp.success} 个`, `⏭️ 已是最新跳过 ${sp.skipped} 个`] + if (sp.failed) { + detail.push(`❌ 拉取失败 ${sp.failed} 个`) + if (sp.failedList.length) detail.push(`失败明细：\n${sp.failedList.map(f => `· ${f}`).join('\n')}`) + } + try { + await ElMessageBox.alert(detail.join('\n'), sp.failed ? '拉取完成（部分失败）' : '拉取完成', { + confirmButtonText: '知道了', + type: sp.failed ? 'warning' : 'success', + }) + } catch { + /* 用户关闭弹窗无需处理 */ + } + } + }, +)