/** * 按 npm 包清单拉取 —— 以私服 VERDACCIO_BASE_URL 为镜像源(上游按 Verdaccio uplinks 路由) * * 与 sync-latest.js 的差异: * sync-latest:存量包查最新版(跳过已缓存版本) * 本模块:按清单逐包拉取(清单即待拉取目标;版本为空时解析 dist-tags.latest) * * 拉取流程(异步状态机,前端轮询进度): * 1. 取清单全部 status='pending' 条目作为目标; * 2. 首轮拉取全部; * 3. 失败的包自动连续重试 3 次(round 1..3),每轮只处理上一轮失败项; * 4. 重试 3 次后仍失败的条目保持在 failedList(状态仍 pending,供角标展示与下次重拉), * 其余成功条目置 status='done'; * 5. 全部成功后清单失效:pending 为 0 → 手动拉取提示「当前无可用npm包」、 * 定时拉取自动停止,直至清单写入新包。 * * 环境变量: * VERDACCIO_BASE_URL 私服地址(默认为 sync-latest 同款缺省值;查询与代拉均走它) */ import { getPendingRows, markDone, countPending, getPkgList } from './pkglist.js' const VERDACCIO_BASE = (process.env.VERDACCIO_BASE_URL || 'http://127.0.0.1:4873').replace(/\/+$/, '') const QUERY_TIMEOUT = 15000 // 元数据查询超时 const FETCH_TIMEOUT = 10 * 60_000 // tgz 拉取超时(大包需要时间) const CONCURRENT = 3 // 同时拉取的包数 const MAX_RETRY = 3 // 失败自动连续重试次数 // —— 拉取状态(前端轮询进度;与 syncState 对称,另含轮次信息) —— export const pkgPullState = { running: false, done: false, total: 0, processed: 0, success: 0, failed: 0, failedList: [], // [{ name, reason }] round: 0, // 当前轮次 0=首轮, 1..3=重试轮 current: null, // 正在处理的包名 source: 'manual', // manual | auto(定时) error: null, startedAt: null, finishedAt: null, } function resetState(source) { pkgPullState.done = false pkgPullState.total = 0 pkgPullState.processed = 0 pkgPullState.success = 0 pkgPullState.failed = 0 pkgPullState.failedList = [] pkgPullState.round = 0 pkgPullState.current = null pkgPullState.source = source pkgPullState.error = null pkgPullState.startedAt = new Date().toISOString() } // 编码包名:scoped 包 @scope/name → %40scope%2fname const enc = name => encodeURIComponent(name) // tarball 文件名:@scope/name → name-version.tgz(Verdaccio 落盘命名不携带 scope 前缀) const fileOf = (name, version) => `${name.split('/').pop()}-${version}.tgz` /** GET JSON,带超时;404 返回 null,其余非 2xx 抛可读错误 */ async function fetchJson(url, timeout) { const ctrl = new AbortController() const timer = setTimeout(() => ctrl.abort(), timeout) try { const r = await fetch(url, { signal: ctrl.signal, headers: { Accept: 'application/json' } }) if (r.status === 404) return null if (!r.ok) throw new Error(`私服响应 ${r.status}`) return await r.json() } catch (e) { if (e && e.name === 'AbortError') throw new Error('请求超时') throw e } finally { clearTimeout(timer) } } /** 触发 Verdaccio 代理拉取:走到该端点时 Verdaccio 会从 uplink 拉取并落盘缓存 */ async function triggerVerdaccio(name, version) { const url = `${VERDACCIO_BASE}/${enc(name)}/-/${enc(fileOf(name, version))}` const ctrl = new AbortController() const timer = setTimeout(() => ctrl.abort(), FETCH_TIMEOUT) try { const r = await fetch(url, { signal: ctrl.signal, headers: { accept: 'application/octet-stream' } }) if (!r.ok) throw new Error(`私服响应 ${r.status}(请确认 VERDACCIO_BASE_URL 配置正确)`) if (r.body) { // 丢弃响应体:Verdaccio 已把包写进存储,本次只需等待传输完成 for await (const _ of r.body) { /* drain */ } } } catch (e) { if (e && e.name === 'AbortError') throw new Error('拉取超时') throw e } finally { clearTimeout(timer) } } /** * 拉取单个清单条目:版本为空 → 解析私服 dist-tags.latest(永远最新) * 成功返回 true;失败抛错(由上层收集 reason) */ async function pullOne(item) { let version = item.version if (!version) { const meta = await fetchJson(`${VERDACCIO_BASE}/${enc(item.pkg_name)}`, QUERY_TIMEOUT) if (!meta || !meta['dist-tags']) throw new Error('私服无此包或未提供元数据(404)') version = meta['dist-tags'].latest if (!version) throw new Error('私服未提供 latest 版本') } await triggerVerdaccio(item.pkg_name, version) return true } /** * 启动按清单拉取(同步启动,立即返回;拉取在后台异步执行,前端轮询进度) * - running 时重复调用直接返回 started:false * - 清单失效(无 pending)→ started:false,active:false(手动由路由层提示,定时自动停止) * @param {{ addLog: Function, invalidateScan: Function, source?: 'manual'|'auto' }} opts * @returns {{ started: boolean, active: boolean, pendingCount: number }} */ export function runPkgListPull({ addLog, invalidateScan, source = 'manual' }) { if (pkgPullState.running) return { started: false, active: countPending() > 0, pendingCount: countPending() } const pending0 = countPending() if (pending0 === 0) { return { started: false, active: false, pendingCount: 0 } } resetState(source) pkgPullState.running = true console.log(`[pkglist-pull] 开始按清单拉取(source=${source},共 ${pending0} 个待拉取包,经私服 ${VERDACCIO_BASE})`) // 后台执行(不阻塞请求/事件循环主路径),结果通过 pkgPullState 暴露 ;(async () => { try { // 目标集合:先取一次 pending 快照;每成功一个置 done 并计数 let targets = getPendingRows() pkgPullState.total = targets.length // —— 首轮 + 重试轮:轮次 0 处理全部,轮次 1..3 仅处理上一轮失败项 —— for (let round = 0; round <= MAX_RETRY && targets.length > 0; round++) { pkgPullState.round = round pkgPullState.failed = 0 pkgPullState.failedList = [] const currentTargets = [...targets] let next = 0 const worker = async () => { while (next < currentTargets.length) { const item = currentTargets[next++] pkgPullState.current = item.pkg_name try { await pullOne(item) pkgPullState.success += 1 markDone(item.id) // 成功 → 清单 side done } catch (e) { pkgPullState.failed += 1 pkgPullState.failedList.push({ name: item.pkg_name, reason: (e && e.message) || '拉取失败' }) } pkgPullState.processed += 1 } } await Promise.all(Array.from({ length: Math.min(CONCURRENT, currentTargets.length) }, worker)) // 本轮结束:失败项作为下一轮重试目标(重试前先等所有 worker 完成) targets = getPendingRows().filter(t => pkgPullState.failedList.some(f => f.name === t.pkg_name)) if (round < MAX_RETRY && targets.length > 0) { console.log(`[pkglist-pull] 第 ${round + 1} 轮结束:失败 ${targets.length} 个,进入第 ${round + 2} 轮重试`) } } // —— 汇总:失败条目保持 pending(下次可再拉),其余已 done —— // 注意:循环结束后的 pkgPullState.failedList 即最后一轮失败项的真实 reason //(若全部成功则为空数组),直接保留即可,不要用统一文案覆盖。 if (!pkgPullState.failedList.length) { pkgPullState.failed = 0 } pkgPullState.round = Math.min(pkgPullState.round, MAX_RETRY) // 标记已尽力完成重试 pkgPullState.current = null addLog({ action: 'list-pull', pkgName: null, deletedCount: pkgPullState.success, deletedVersions: JSON.stringify({ source, round: pkgPullState.round, total: pkgPullState.total, success: pkgPullState.success, failed: pkgPullState.failedList, }), }) invalidateScan() // 有新包入库 → 废弃扫描缓存,列表下次请求重扫 console.log(`[pkglist-pull] 完成(source=${source}):成功 ${pkgPullState.success} · 失败 ${pkgPullState.failed}`) } catch (e) { pkgPullState.error = e.message console.error('[pkglist-pull] 任务异常:', e.message) } finally { pkgPullState.running = false pkgPullState.done = true pkgPullState.finishedAt = new Date().toISOString() } })() // 立即执行(异步,不阻塞调用方) return { started: true, active: true, pendingCount: pending0 } } export { VERDACCIO_BASE as PKGLIST_VERDACCIO_BASE }