diff --git a/backend/db.js b/backend/db.js index 0a5d9cc..06b4199 100644 --- a/backend/db.js +++ b/backend/db.js @@ -33,6 +33,15 @@ CREATE TABLE IF NOT EXISTS logs ( freed_space TEXT, created_at DATETIME DEFAULT CURRENT_TIMESTAMP ); +CREATE TABLE IF NOT EXISTS pkglist ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + pkg_name TEXT NOT NULL UNIQUE, + version TEXT, + category TEXT DEFAULT '', + status TEXT DEFAULT 'pending', + auto INTEGER DEFAULT 0, + created_at DATETIME +); `) // 北京时间（Asia/Shanghai 固定 UTC+8，无夏令时）：SQLite DEFAULT CURRENT_TIMESTAMP 是 UTC， diff --git a/backend/pkglist-pull.js b/backend/pkglist-pull.js new file mode 100644 index 0000000..1e95bf1 --- /dev/null +++ b/backend/pkglist-pull.js @@ -0,0 +1,202 @@ +/** + * 按 npm 包清单拉取 —— 以私服 VERDACCIO_BASE_URL 为镜像源（上游按 Verdaccio uplinks 路由） + * + * 与 sync-latest.js 的差异： + * sync-latest：存量包查最新版（跳过已缓存版本） + * 本模块：按清单逐包拉取（清单即待拉取目标；版本为空时解析 dist-tags.latest） + * + * 拉取流程（异步状态机，前端轮询进度）： + * 1. 取清单全部 status='pending' 条目作为目标； + * 2. 首轮拉取全部； + * 3. 失败的包自动连续重试 3 次（round 1..3），每轮只处理上一轮失败项； + * 4. 重试 3 次后仍失败的条目保持在 failedList（状态仍 pending，供角标展示与下次重拉）， + * 其余成功条目置 status='done'； + * 5. 全部成功后清单失效：pending 为 0 → 手动拉取提示「当前无可用npm包」、 + * 定时拉取自动停止，直至清单写入新包。 + * + * 环境变量： + * VERDACCIO_BASE_URL 私服地址（默认为 sync-latest 同款缺省值；查询与代拉均走它） + */ +import { getPendingRows, markDone, countPending, getPkgList } from './pkglist.js' + +const VERDACCIO_BASE = (process.env.VERDACCIO_BASE_URL || 'http://127.0.0.1:4873').replace(/\/+$/, '') +const QUERY_TIMEOUT = 15000 // 元数据查询超时 +const FETCH_TIMEOUT = 10 * 60_000 // tgz 拉取超时（大包需要时间） +const CONCURRENT = 3 // 同时拉取的包数 +const MAX_RETRY = 3 // 失败自动连续重试次数 + +// —— 拉取状态（前端轮询进度；与 syncState 对称，另含轮次信息） —— +export const pkgPullState = { + running: false, done: false, + total: 0, processed: 0, + success: 0, failed: 0, + failedList: [], // [{ name, reason }] + round: 0, // 当前轮次 0=首轮, 1..3=重试轮 + current: null, // 正在处理的包名 + source: 'manual', // manual | auto（定时） + error: null, + startedAt: null, finishedAt: null, +} + +function resetState(source) { + pkgPullState.done = false + pkgPullState.total = 0 + pkgPullState.processed = 0 + pkgPullState.success = 0 + pkgPullState.failed = 0 + pkgPullState.failedList = [] + pkgPullState.round = 0 + pkgPullState.current = null + pkgPullState.source = source + pkgPullState.error = null + pkgPullState.startedAt = new Date().toISOString() +} + +// 编码包名：scoped 包 @scope/name → %40scope%2fname +const enc = name => encodeURIComponent(name) +// tarball 文件名：@scope/name → name-version.tgz（Verdaccio 落盘命名不携带 scope 前缀） +const fileOf = (name, version) => `${name.split('/').pop()}-${version}.tgz` + +/** GET JSON，带超时；404 返回 null，其余非 2xx 抛可读错误 */ +async function fetchJson(url, timeout) { + const ctrl = new AbortController() + const timer = setTimeout(() => ctrl.abort(), timeout) + try { + const r = await fetch(url, { signal: ctrl.signal, headers: { Accept: 'application/json' } }) + if (r.status === 404) return null + if (!r.ok) throw new Error(`私服响应 ${r.status}`) + return await r.json() + } catch (e) { + if (e && e.name === 'AbortError') throw new Error('请求超时') + throw e + } finally { + clearTimeout(timer) + } +} + +/** 触发 Verdaccio 代理拉取：走到该端点时 Verdaccio 会从 uplink 拉取并落盘缓存 */ +async function triggerVerdaccio(name, version) { + const url = `${VERDACCIO_BASE}/${enc(name)}/-/${enc(fileOf(name, version))}` + const ctrl = new AbortController() + const timer = setTimeout(() => ctrl.abort(), FETCH_TIMEOUT) + try { + const r = await fetch(url, { signal: ctrl.signal, headers: { accept: 'application/octet-stream' } }) + if (!r.ok) throw new Error(`私服响应 ${r.status}（请确认 VERDACCIO_BASE_URL 配置正确）`) + if (r.body) { + // 丢弃响应体：Verdaccio 已把包写进存储，本次只需等待传输完成 + for await (const _ of r.body) { /* drain */ } + } + } catch (e) { + if (e && e.name === 'AbortError') throw new Error('拉取超时') + throw e + } finally { + clearTimeout(timer) + } +} + +/** + * 拉取单个清单条目：版本为空 → 解析私服 dist-tags.latest（永远最新） + * 成功返回 true；失败抛错（由上层收集 reason） + */ +async function pullOne(item) { + let version = item.version + if (!version) { + const meta = await fetchJson(`${VERDACCIO_BASE}/${enc(item.pkg_name)}`, QUERY_TIMEOUT) + if (!meta || !meta['dist-tags']) throw new Error('私服无此包或未提供元数据（404）') + version = meta['dist-tags'].latest + if (!version) throw new Error('私服未提供 latest 版本') + } + await triggerVerdaccio(item.pkg_name, version) + return true +} + +/** + * 启动按清单拉取（同步启动，立即返回；拉取在后台异步执行，前端轮询进度） + * - running 时重复调用直接返回 started:false + * - 清单失效（无 pending）→ started:false，active:false（手动由路由层提示，定时自动停止） + * @param {{ addLog: Function, invalidateScan: Function, source?: 'manual'|'auto' }} opts + * @returns {{ started: boolean, active: boolean, pendingCount: number }} + */ +export function runPkgListPull({ addLog, invalidateScan, source = 'manual' }) { + if (pkgPullState.running) return { started: false, active: countPending() > 0, pendingCount: countPending() } + const pending0 = countPending() + if (pending0 === 0) { + return { started: false, active: false, pendingCount: 0 } + } + + resetState(source) + pkgPullState.running = true + console.log(`[pkglist-pull] 开始按清单拉取（source=${source}，共 ${pending0} 个待拉取包，经私服 ${VERDACCIO_BASE}）`) + // 后台执行（不阻塞请求/事件循环主路径），结果通过 pkgPullState 暴露 + ;(async () => { + try { + // 目标集合：先取一次 pending 快照；每成功一个置 done 并计数 + let targets = getPendingRows() + pkgPullState.total = targets.length + + // —— 首轮 + 重试轮：轮次 0 处理全部，轮次 1..3 仅处理上一轮失败项 —— + for (let round = 0; round <= MAX_RETRY && targets.length > 0; round++) { + pkgPullState.round = round + pkgPullState.failed = 0 + pkgPullState.failedList = [] + const currentTargets = [...targets] + let next = 0 + const worker = async () => { + while (next < currentTargets.length) { + const item = currentTargets[next++] + pkgPullState.current = item.pkg_name + try { + await pullOne(item) + pkgPullState.success += 1 + markDone(item.id) // 成功 → 清单 side done + } catch (e) { + pkgPullState.failed += 1 + pkgPullState.failedList.push({ name: item.pkg_name, reason: (e && e.message) || '拉取失败' }) + } + pkgPullState.processed += 1 + } + } + await Promise.all(Array.from({ length: Math.min(CONCURRENT, currentTargets.length) }, worker)) + // 本轮结束：失败项作为下一轮重试目标（重试前先等所有 worker 完成） + targets = getPendingRows().filter(t => pkgPullState.failedList.some(f => f.name === t.pkg_name)) + if (round < MAX_RETRY && targets.length > 0) { + console.log(`[pkglist-pull] 第 ${round + 1} 轮结束：失败 ${targets.length} 个，进入第 ${round + 2} 轮重试`) + } + } + + // —— 汇总：失败条目保持 pending（下次可再拉），其余已 done —— + // 注意：循环结束后的 pkgPullState.failedList 即最后一轮失败项的真实 reason + //（若全部成功则为空数组），直接保留即可，不要用统一文案覆盖。 + if (!pkgPullState.failedList.length) { + pkgPullState.failed = 0 + } + pkgPullState.round = Math.min(pkgPullState.round, MAX_RETRY) // 标记已尽力完成重试 + pkgPullState.current = null + + addLog({ + action: 'list-pull', + pkgName: null, + deletedCount: pkgPullState.success, + deletedVersions: JSON.stringify({ + source, + round: pkgPullState.round, + total: pkgPullState.total, + success: pkgPullState.success, + failed: pkgPullState.failedList, + }), + }) + invalidateScan() // 有新包入库 → 废弃扫描缓存，列表下次请求重扫 + console.log(`[pkglist-pull] 完成（source=${source}）：成功 ${pkgPullState.success} · 失败 ${pkgPullState.failed}`) + } catch (e) { + pkgPullState.error = e.message + console.error('[pkglist-pull] 任务异常:', e.message) + } finally { + pkgPullState.running = false + pkgPullState.done = true + pkgPullState.finishedAt = new Date().toISOString() + } + })() // 立即执行（异步，不阻塞调用方） + return { started: true, active: true, pendingCount: pending0 } +} + +export { VERDACCIO_BASE as PKGLIST_VERDACCIO_BASE } \ No newline at end of file diff --git a/backend/pkglist.js b/backend/pkglist.js new file mode 100644 index 0000000..cd28ad0 --- /dev/null +++ b/backend/pkglist.js @@ -0,0 +1,654 @@ +/** + * npm 包清单（热门包缓存扩充）—— 数据层 + 配置 + 定时调度 + * + * 背景：前端 npm 包层出不穷，存量缓存覆盖不全。本模块维护一份「存量之外」的 + * 热门 npm 包清单（vue / react / nodejs 生态为主），定时更新收录最新热门包， + * 并以私服 VERDACCIO_BASE_URL 为镜像源按清单拉取，尽可能多地缓存 npm 包， + * 提高依赖下载速度。 + * + * 清单语义： + * - status='pending' 待拉取（含新写入 / 拉取失败未成功） + * - status='done' 已成功拉取到私服缓存 + * - 全部条目 done ⇔ 清单失效（当前无可用 npm 包）：手动拉取提示、定时拉取自动停止， + * 直到清单写入新的 pending 条目（手工新增 / 定时更新收录新热门包）后恢复。 + * + * 配置存储（settings KV，JSON）： + * pkglist_schedule_update { enabled, cron } 定时更新清单（默认每天 03:00） + * pkglist_schedule_pull { enabled, cron } 定时按清单拉取（默认每天 06:00） + * pkglist_seeded '1' 内置清单是否已初始化 + */ +import db, { getSetting, setSetting, addLog, nowIso } from './db.js' + +// ================= 内置热门清单（存量之外，按品类收录） ================= +// version 全部留空 = 拉取时解析私服 dist-tags.latest（永远最新版） +const HOT_POOL = [ + // —— 代码编辑器 —— + { name: 'monaco-editor', category: '代码编辑器' }, + { name: '@monaco-editor/react', category: '代码编辑器' }, + { name: 'codemirror', category: '代码编辑器' }, + { name: '@codemirror/view', category: '代码编辑器' }, + { name: '@codemirror/state', category: '代码编辑器' }, + { name: '@codemirror/lang-javascript', category: '代码编辑器' }, + { name: '@codemirror/lang-html', category: '代码编辑器' }, + { name: '@codemirror/lang-css', category: '代码编辑器' }, + { name: '@codemirror/lang-json', category: '代码编辑器' }, + { name: '@uiw/react-codemirror', category: '代码编辑器' }, + { name: 'ace-builds', category: '代码编辑器' }, + { name: '@toast-ui/editor', category: '代码编辑器' }, + { name: 'highlight.js', category: '代码编辑器' }, + { name: 'prismjs', category: '代码编辑器' }, + { name: 'shiki', category: '代码编辑器' }, + { name: 'marked', category: '代码编辑器' }, + { name: 'markdown-it', category: '代码编辑器' }, + { name: 'dompurify', category: '代码编辑器' }, + // —— JSON 编辑器 —— + { name: 'vanilla-jsoneditor', category: 'JSON 编辑器' }, + { name: '@json-editor/json-editor', category: 'JSON 编辑器' }, + { name: 'jsoneditor', category: 'JSON 编辑器' }, + { name: 'react-json-view', category: 'JSON 编辑器' }, + { name: 'react-json-view-lite', category: 'JSON 编辑器' }, + { name: '@uiw/react-json-view', category: 'JSON 编辑器' }, + { name: 'jsonc-parser', category: 'JSON 编辑器' }, + // —— 可视化图表 / 数据可视化 —— + { name: 'echarts', category: '可视化图表' }, + { name: 'echarts-for-react', category: '可视化图表' }, + { name: 'vue-echarts', category: '可视化图表' }, + { name: '@antv/g2', category: '可视化图表' }, + { name: '@antv/g2plot', category: '可视化图表' }, + { name: '@ant-design/plots', category: '可视化图表' }, + { name: '@ant-design/charts', category: '可视化图表' }, + { name: 'bizcharts', category: '可视化图表' }, + { name: 'chart.js', category: '可视化图表' }, + { name: 'react-chartjs-2', category: '可视化图表' }, + { name: 'd3', category: '可视化图表' }, + { name: 'd3-scale', category: '可视化图表' }, + { name: 'd3-shape', category: '可视化图表' }, + { name: 'three', category: '可视化图表' }, + { name: 'plotly.js', category: '可视化图表' }, + { name: 'recharts', category: '可视化图表' }, + { name: 'apexcharts', category: '可视化图表' }, + { name: 'lightweight-charts', category: '可视化图表' }, + { name: 'zrender', category: '可视化图表' }, + { name: 'konva', category: '可视化图表' }, + { name: 'vega', category: '可视化图表' }, + { name: 'vega-lite', category: '可视化图表' }, + { name: 'leaflet', category: '可视化图表' }, + { name: 'maptalks', category: '可视化图表' }, + { name: 'openlayers', category: '可视化图表' }, + // —— 流程图 / 关系图 / 思维导图 —— + { name: '@antv/x6', category: '流程图' }, + { name: '@antv/g6', category: '流程图' }, + { name: 'dagre', category: '流程图' }, + { name: 'dagre-d3', category: '流程图' }, + { name: '@xyflow/react', category: '流程图' }, + { name: 'reactflow', category: '流程图' }, + { name: '@vue-flow/core', category: '流程图' }, + { name: 'gojs', category: '流程图' }, + { name: 'jointjs', category: '流程图' }, + { name: 'cytoscape', category: '流程图' }, + { name: 'vis-network', category: '流程图' }, + { name: 'elkjs', category: '流程图' }, + { name: '@logicflow/core', category: '流程图' }, + { name: 'mermaid', category: '流程图' }, + { name: 'markmap-lib', category: '思维导图' }, + { name: 'markmap-view', category: '思维导图' }, + { name: 'mind-elixir', category: '思维导图' }, + // —— 文档在线预览 / 文档处理 —— + { name: 'pdfjs-dist', category: '文档预览' }, + { name: 'pdf-lib', category: '文档预览' }, + { name: 'pdfmake', category: '文档预览' }, + { name: 'jspdf', category: '文档预览' }, + { name: 'docx', category: '文档预览' }, + { name: 'pptxgenjs', category: '文档预览' }, + { name: 'exceljs', category: '文档预览' }, + { name: 'xlsx', category: '文档预览' }, + { name: 'file-saver', category: '文档预览' }, + { name: 'mammoth', category: '文档预览' }, + { name: 'html2canvas', category: '文档预览' }, + { name: 'html2pdf.js', category: '文档预览' }, + { name: 'docx-preview', category: '文档预览' }, + { name: '@vue-office/docx', category: '文档预览' }, + { name: '@vue-office/pdf', category: '文档预览' }, + { name: '@vue-office/excel', category: '文档预览' }, + { name: '@react-pdf/renderer', category: '文档预览' }, + { name: 'viewerjs', category: '文档预览' }, + // —— 工具函数 / 基础库 —— + { name: 'lodash', category: '工具函数' }, + { name: 'lodash-es', category: '工具函数' }, + { name: 'dayjs', category: '工具函数' }, + { name: 'moment', category: '工具函数' }, + { name: 'date-fns', category: '工具函数' }, + { name: 'rxjs', category: '工具函数' }, + { name: 'ramda', category: '工具函数' }, + { name: 'immer', category: '工具函数' }, + { name: 'nanoid', category: '工具函数' }, + { name: 'uuid', category: '工具函数' }, + { name: 'clsx', category: '工具函数' }, + { name: 'classnames', category: '工具函数' }, + { name: 'fast-deep-equal', category: '工具函数' }, + { name: 'deepmerge', category: '工具函数' }, + { name: 'qs', category: '工具函数' }, + { name: 'semver', category: '工具函数' }, + { name: 'zod', category: '工具函数' }, + { name: 'ajv', category: '工具函数' }, + { name: 'yup', category: '工具函数' }, + { name: 'joi', category: '工具函数' }, + { name: 'mitt', category: '工具函数' }, + { name: 'eventemitter3', category: '工具函数' }, + { name: 'p-limit', category: '工具函数' }, + { name: 'p-map', category: '工具函数' }, + { name: 'p-queue', category: '工具函数' }, + { name: 'fast-glob', category: '工具函数' }, + { name: 'picomatch', category: '工具函数' }, + { name: 'minimatch', category: '工具函数' }, + { name: 'js-yaml', category: '工具函数' }, + { name: 'yaml', category: '工具函数' }, + { name: 'dotenv', category: '工具函数' }, + { name: 'cross-env', category: '工具函数' }, + { name: 'execa', category: '工具函数' }, + { name: 'chalk', category: '工具函数' }, + { name: 'ora', category: '工具函数' }, + { name: 'commander', category: '工具函数' }, + { name: 'minimist', category: '工具函数' }, + { name: 'chokidar', category: '工具函数' }, + { name: 'fs-extra', category: '工具函数' }, + { name: 'rimraf', category: '工具函数' }, + { name: 'glob', category: '工具函数' }, + { name: 'archiver', category: '工具函数' }, + { name: 'adm-zip', category: '工具函数' }, + { name: 'tar', category: '工具函数' }, + { name: 'qrcode', category: '工具函数' }, + { name: 'js-cookie', category: '工具函数' }, + { name: 'js-base64', category: '工具函数' }, + { name: 'crypto-js', category: '工具函数' }, + { name: 'he', category: '工具函数' }, + // —— Vue 生态 —— + { name: 'vue', category: 'Vue 生态' }, + { name: 'vue-router', category: 'Vue 生态' }, + { name: 'pinia', category: 'Vue 生态' }, + { name: 'vuex', category: 'Vue 生态' }, + { name: '@vueuse/core', category: 'Vue 生态' }, + { name: 'vue-i18n', category: 'Vue 生态' }, + { name: 'element-plus', category: 'Vue 生态' }, + { name: '@element-plus/icons-vue', category: 'Vue 生态' }, + { name: 'ant-design-vue', category: 'Vue 生态' }, + { name: 'naive-ui', category: 'Vue 生态' }, + { name: 'vant', category: 'Vue 生态' }, + { name: 'quasar', category: 'Vue 生态' }, + { name: 'vite', category: 'Vue 生态' }, + { name: '@vitejs/plugin-vue', category: 'Vue 生态' }, + { name: '@vue/compiler-sfc', category: 'Vue 生态' }, + { name: 'vue-loader', category: 'Vue 生态' }, + { name: '@vue/test-utils', category: 'Vue 生态' }, + { name: 'async-validator', category: 'Vue 生态' }, + { name: 'sortablejs', category: 'Vue 生态' }, + { name: 'vuedraggable', category: 'Vue 生态' }, + { name: '@tiptap/core', category: '富文本编辑器' }, + { name: '@tiptap/starter-kit', category: '富文本编辑器' }, + { name: '@tiptap/vue-3', category: '富文本编辑器' }, + { name: '@tiptap/react', category: '富文本编辑器' }, + { name: 'slate', category: '富文本编辑器' }, + { name: 'slate-react', category: '富文本编辑器' }, + { name: 'prosemirror-model', category: '富文本编辑器' }, + { name: 'prosemirror-view', category: '富文本编辑器' }, + { name: 'quill', category: '富文本编辑器' }, + // —— React 生态 —— + { name: 'react', category: 'React 生态' }, + { name: 'react-dom', category: 'React 生态' }, + { name: 'react-router-dom', category: 'React 生态' }, + { name: '@reduxjs/toolkit', category: 'React 生态' }, + { name: 'react-redux', category: 'React 生态' }, + { name: 'zustand', category: 'React 生态' }, + { name: 'jotai', category: 'React 生态' }, + { name: 'mobx', category: 'React 生态' }, + { name: '@tanstack/react-query', category: 'React 生态' }, + { name: '@tanstack/react-table', category: 'React 生态' }, + { name: 'next', category: 'React 生态' }, + { name: 'antd', category: 'React 生态' }, + { name: '@ant-design/icons', category: 'React 生态' }, + { name: '@mui/material', category: 'React 生态' }, + { name: '@emotion/react', category: 'React 生态' }, + { name: 'styled-components', category: 'React 生态' }, + { name: 'tailwindcss', category: 'React 生态' }, + { name: '@headlessui/react', category: 'React 生态' }, + { name: 'framer-motion', category: 'React 生态' }, + { name: 'react-hook-form', category: 'React 生态' }, + { name: 'formik', category: 'React 生态' }, + { name: 'react-dropzone', category: 'React 生态' }, + { name: 'react-select', category: 'React 生态' }, + { name: 'react-window', category: 'React 生态' }, + { name: 'prop-types', category: 'React 生态' }, + { name: '@vitejs/plugin-react', category: 'React 生态' }, + // —— Node 生态 —— + { name: 'express', category: 'Node 生态' }, + { name: 'koa', category: 'Node 生态' }, + { name: 'fastify', category: 'Node 生态' }, + { name: '@nestjs/core', category: 'Node 生态' }, + { name: '@nestjs/common', category: 'Node 生态' }, + { name: 'socket.io', category: 'Node 生态' }, + { name: 'ws', category: 'Node 生态' }, + { name: 'node-fetch', category: 'Node 生态' }, + { name: 'undici', category: 'Node 生态' }, + { name: 'axios', category: 'Node 生态' }, + { name: 'got', category: 'Node 生态' }, + { name: 'mongoose', category: 'Node 生态' }, + { name: 'pg', category: 'Node 生态' }, + { name: 'mysql2', category: 'Node 生态' }, + { name: 'better-sqlite3', category: 'Node 生态' }, + { name: 'prisma', category: 'Node 生态' }, + { name: '@prisma/client', category: 'Node 生态' }, + { name: 'sequelize', category: 'Node 生态' }, + { name: 'knex', category: 'Node 生态' }, + { name: 'typeorm', category: 'Node 生态' }, + { name: 'redis', category: 'Node 生态' }, + { name: 'ioredis', category: 'Node 生态' }, + { name: 'bullmq', category: 'Node 生态' }, + { name: 'node-cron', category: 'Node 生态' }, + { name: 'nodemailer', category: 'Node 生态' }, + { name: 'jsonwebtoken', category: 'Node 生态' }, + { name: 'bcryptjs', category: 'Node 生态' }, + { name: 'jose', category: 'Node 生态' }, + { name: 'helmet', category: 'Node 生态' }, + { name: 'cors', category: 'Node 生态' }, + { name: 'morgan', category: 'Node 生态' }, + { name: 'pino', category: 'Node 生态' }, + { name: 'winston', category: 'Node 生态' }, + { name: 'typescript', category: 'Node 生态' }, + { name: 'ts-node', category: 'Node 生态' }, + { name: 'tsx', category: 'Node 生态' }, + { name: 'eslint', category: 'Node 生态' }, + { name: 'prettier', category: 'Node 生态' }, + { name: 'vitest', category: 'Node 生态' }, + { name: 'jest', category: 'Node 生态' }, + { name: 'mocha', category: 'Node 生态' }, + { name: 'supertest', category: 'Node 生态' }, + { name: 'chai', category: 'Node 生态' }, + { name: 'esbuild', category: 'Node 生态' }, + { name: 'rollup', category: 'Node 生态' }, + { name: 'webpack', category: 'Node 生态' }, + { name: 'terser', category: 'Node 生态' }, + { name: 'postcss', category: 'Node 生态' }, + { name: 'autoprefixer', category: 'Node 生态' }, + { name: 'sass', category: 'Node 生态' }, + { name: 'less', category: 'Node 生态' }, + // —— 动画 / 视觉辅助 —— + { name: 'gsap', category: '动画' }, + { name: 'lottie-web', category: '动画' }, + { name: 'canvas-confetti', category: '动画' }, + { name: 'nprogress', category: '动画' }, + { name: 'animejs', category: '动画' }, + { name: 'intro.js', category: '动画' }, + { name: 'driver.js', category: '动画' }, + { name: 'swiper', category: '动画' }, +] + +// ================= 数据访问 ================= + +export const HOT_POOL_NAMES = HOT_POOL.map(p => p.name) + +const HOT_BY_NAME = new Map(HOT_POOL.map(p => [p.name, p.category])) + +/** 首次初始化：把内置热门清单写入 DB（powered by settings 标记，幂等） */ +export function seedIfNeeded() { + if (getSetting('pkglist_seeded', '') === '1') return 0 + const ins = db.prepare( + 'INSERT OR IGNORE INTO pkglist (pkg_name, version, category, status, auto, created_at) VALUES (?, NULL, ?, ?, 1, ?)', + ) + let n = 0 + db.transaction(() => { + for (const p of HOT_POOL) { + n += ins.run(p.name, p.category, 'pending', nowIso()).changes + } + setSetting('pkglist_seeded', '1') + })() + console.log(`[pkglist] 内置热门清单初始化完成：写入 ${n} 条`) + return n +} + +/** 全部清单条目（id desc） */ +export function getAllRows() { + return db.prepare('SELECT id, pkg_name, version, category, status, auto, created_at FROM pkglist ORDER BY id DESC').all() +} + +/** pending 条目（待拉取） */ +export function getPendingRows() { + return db.prepare("SELECT id, pkg_name, version, category FROM pkglist WHERE status = 'pending' ORDER BY id ASC").all() +} + +export function countPending() { + return db.prepare("SELECT COUNT(*) AS c FROM pkglist WHERE status = 'pending'").get().c +} + +/** 标记某包已成功拉取（清单侧置 done；文件落盘由拉取模块完成） */ +export function markDone(id) { + return db.prepare("UPDATE pkglist SET status = 'done' WHERE id = ?").run(id).changes > 0 +} + +/** 重置为待拉取（清单有新的变更写入时调用，使定时拉取恢复） */ +export function resetPending(id) { + db.prepare("UPDATE pkglist SET status = 'pending' WHERE id = ?").run(id) +} + +/** 清单对外视图：items + 是否还有可用包（active = 存在 pending） */ +export function getPkgList() { + const items = getAllRows().map(r => ({ + id: r.id, + pkgName: r.pkg_name, + version: r.version || '', // 空 = 最新版 + category: r.category || '', + status: r.status, + auto: !!r.auto, + createdAt: r.created_at, + })) + return { items, active: countPending() > 0 } +} + +/** 新增条目（包名必填且唯一；version 可空 = 最新版；自动来源 auto=0） */ +export function addPkg({ pkgName, version = '', category = '' }) { + const name = String(pkgName || '').trim() + if (!name) return { ok: false, message: '包名不能为空' } + const exists = db.prepare('SELECT 1 FROM pkglist WHERE pkg_name = ?').get(name) + if (exists) return { ok: false, message: `「${name}」已在清单中` } + const cat = String(category || '').trim() || HOT_BY_NAME.get(name) || '其他' + db.prepare('INSERT INTO pkglist (pkg_name, version, category, status, auto, created_at) VALUES (?, ?, ?, ?, 0, ?)') + .run(name, String(version || '').trim() || null, cat, 'pending', nowIso()) + addLog({ action: 'list-add', pkgName: name, deletedVersions: JSON.stringify({ version: version || null, category: cat }) }) + return { ok: true } +} + +/** 编辑条目（改包名/版本/品类；内容变更后重置为 pending，相当于清单写入新内容） */ +export function editPkg(id, { pkgName, version, category }) { + const row = db.prepare('SELECT * FROM pkglist WHERE id = ?').get(Number(id)) + if (!row) return { ok: false, message: '清单条目不存在' } + const name = pkgName != null ? String(pkgName).trim() : row.pkg_name + if (!name) return { ok: false, message: '包名不能为空' } + if (name !== row.pkg_name) { + const dup = db.prepare('SELECT 1 FROM pkglist WHERE pkg_name = ? AND id != ?').get(name, row.id) + if (dup) return { ok: false, message: `「${name}」已在清单中` } + } + const ver = version != null ? String(version).trim() || null : row.version + const cat = category != null && String(category).trim() ? String(category).trim() : row.category + // 内容有变化 → 重置 pending（新目标需要重新拉取；否则保持原状态） + const changed = name !== row.pkg_name || ver !== row.version || cat !== row.category + db.prepare('UPDATE pkglist SET pkg_name = ?, version = ?, category = ?, status = ? WHERE id = ?') + .run(name, ver, cat, changed ? 'pending' : row.status, row.id) + if (changed) addLog({ action: 'list-edit', pkgName: name, deletedVersions: JSON.stringify({ from: row.pkg_name, version: ver, category: cat }) }) + return { ok: true } +} + +/** 删除条目 */ +export function removePkg(id) { + const row = db.prepare('SELECT * FROM pkglist WHERE id = ?').get(Number(id)) + if (!row) return { ok: false, message: '清单条目不存在' } + db.prepare('DELETE FROM pkglist WHERE id = ?').run(row.id) + addLog({ action: 'list-del', pkgName: row.pkg_name, deletedVersions: JSON.stringify({ version: row.version }) }) + return { ok: true } +} + +/** 定时更新清单：把内置热门池中尚未收录的包批量加入（pending，auto 来源），返回新增条数 */ +export function addFromHotPool() { + const ins = db.prepare( + 'INSERT OR IGNORE INTO pkglist (pkg_name, version, category, status, auto, created_at) VALUES (?, NULL, ?, ?, 1, ?)', + ) + let n = 0 + db.transaction(() => { + for (const p of HOT_POOL) { + n += ins.run(p.name, p.category, 'pending', nowIso()).changes + } + })() + if (n > 0) console.log(`[pkglist] 定时收录 ${n} 个新热门包`) + return n +} + +// ================= npm-search 热门榜单抓取（并入定时更新，动态收录） ================= +// 与「最新最热门」内置池合并为同一个「定时更新清单」能力：内置池 ∪ 榜单，按包名去重。 +// 榜单源优先级：私服 search 端点 → 官方 registry npm-search；全部不可用时降级为仅内置池。 + +const VERDACCIO_BASE = (process.env.VERDACCIO_BASE_URL || 'http://127.0.0.1:4873').replace(/\/+$/, '') +const NPMJS_SEARCH_BASE = 'https://registry.npmjs.org' + +// 关键词 → 品类（命中同一包时首个关键词的品类生效；未命中的包归为「热门榜单」） +const SEARCH_KEYWORDS = [ + ['vue', 'Vue 生态'], + ['react', 'React 生态'], + ['nodejs', 'Node 生态'], + ['typescript', 'Node 生态'], + ['chart', '可视化图表'], + ['code editor', '代码编辑器'], + ['json editor', 'JSON 编辑器'], + ['flowchart', '流程图'], + ['pdf', '文档预览'], + ['utility', '工具函数'], + ['form', '表单'], + ['animation', '动画'], + ['rich text', '富文本编辑器'], + ['state management', '状态管理'], +] + +const SEARCH_SIZE = 50 // 每关键词取前 50（按搜索综合分） +const SEARCH_TIMEOUT_MS = 6000 // 单次请求超时，避免私服 search 挂起阻塞定时任务 +const SEARCH_BATCH = 4 // 并发请求数，避免触发 registry 限流 +const RANK_LIMIT = 300 // 榜单合并去重后最多收录条数 + +const batchOf = (arr, size) => { + const out = [] + for (let i = 0; i < arr.length; i += size) out.push(arr.slice(i, i + size)) + return out +} + +async function fetchSearchJson(base, keyword) { + const url = `${base}/-/v1/search?text=${encodeURIComponent(keyword)}&size=${SEARCH_SIZE}` + const ctrl = new AbortController() + const timer = setTimeout(() => ctrl.abort(), SEARCH_TIMEOUT_MS) + try { + const r = await fetch(url, { signal: ctrl.signal, headers: { Accept: 'application/json' } }) + if (!r.ok) throw new Error(`search 响应 ${r.status}`) + return await r.json() + } finally { + clearTimeout(timer) + } +} + +/** 从单个来源分批并发轮询关键词抓取热门包（内部按包名去重，品类随首个命中关键词），按热度降序裁剪到 RANK_LIMIT；源不可用返回 [] */ +export async function fetchRankFromSource(base, keywords = SEARCH_KEYWORDS) { + const byName = new Map() + const kwCat = new Map(keywords) // kw -> category + let gotAny = false + for (const batch of batchOf(keywords, SEARCH_BATCH)) { + const settled = await Promise.allSettled( + batch.map(async ([kw]) => ({ kw, data: await fetchSearchJson(base, kw) })), + ) + for (const s of settled) { + if (s.status !== 'fulfilled') continue + gotAny = true + const category = kwCat.get(s.value.kw) || '热门榜单' + const objs = s.value.data && Array.isArray(s.value.data.objects) ? s.value.data.objects : [] + for (const o of objs) { + const name = o && o.package && typeof o.package.name === 'string' ? o.package.name.trim() : '' + if (!name || name.length > 100 || byName.has(name)) continue + const popular = (o.score && o.score.detail && o.score.detail.popularity) || 0 + byName.set(name, { name, popular, category }) + } + } + } + // 一个关键词都没有成功 → 该源不可用（如 search 端点未启用 / 挂起），返回空让上层回退 + if (!gotAny) return [] + return [...byName.values()] + .sort((a, b) => b.popular - a.popular) + .slice(0, RANK_LIMIT) + .map(p => ({ name: p.name, category: p.category })) +} + +/** + * 榜单抓取：私服 search 端点优先，不可用时回退官方 registry npm-search。 + * 全部失败返回 null（调用方据此降级为仅内置池）。onSource(base) 用于记录实际来源。 + */ +export async function fetchNpmSearchRank({ onSource } = {}) { + for (const base of [VERDACCIO_BASE, NPMJS_SEARCH_BASE]) { + try { + const items = await fetchRankFromSource(base) + if (items && items.length > 0) { + if (onSource) onSource(base) + return { source: base, items } + } + } catch (e) { /* 尝试下一来源 */ } + } + return null +} + +/** + * 定时更新清单（唯一入口，不新增配置开关）： + * 内置「最新最热门」热门池 ∪ npm-search 榜单，按包名去重后增量写入（pending，auto 来源）。 + * 榜单抓取失败时降级为仅内置池，不影响收录。 + * @returns {{ added: number, rankSource: string }} added 本次新增条数；rankSource 榜单实际来源（'' = 榜单不可用） + */ +export async function updateFromAllSources() { + const merged = new Map() // name -> category（内置池优先，保证品类与唯一性） + for (const p of HOT_POOL) { + if (!merged.has(p.name)) merged.set(p.name, p.category) + } + let rankSource = '' + try { + const rank = await fetchNpmSearchRank() + if (rank) { + rankSource = rank.source + for (const p of rank.items) { + if (!merged.has(p.name)) merged.set(p.name, p.category || '热门榜单') + } + } + } catch (e) { + // 榜单解析异常 → 仅内置池 + } + const ins = db.prepare( + 'INSERT OR IGNORE INTO pkglist (pkg_name, version, category, status, auto, created_at) VALUES (?, NULL, ?, ?, 1, ?)', + ) + let added = 0 + try { + db.transaction(() => { + for (const [name, category] of merged) { + added += ins.run(name, category, 'pending', nowIso()).changes + } + })() + } catch (e) { + console.error('[pkglist] 定时收录写入失败:', e.message) + return { added: 0, rankSource: rankSource || '' } + } + const note = rankSource + ? `定时收录热门 npm 包：内置池+榜单 新增 ${added} 个（榜单源 ${rankSource}）` + : `定时收录热门 npm 包：内置池 新增 ${added} 个（npm-search 榜单不可用，已降级）` + if (added > 0 || rankSource) console.log(`[pkglist] ${note}`) + return { added, rankSource: rankSource || '' } +} + +// ================= 定时配置 ================= + +const DEFAULT_CONFIG = { + update: { enabled: false, cron: '0 3 * * *' }, // 每天 03:00 收录最新热门包 + pull: { enabled: false, cron: '0 6 * * *' }, // 每天 06:00 按清单拉取 +} + +function readJson(key, fallback) { + try { + const v = JSON.parse(getSetting(key, '')) + if (v && typeof v === 'object') return v + } catch { /* fallthrough */ } + return fallback +} + +function writeCfg(key, cfg) { + setSetting(key, JSON.stringify(cfg)) +} + +export function getConfig() { + const update = { ...DEFAULT_CONFIG.update, ...readJson('pkglist_schedule_update', {}) } + const pull = { ...DEFAULT_CONFIG.pull, ...readJson('pkglist_schedule_pull', {}) } + return { update, pull } +} + +/** 保存配置：{ update?: {enabled, cron}, pull?: {enabled, cron} }；cron 校验 5 段 */ +export function saveConfig(patch = {}) { + const cur = getConfig() + for (const key of ['update', 'pull']) { + const p = patch[key] + if (!p || typeof p !== 'object') continue + const suffix = key === 'update' ? 'pkglist_schedule_update' : 'pkglist_schedule_pull' + const next = { ...cur[key], enabled: !!p.enabled } + if (p.cron != null) { + const cron = String(p.cron).trim() + if (!/^(\S+ ){4}\S+$/.test(cron)) return { ok: false, message: `定时${key === 'update' ? '更新' : '拉取'}的 cron 表达式格式应为 5 段（分 时 日 月 周）` } + next.cron = cron + } + writeCfg(suffix, next) + } + return { ok: true, config: getConfig() } +} + +// ================= 定时调度（依赖注入，避免与 server 循环引用） ================= + +const pad2 = n => String(n).padStart(2, '0') +const cronKey = d => `${d.getFullYear()}${pad2(d.getMonth() + 1)}${pad2(d.getDate())}${pad2(d.getHours())}${pad2(d.getMinutes())}` + +/** 简化 cron 匹配：分 时 日 月 周（0/7=周日），支持 * / , - */ +export function cronMatch(expr, date) { + const parts = String(expr || '').trim().split(/\s+/) + if (parts.length !== 5) return false + const [min, hour, day, mon, dow] = parts + const d = date || new Date() + const mins = expandCron(min, 0, 59) + const hours = expandCron(hour, 0, 23) + const days = expandCron(day, 1, 31) + const months = expandCron(mon, 1, 12) + const dows = expandCron(dow, 0, 7) // 7 → 0（周日） + const dowsNorm = new Set([...dows].map(x => (x === 7 ? 0 : x))) + return mins.has(d.getMinutes()) + && hours.has(d.getHours()) + && days.has(d.getDate()) + && months.has(d.getMonth() + 1) + && dowsNorm.has(d.getDay()) +} + +function expandCron(field, lo, hi) { + const out = new Set() + for (const part of String(field).split(',')) { + if (part === '*') { for (let v = lo; v <= hi; v++) out.add(v); continue } + let step = 1 + let base = part + const slash = part.indexOf('/') + if (slash !== -1) { base = part.slice(0, slash); step = parseInt(part.slice(slash + 1), 10) || 1 } + if (base === '*') { for (let v = lo; v <= hi; v += step) out.add(v); continue } + const dash = base.indexOf('-') + if (dash !== -1) { + const a = parseInt(base.slice(0, dash), 10) + const b = parseInt(base.slice(dash + 1), 10) + for (let v = a; v <= b; v += step) out.add(v) + continue + } + const n = parseInt(base, 10) + if (!Number.isNaN(n)) out.add(n) + } + return out +} + +let lastRunMinute = '' + +/** + * 启动清单守护（每分钟判定一次 cron；同一分钟只执行一次，幂等） + * @param {{ addLog, invalidateScan, runPull }} services + * runPull(source) 由外部注入拉取执行器（pkglist-pull.js） + */ +export function startPkgListDaemon({ runAutoUpdate, runAutoPull }) { + setInterval(() => { + try { + const now = new Date() + const key = cronKey(now) + if (key === lastRunMinute) return + lastRunMinute = key + const cfg = getConfig() + if (cfg.update.enabled && cronMatch(cfg.update.cron, now)) runAutoUpdate() + if (cfg.pull.enabled && cronMatch(cfg.pull.cron, now)) runAutoPull() + } catch (e) { + console.error('[pkglist] 定时守护执行异常:', e.message) + } + }, 30_000).unref() + console.log('[pkglist] 定时守护已启动（每分钟检查 update/pull cron）') +} \ No newline at end of file diff --git a/backend/server.js b/backend/server.js index 6998996..eaacc3c 100644 --- a/backend/server.js +++ b/backend/server.js @@ -28,6 +28,11 @@ import { import { cleanPackage, deleteSingleVersion } from './clean.js' import semverCompare from './semver.js' import { syncState, runSyncLatest } from './sync-latest.js' +import { + seedIfNeeded, countPending, getPkgList, addPkg, editPkg, removePkg, + getConfig, saveConfig, startPkgListDaemon, updateFromAllSources, +} from './pkglist.js' +import { pkgPullState, runPkgListPull } from './pkglist-pull.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) const app = express() @@ -441,6 +446,75 @@ app.get('/api/sync/progress', (req, res) => { }) }) +// —— API：npm 包清单（热门包缓存扩充） ----- +// 清单维护：查看 / 新增 / 编辑 / 删除（含 valid 性校验与操作日志） +app.get('/api/pkglist', (req, res) => { + res.json({ ok: true, ...getPkgList() }) +}) +app.post('/api/pkglist', (req, res) => { + const r = addPkg(req.body || {}) + if (!r.ok) return res.status(400).json(r) + res.json({ ok: true, ...getPkgList() }) +}) +app.put('/api/pkglist/:id', (req, res) => { + const r = editPkg(req.params.id, req.body || {}) + if (!r.ok) return res.status(400).json(r) + res.json({ ok: true, ...getPkgList() }) +}) +app.delete('/api/pkglist/:id', (req, res) => { + const r = removePkg(req.params.id) + if (!r.ok) return res.status(400).json(r) + res.json({ ok: true, ...getPkgList() }) +}) + +// —— API：按清单手动拉取（同步启动后台任务，不阻塞；失败自动重试 3 次；清单失效时提示） —— +app.post('/api/pkglist/pull', (req, res) => { + if (isDemoMode()) { + return res.json({ ok: true, demo: true, running: false, message: '演示模式：未配置 Verdaccio 缓存目录，不支持拉取' }) + } + // 清单失效（全部条目已成功拉取）→ 当前无可用 npm 包 + if (countPending() === 0) { + return res.json({ ok: true, active: false, running: false, message: '当前无可用npm包' }) + } + const r = runPkgListPull({ addLog, invalidateScan, source: 'manual' }) + res.json({ + ok: true, demo: false, + running: r.started ? true : pkgPullState.running, + active: r.active, + pendingCount: r.pendingCount, + startedAt: pkgPullState.startedAt, + }) +}) + +// —— API：按清单拉取进度（拉取期间由前端轮询；含重试轮次与失败明细） —— +app.get('/api/pkglist/progress', (req, res) => { + res.json({ + running: pkgPullState.running, + done: pkgPullState.done, + total: pkgPullState.total, + processed: pkgPullState.processed, + success: pkgPullState.success, + failed: pkgPullState.failed, + failedList: pkgPullState.failedList, + round: pkgPullState.round, + current: pkgPullState.current, + source: pkgPullState.source, + error: pkgPullState.error, + startedAt: pkgPullState.startedAt, + finishedAt: pkgPullState.finishedAt, + }) +}) + +// —— API：清单定时更新 / 定时拉取配置（右上角用户名下拉入口读写） —— +app.get('/api/pkglist/config', (req, res) => { + res.json({ ok: true, config: getConfig() }) +}) +app.post('/api/pkglist/config', (req, res) => { + const r = saveConfig(req.body || {}) + if (!r.ok) return res.status(400).json(r) + res.json({ ok: true, config: r.config }) +}) + // —— API：清理 —— app.post('/api/clean', (req, res) => { const { pkgName, keep } = req.body || {} @@ -530,6 +604,32 @@ app.listen(PORT, () => { if (mode === 'storage') console.log(`[verdaccio-manager] storage: ${storage}`) else console.log('[verdaccio-manager] 未配置 VERDACCIO_STORAGE，运行于演示模式（设置 VCM_DEMO=0 + 真实目录可关闭）') scheduleDailyScan() + // —— npm 包清单：初始化内置热门包 + 启动定时守护（定时更新收录 / 定时按清单拉取） —— + try { + seedIfNeeded() + startPkgListDaemon({ + // 定时更新清单：内置热门池 ∪ npm-search 榜单，按包名去重后补入（pending） + runAutoUpdate() { + updateFromAllSources().then(({ added, rankSource }) => { + addLog({ + action: 'list-auto', + pkgName: null, + deletedCount: added, + deletedVersions: JSON.stringify({ + note: rankSource ? `定时收录热门 npm 包（榜单源 ${rankSource}）` : '定时收录热门 npm 包（npm-search 榜单不可用，仅内置池）', + }), + }) + }) + }, + // 定时拉取：清单已失效（无 pending）时自动停止，直到清单写入新包 + runAutoPull() { + if (isDemoMode()) return + runPkgListPull({ addLog, invalidateScan, source: 'auto' }) + }, + }) + } catch (e) { + console.error('[pkglist] 初始化失败:', e.message) + } }) // 清理登录限速表：超过窗口 + 锁定期的记录不再保留（防 Map 无限增长） diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 29748f6..40233e1 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -347,7 +347,7 @@ async login(username, password, remember = true) { const r = await authFetch(`/api/packages/${encodeURIComponent(pkgName)}/versions/${encodeURIComponent(version)}`, { method: 'DELETE', }) - return r.json() +return r.json() }, async getLogs() { @@ -358,6 +358,135 @@ async login(username, password, remember = true) { const r = await authFetch('/api/logs') return r.json() }, + + // —— npm 包清单（热门包缓存扩充） —— + + /** 读取清单：items（含 status/auto）+ active（是否有待拉取包） */ + async getPkgList() { + if (USE_MOCK) { + await delay(80) + const items = clone(mock.pkglistItems) + return { ok: true, items, active: items.some(i => i.status === 'pending') } + } + const r = await authFetch('/api/pkglist') + return r.json() + }, + + /** 新增清单条目（pkgName 必填；version 空 = 最新版） */ + async addPkgItem(payload) { + if (USE_MOCK) { + await delay() + const { pkgName = '', version = '', category = '' } = payload + const name = String(pkgName).trim() + if (!name) return { ok: false, message: '包名不能为空' } + if (mock.pkglistItems.some(i => i.pkgName === name)) return { ok: false, message: `「${name}」已在清单中` } + mock.pkglistItems.push({ + id: Date.now(), pkgName: name, version: String(version || '').trim(), + category: String(category || '').trim() || '其他', status: 'pending', auto: 0, + createdAt: new Date().toISOString().slice(0, 10), + }) + return { ok: true, items: clone(mock.pkglistItems), active: true } + } + const r = await authFetch('/api/pkglist', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }) + return r.json() + }, + + /** 编辑清单条目（包名/版本号/分类） */ + async updatePkgItem(id, payload) { + if (USE_MOCK) { + await delay() + const it = mock.pkglistItems.find(i => i.id === Number(id)) + if (!it) return { ok: false, message: '清单条目不存在' } + const { pkgName = '', version = '', category = '' } = payload + const name = String(pkgName).trim() + if (!name) return { ok: false, message: '包名不能为空' } + const dup = mock.pkglistItems.some(i => i.id !== Number(id) && i.pkgName === name) + if (dup) return { ok: false, message: `「${name}」已在清单中` } + it.pkgName = name + it.version = String(version || '').trim() + it.category = String(category || '').trim() || '其他' + return { ok: true, items: clone(mock.pkglistItems), active: true } + } + const r = await authFetch(`/api/pkglist/${id}`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }) + return r.json() + }, + + /** 删除清单条目 */ + async deletePkgItem(id) { + if (USE_MOCK) { + await delay() + const idx = mock.pkglistItems.findIndex(i => i.id === Number(id)) + if (idx === -1) return { ok: false, message: '清单条目不存在' } + mock.pkglistItems.splice(idx, 1) + return { ok: true, items: clone(mock.pkglistItems), active: mock.pkglistItems.some(i => i.status === 'pending') } + } + const r = await authFetch(`/api/pkglist/${id}`, { method: 'DELETE' }) + return r.json() + }, + + /** 按清单手动拉取（同步启动后台任务，立即返回；清单失效时返回 active=false） */ + async pullPkgList() { + if (USE_MOCK) { + await delay(200) + const active = mock.pkglistItems.some(i => i.status === 'pending') + if (!active) return { ok: true, active: false, running: false, message: '当前无可用npm包' } + return { ok: true, demo: false, running: true, active: true, pendingCount: mock.pkglistItems.filter(i => i.status === 'pending').length } + } + const r = await authFetch('/api/pkglist/pull', { method: 'POST' }) + return r.json() + }, + + /** 查询按清单拉取进度（含重试轮次 round 与失败明细 failedList） */ + async getPkgPullProgress() { + if (USE_MOCK) { + await delay(60) + const pending = mock.pkglistItems.filter(i => i.status === 'pending') + const done = pending.length === 0 + return { + running: false, done, + total: mock.pkglistItems.length, processed: mock.pkglistItems.length, + success: mock.pkglistItems.length - pending.length, failed: 0, + failedList: [], round: 3, current: null, source: 'manual', error: null, + startedAt: null, finishedAt: new Date().toISOString(), + } + } + const r = await authFetch('/api/pkglist/progress') + return r.json() + }, + + /** 读取清单定时更新/定时拉取配置 */ + async getPkgListConfig() { + if (USE_MOCK) { + await delay(80) + return { ok: true, config: clone(mock.pkglistConfig) } + } + const r = await authFetch('/api/pkglist/config') + return r.json() + }, + + /** 保存清单定时更新/定时拉取配置 */ + async savePkgListConfig(payload) { + if (USE_MOCK) { + await delay() + mock.pkglistConfig.update = clone(payload.update || mock.pkglistConfig.update) + mock.pkglistConfig.pull = clone(payload.pull || mock.pkglistConfig.pull) + return { ok: true, config: clone(mock.pkglistConfig) } + } + const r = await authFetch('/api/pkglist/config', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }) + return r.json() + }, } export default api diff --git a/frontend/src/components/AppTopbar.vue b/frontend/src/components/AppTopbar.vue index 9ecfe14..b66a206 100644 --- a/frontend/src/components/AppTopbar.vue +++ b/frontend/src/components/AppTopbar.vue @@ -4,6 +4,7 @@ import { useRoute, useRouter } from 'vue-router' import { useTheme } from '../composables/useTheme' import { useStore } from '../store' import api from '../api' +import PkgListConfigDialog from './PkgListConfigDialog.vue' const route = useRoute() const router = useRouter() @@ -13,6 +14,7 @@ const { state } = useStore() const emit = defineEmits(['rescan']) const menuOpen = ref(false) +const pkgConfigOpen = ref(false) const navs = [ { path: '/', label: '缓存概览', emo: '🏠' }, @@ -29,12 +31,13 @@ function go(path) { async function pick(action) { if (action === 'theme') { // 主题切换即时生效，保持菜单打开以便反复对比 - toggle() +toggle() return } menuOpen.value = false if (action === 'rescan') emit('rescan') else if (action === 'thresholds') router.push('/thresholds') + else if (action === 'pkgConfig') pkgConfigOpen.value = true else if (action === 'logout') { await api.logout() // 先销毁后端会话并清本地 token，再跳登录页（否则守卫会因 token 仍在而重定向回首页） router.push('/login') @@ -80,17 +83,20 @@ onBeforeUnmount(() => document.removeEventListener('click', onDocClick))
👤
-
+
+
📦 版本信息 v1.0.0
-
+
+ + \ No newline at end of file diff --git a/frontend/src/components/PkgListDrawer.vue b/frontend/src/components/PkgListDrawer.vue new file mode 100644 index 0000000..6fd10b1 --- /dev/null +++ b/frontend/src/components/PkgListDrawer.vue @@ -0,0 +1,336 @@ + + + + + \ No newline at end of file diff --git a/frontend/src/mock/data.js b/frontend/src/mock/data.js index 839e54d..731d71b 100644 --- a/frontend/src/mock/data.js +++ b/frontend/src/mock/data.js @@ -150,6 +150,21 @@ const logs = [ { id: 1, action: 'clean', pkgName: 'axios', deletedVersions: '0.19.0', deletedCount: 1, freedSpace: '1.1 MB', created_at: dateStr(2) + ' 11:05:47' }, ] +// —— npm 包清单（热门包缓存扩充，mock 分支使用） —— +const pkglistItems = [ + { id: 1, pkgName: 'monaco-editor', version: '', category: '代码编辑器', status: 'done', auto: 1, createdAt: dateStr(2) }, + { id: 2, pkgName: '@monaco-editor/react', version: '', category: '代码编辑器', status: 'done', auto: 1, createdAt: dateStr(2) }, + { id: 3, pkgName: 'codemirror', version: '', category: '代码编辑器', status: 'pending', auto: 1, createdAt: dateStr(2) }, + { id: 4, pkgName: '@codemirror/lang-json', version: '', category: 'JSON 编辑器', status: 'pending', auto: 1, createdAt: dateStr(2) }, + { id: 5, pkgName: 'echarts', version: '', category: '可视化图表', status: 'pending', auto: 1, createdAt: dateStr(2) }, + { id: 6, pkgName: '@antv/g6', version: '5.0.0', category: '流程图', status: 'pending', auto: 1, createdAt: dateStr(2) }, + { id: 7, pkgName: 'xlsx', version: '', category: '工具函数', status: 'pending', auto: 0, createdAt: dateStr(1) }, +] +const pkglistConfig = { + update: { enabled: false, cron: '0 3 * * *' }, + pull: { enabled: false, cron: '0 6 * * *' }, +} + // 状态统计（与 overview 对齐） function summary() { const valid = packages.filter(p => p.status !== 'broken') @@ -173,4 +188,4 @@ function summary() { } } -export { packages, top30, logs, summary, fmtSize } +export { packages, top30, logs, summary, fmtSize, pkglistItems, pkglistConfig } diff --git a/frontend/src/store/index.js b/frontend/src/store/index.js index e2ef6c3..0845c2a 100644 --- a/frontend/src/store/index.js +++ b/frontend/src/store/index.js @@ -24,6 +24,17 @@ const state = reactive({ running: false, done: false, total: 0, processed: 0, success: 0, skipped: 0, failed: 0, failedList: [], error: null, }, + // npm 包清单（热门包缓存扩充）：items + active（是否存在待拉取包） + pkgList: { items: [], active: false }, + // 按清单拉取进度（含重试轮次 round 与失败明细 failedList，供角标展示） + pkgPullProgress: { + running: false, done: false, total: 0, processed: 0, + success: 0, failed: 0, failedList: [], round: 0, + current: null, source: 'manual', error: null, + startedAt: null, finishedAt: null, + }, + // 清单定时更新 / 定时拉取配置 + pkgListConfig: { update: { enabled: false, cron: '' }, pull: { enabled: false, cron: '' } }, }) // —— 派生统计 —— @@ -258,6 +269,128 @@ async function deleteVersion(name, version) { return res } +// —— npm 包清单（热门包缓存扩充） —— + +async function refreshPkgList() { + const r = await api.getPkgList() + if (r && r.ok) { + state.pkgList.items = r.items || [] + state.pkgList.active = !!r.active + } + return r +} + +/** 新增/编辑/删除清单条目后刷新清单与配置（新包写入会使定时拉取恢复） */ +async function addPkgItem(payload) { + const r = await api.addPkgItem(payload) + if (r && r.ok) { + state.pkgList.items = r.items || [] + state.pkgList.active = !!r.active + } + return r +} + +async function updatePkgItem(id, payload) { + const r = await api.updatePkgItem(id, payload) + if (r && r.ok) { + state.pkgList.items = r.items || [] + state.pkgList.active = !!r.active + } + return r +} + +async function deletePkgItem(id) { + const r = await api.deletePkgItem(id) + if (r && r.ok) { + state.pkgList.items = r.items || [] + state.pkgList.active = !!r.active + } + return r +} + +function resetPkgPullProgress(res) { + state.pkgPullProgress.running = true + state.pkgPullProgress.done = false + state.pkgPullProgress.total = res.total || 0 + state.pkgPullProgress.processed = 0 + state.pkgPullProgress.success = 0 + state.pkgPullProgress.failed = 0 + state.pkgPullProgress.failedList = [] + state.pkgPullProgress.round = 0 + state.pkgPullProgress.current = null + state.pkgPullProgress.error = null + state.pkgPullProgress.startedAt = res.startedAt || null +} + +/** 按清单手动拉取：触发后台任务并轮询进度；清单失效时提示无可用包 */ +async function triggerPkgPull() { + try { + const res = await api.pullPkgList() + if (!res || !res.ok) return res || { ok: false } + if (res.demo) return res + if (res.active === false) { + state.pkgPullProgress.running = false + state.pkgPullProgress.done = true + return res // 当前无可用 npm 包 + } + resetPkgPullProgress(res) + pollPkgPullProgress() + return res + } catch (e) { + ElMessage.error(e?.message || '发起拉取失败，请稍后重试') + return { ok: false } + } +} + +let pkgPullTimer = null + +/** 轮询按清单拉取进度；结束后刷新清单（状态置 done）与操作日志 */ +function pollPkgPullProgress() { + clearTimeout(pkgPullTimer) + const tick = async () => { + let p + try { + p = await api.getPkgPullProgress() + } catch { + state.pkgPullProgress.running = false + state.pkgPullProgress.error = '拉取进度查询失败' + return + } + state.pkgPullProgress.running = !!p.running + state.pkgPullProgress.done = !!p.done + state.pkgPullProgress.total = p.total || 0 + state.pkgPullProgress.processed = p.processed || 0 + state.pkgPullProgress.success = p.success || 0 + state.pkgPullProgress.failed = p.failed || 0 + state.pkgPullProgress.failedList = p.failedList || [] + state.pkgPullProgress.round = p.round || 0 + state.pkgPullProgress.current = p.current || null + state.pkgPullProgress.source = p.source || state.pkgPullProgress.source + state.pkgPullProgress.error = p.error || null + if (p.startedAt) state.pkgPullProgress.startedAt = p.startedAt + if (p.finishedAt) state.pkgPullProgress.finishedAt = p.finishedAt + if (p.running) { + pkgPullTimer = setTimeout(tick, 800) + } else { + clearTimeout(pkgPullTimer) + await Promise.all([refreshPkgList(), refreshLogs()]) + } + } + tick() +} + +async function refreshPkgListConfig() { + const r = await api.getPkgListConfig() + if (r && r.ok) state.pkgListConfig = r.config + return r +} + +async function savePkgListConfig(payload) { + const r = await api.savePkgListConfig(payload) + if (r && r.ok) state.pkgListConfig = r.config + return r +} + export function useStore() { return { state, @@ -274,5 +407,13 @@ scanNow, unignorePkg, cleanPkg, deleteVersion, + refreshPkgList, + addPkgItem, + updatePkgItem, + deletePkgItem, + triggerPkgPull, + pollPkgPullProgress, + refreshPkgListConfig, + savePkgListConfig, } } diff --git a/frontend/src/views/SearchView.vue b/frontend/src/views/SearchView.vue index 2823fac..797d542 100644 --- a/frontend/src/views/SearchView.vue +++ b/frontend/src/views/SearchView.vue @@ -3,12 +3,14 @@ import { ref, computed, watch, onMounted, onUnmounted } from 'vue' import { useRoute, useRouter } from 'vue-router' import PackageDrawer from '../components/PackageDrawer.vue' import CleanDialog from '../components/CleanDialog.vue' +import PkgListDrawer from '../components/PkgListDrawer.vue' import StatusTag from '../components/StatusTag.vue' import PkgLogo from '../components/PkgLogo.vue' import { useStore, fmtSize } from '../store' import { storage } from '../utils/storage' -const { state, stats, refreshPackages, ignorePkg, unignorePkg, cleanPkg, deleteVersion, syncLatest } = useStore() +const { state, stats, refreshPackages, ignorePkg, unignorePkg, cleanPkg, deleteVersion, syncLatest, + refreshPkgList, refreshPkgListConfig, triggerPkgPull } = useStore() const route = useRoute() const router = useRouter() @@ -92,6 +94,8 @@ onMounted(() => { statusFilter.value = route.query.status } if (route.query.updatedToday === '1') todayFilter.value = true + refreshPkgList() + refreshPkgListConfig() }) onUnmounted(() => { document.removeEventListener('click', closeMenus) @@ -221,6 +225,82 @@ watch( } }, ) + +// —— npm 包清单：查看/编辑入口 + 按清单手动拉取（后台执行，失败自动重试 3 次） —— +const pkgListOpen = ref(false) +const pullLoading = ref(null) +let retryNotified = false + +function closePullLoading() { + if (pullLoading.value) { + pullLoading.value.close() + pullLoading.value = null + } +} + +async function doPkgPull() { + const pp = state.pkgPullProgress + if (pp.running) return + const res = await triggerPkgPull() + if (!res || !res.ok) return + if (res.demo) { + ElMessage.info(res.message || '演示模式：不支持拉取') + return + } + if (res.active === false) { + ElMessage.info('当前无可用npm包') + await refreshPkgList() + return + } + retryNotified = false + closePullLoading() + pullLoading.value = ElMessage({ type: 'loading', message: '正在拉取npm包…', duration: 0 }) + // 任务已进入后台执行，进度由 store 轮询更新（running … done），不阻塞当前操作 +} + +// 重试轮次变化（round ≥ 2 表示第一次重试开始）→ 轻量提示 +watch( + () => state.pkgPullProgress.round, + (round) => { + if (round >= 2 && !retryNotified) { + retryNotified = true + closePullLoading() + pullLoading.value = ElMessage({ type: 'loading', message: '正在重新拉取npm包…', duration: 0 }) + } + }, +) + +// 拉取结束（running → false）→ 结束轻量提示；全部成功则清单失效 +watch( + () => state.pkgPullProgress.running, + async (running, prev) => { + if (prev === true && running === false) { + closePullLoading() + const pp = state.pkgPullProgress + if (pp.failed === 0 && pp.total > 0) { + ElMessage.success(`npm 包拉取完成，共缓存 ${pp.success} 个，清单已全部覆盖`) + } else if (pp.failed > 0) { + ElMessage.warning(`拉取完成：成功 ${pp.success} 个，失败 ${pp.failed} 个（未缓存）`) + } + await refreshPkgList() + } + }, +) + +// 失败包角标：展示累计失败数量，点击弹出明细列表 +function showFailedList() { + const pp = state.pkgPullProgress + if (!pp.failedList.length) return + const detail = pp.failedList.map(f => `· ${f.name || f}${f.reason ? `：${f.reason}` : ''}`) + ElMessageBox.alert(detail.join('\n'), `拉取失败 ${pp.failed} 个包（连续重试 3 次后仍未成功）`, { + confirmButtonText: '知道了', + type: 'warning', + customStyle: { maxHeight: '60vh', overflow: 'auto' }, + }).catch(() => {}) +} + +// 清单待拉取数量（显示在「包清单」入口角标上） +const pendingCount = computed(() => state.pkgList.items.filter(i => i.status === 'pending').length)