diff --git a/backend/db.js b/backend/db.js index 06b4199..c2d813c 100644 --- a/backend/db.js +++ b/backend/db.js @@ -70,6 +70,13 @@ if (!logCols.includes('action')) { db.exec("ALTER TABLE logs ADD COLUMN action TEXT DEFAULT 'clean'") } +// 旧库迁移 v2：pkglist 增加 cached 列——标记「写入清单时 storage 已缓存该包」（done 的来源之一， +// 与「通过清单拉取完成」区分，前端据此展示「已在缓存 / 已完成」两种 done） +const pkglistCols = db.prepare('PRAGMA table_info(pkglist)').all().map(c => c.name) +if (!pkglistCols.includes('cached')) { + db.exec("ALTER TABLE pkglist ADD COLUMN cached INTEGER DEFAULT 0") +} + // 旧库迁移 v1：历史日志/忽略记录的时间戳是 UTC，统一转成北京时间；settings 标志位防重复执行 function migrateTimezone() { if (getSetting('timezone_migrated_v1')) return diff --git a/backend/pkglist.js b/backend/pkglist.js index cd28ad0..549a30d 100644 --- a/backend/pkglist.js +++ b/backend/pkglist.js @@ -17,7 +17,36 @@ * pkglist_schedule_pull { enabled, cron } 定时按清单拉取（默认每天 06:00） * pkglist_seeded '1' 内置清单是否已初始化 */ +import fs from 'node:fs' +import path from 'node:path' import db, { getSetting, setSetting, addLog, nowIso } from './db.js' +import { resolveStorageDir, isValidStorage } from './scanner.js' + +// ================= 轻量：读取 storage 中已缓存的包名集合（仅列目录，不读文件） ================= +function listCachedPkgNames() { + const storageDir = resolveStorageDir() + if (!isValidStorage(storageDir)) return new Set() + const out = new Set() + let entries + try { + entries = fs.readdirSync(storageDir, { withFileTypes: true }) + } catch { return out } + for (const e of entries) { + if (!e.isDirectory() || e.name === 'node_modules' || e.name.startsWith('.')) continue + if (e.name.startsWith('@')) { + // scoped 包目录：@scope 下每个子目录是一个包 + try { + const subs = fs.readdirSync(path.join(storageDir, e.name), { withFileTypes: true }) + for (const s of subs) { + if (s.isDirectory()) out.add(`${e.name}/${s.name}`) + } + } catch { /* 忽略不可读目录 */ } + } else { + out.add(e.name) + } + } + return out +} // ================= 内置热门清单（存量之外，按品类收录） ================= // version 全部留空 = 拉取时解析私服 dist-tags.latest（永远最新版） @@ -288,26 +317,33 @@ export const HOT_POOL_NAMES = HOT_POOL.map(p => p.name) const HOT_BY_NAME = new Map(HOT_POOL.map(p => [p.name, p.category])) -/** 首次初始化：把内置热门清单写入 DB（powered by settings 标记，幂等） */ +/** 首次初始化：把内置热门清单写入 DB（过滤已缓存包，直接置 done；幂等） */ export function seedIfNeeded() { if (getSetting('pkglist_seeded', '') === '1') return 0 + const cached = listCachedPkgNames() const ins = db.prepare( - 'INSERT OR IGNORE INTO pkglist (pkg_name, version, category, status, auto, created_at) VALUES (?, NULL, ?, ?, 1, ?)', + 'INSERT OR IGNORE INTO pkglist (pkg_name, version, category, status, auto, cached, created_at) VALUES (?, NULL, ?, ?, 1, ?, ?)', ) - let n = 0 + let n = 0, skipped = 0 db.transaction(() => { for (const p of HOT_POOL) { - n += ins.run(p.name, p.category, 'pending', nowIso()).changes + if (cached.has(p.name)) { + // 已缓存：直接插入并置 done（auto=1），不计入待拉取 + ins.run(p.name, p.category, 'done', 1, nowIso()) + skipped++ + } else { + n += ins.run(p.name, p.category, 'pending', 0, nowIso()).changes + } } setSetting('pkglist_seeded', '1') })() - console.log(`[pkglist] 内置热门清单初始化完成：写入 ${n} 条`) + console.log(`[pkglist] 内置热门清单初始化完成：写入 ${n} 条，跳过（已缓存）${skipped} 条`) return n } /** 全部清单条目（id desc） */ export function getAllRows() { - return db.prepare('SELECT id, pkg_name, version, category, status, auto, created_at FROM pkglist ORDER BY id DESC').all() + return db.prepare('SELECT id, pkg_name, version, category, status, auto, cached, created_at FROM pkglist ORDER BY id DESC').all() } /** pending 条目（待拉取） */ @@ -337,6 +373,7 @@ export function getPkgList() { version: r.version || '', // 空 = 最新版 category: r.category || '', status: r.status, + cached: !!r.cached, auto: !!r.auto, createdAt: r.created_at, })) @@ -350,10 +387,13 @@ export function addPkg({ pkgName, version = '', category = '' }) { const exists = db.prepare('SELECT 1 FROM pkglist WHERE pkg_name = ?').get(name) if (exists) return { ok: false, message: `「${name}」已在清单中` } const cat = String(category || '').trim() || HOT_BY_NAME.get(name) || '其他' - db.prepare('INSERT INTO pkglist (pkg_name, version, category, status, auto, created_at) VALUES (?, ?, ?, ?, 0, ?)') - .run(name, String(version || '').trim() || null, cat, 'pending', nowIso()) - addLog({ action: 'list-add', pkgName: name, deletedVersions: JSON.stringify({ version: version || null, category: cat }) }) - return { ok: true } + // 插入时自动检测：storage 中已有则直接置 done，否则 pending + const cached = listCachedPkgNames() + const status = cached.has(name) ? 'done' : 'pending' + db.prepare('INSERT INTO pkglist (pkg_name, version, category, status, auto, cached, created_at) VALUES (?, ?, ?, ?, 0, ?, ?)') + .run(name, String(version || '').trim() || null, cat, status, status === 'done' ? 1 : 0, nowIso()) + addLog({ action: 'list-add', pkgName: name, deletedVersions: JSON.stringify({ version: version || null, category: cat, preCached: status === 'done' }) }) + return { ok: true, status } } /** 编辑条目（改包名/版本/品类；内容变更后重置为 pending，相当于清单写入新内容） */ @@ -370,8 +410,10 @@ export function editPkg(id, { pkgName, version, category }) { const cat = category != null && String(category).trim() ? String(category).trim() : row.category // 内容有变化 → 重置 pending（新目标需要重新拉取；否则保持原状态） const changed = name !== row.pkg_name || ver !== row.version || cat !== row.category - db.prepare('UPDATE pkglist SET pkg_name = ?, version = ?, category = ?, status = ? WHERE id = ?') - .run(name, ver, cat, changed ? 'pending' : row.status, row.id) + const newStatus = changed ? 'pending' : row.status + const newCached = newStatus === 'done' ? (row.cached || 0) : 0 + db.prepare('UPDATE pkglist SET pkg_name = ?, version = ?, category = ?, status = ?, cached = ? WHERE id = ?') + .run(name, ver, cat, newStatus, newCached, row.id) if (changed) addLog({ action: 'list-edit', pkgName: name, deletedVersions: JSON.stringify({ from: row.pkg_name, version: ver, category: cat }) }) return { ok: true } } @@ -388,7 +430,7 @@ export function removePkg(id) { /** 定时更新清单：把内置热门池中尚未收录的包批量加入（pending，auto 来源），返回新增条数 */ export function addFromHotPool() { const ins = db.prepare( - 'INSERT OR IGNORE INTO pkglist (pkg_name, version, category, status, auto, created_at) VALUES (?, NULL, ?, ?, 1, ?)', + 'INSERT OR IGNORE INTO pkglist (pkg_name, version, category, status, auto, cached, created_at) VALUES (?, NULL, ?, ?, 1, 0, ?)', ) let n = 0 db.transaction(() => { @@ -519,14 +561,22 @@ export async function updateFromAllSources() { } catch (e) { // 榜单解析异常 → 仅内置池 } + // 定时新增前同样做存量去重：storage 已缓存的包直接置 done（不重复拉取），其余置 pending + const cached = listCachedPkgNames() const ins = db.prepare( - 'INSERT OR IGNORE INTO pkglist (pkg_name, version, category, status, auto, created_at) VALUES (?, NULL, ?, ?, 1, ?)', + 'INSERT OR IGNORE INTO pkglist (pkg_name, version, category, status, auto, cached, created_at) VALUES (?, NULL, ?, ?, 1, ?, ?)', ) - let added = 0 + let added = 0, skipped = 0 try { db.transaction(() => { for (const [name, category] of merged) { - added += ins.run(name, category, 'pending', nowIso()).changes + const isCached = cached.has(name) + const status = isCached ? 'done' : 'pending' + const changes = ins.run(name, category, status, isCached ? 1 : 0, nowIso()).changes + if (changes) { + if (status === 'done') skipped++ + else added++ + } } })() } catch (e) { @@ -534,12 +584,39 @@ export async function updateFromAllSources() { return { added: 0, rankSource: rankSource || '' } } const note = rankSource - ? `定时收录热门 npm 包：内置池+榜单 新增 ${added} 个（榜单源 ${rankSource}）` - : `定时收录热门 npm 包：内置池 新增 ${added} 个（npm-search 榜单不可用，已降级）` - if (added > 0 || rankSource) console.log(`[pkglist] ${note}`) + ? `定时收录热门 npm 包：内置池+榜单 新增 ${added} 个、已缓存 ${skipped} 个（榜单源 ${rankSource}）` + : `定时收录热门 npm 包：内置池 新增 ${added} 个、已缓存 ${skipped} 个（npm-search 榜单不可用，已降级）` + if (added > 0 || skipped > 0 || rankSource) console.log(`[pkglist] ${note}`) return { added, rankSource: rankSource || '' } } +/** 按包名将清单条目标记为 done（手动去重时用） */ +export function markDoneByName(pkgName) { + return db.prepare("UPDATE pkglist SET status = 'done', cached = 1 WHERE pkg_name = ? AND status != 'done'").run(pkgName).changes +} + +/** + * 手动去重：把清单中「storage 已缓存」的 pending 条目直接置 done， + * 避免对存量包重复拉取。返回处理条数与剩余 pending 数。 + */ +export function dedupPkgList() { + const cached = listCachedPkgNames() + const rows = db.prepare("SELECT id, pkg_name FROM pkglist WHERE status = 'pending'").all() + const upd = db.prepare("UPDATE pkglist SET status = 'done', cached = 1 WHERE id = ?") + let matched = 0 + db.transaction(() => { + for (const r of rows) { + if (cached.has(r.pkg_name)) { + upd.run(r.id) + matched++ + } + } + })() + console.log(`[pkglist] 手动去重：${matched} 条已缓存条目置 done，剩余 pending ${rows.length - matched} 条`) + if (matched > 0) addLog({ action: 'list-dedup', pkgName: null, deletedCount: matched }) + return { matched, pending: rows.length - matched } +} + // ================= 定时配置 ================= const DEFAULT_CONFIG = { diff --git a/backend/server.js b/backend/server.js index ebbde25..3c11872 100644 --- a/backend/server.js +++ b/backend/server.js @@ -30,7 +30,7 @@ import semverCompare from './semver.js' import { syncState, runSyncLatest } from './sync-latest.js' import { seedIfNeeded, countPending, getPkgList, addPkg, editPkg, removePkg, - getConfig, saveConfig, startPkgListDaemon, updateFromAllSources, + getConfig, saveConfig, startPkgListDaemon, updateFromAllSources, dedupPkgList, } from './pkglist.js' import { pkgPullState, runPkgListPull } from './pkglist-pull.js' @@ -467,6 +467,12 @@ app.delete('/api/pkglist/:id', (req, res) => { res.json({ ok: true, ...getPkgList() }) }) +// —— API：清单手动去重（将 storage 已缓存但清单仍为 pending 的条目置 done，避免重复拉取） —— +app.post('/api/pkglist/dedup', (req, res) => { + const r = dedupPkgList() + res.json({ ok: true, ...r, ...getPkgList() }) +}) + // —— API：按清单手动拉取（同步启动后台任务，不阻塞；失败自动重试 3 次；清单失效时提示） —— app.post('/api/pkglist/pull', (req, res) => { if (isDemoMode()) { diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 62ed597..df3d9c1 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -487,6 +487,18 @@ return r.json() }) return r.json() }, + + /** 手动去重：将 storage 已缓存但清单仍为 pending 的条目置 done */ + async dedupPkgList() { + if (USE_MOCK) { + await delay() + const matched = 0 // mock 无 storage 概念，视为 0 + const pending = mock.pkglistItems.filter(i => i.status === 'pending').length + return { ok: true, matched, pending, items: clone(mock.pkglistItems) } + } + const r = await authFetch('/api/pkglist/dedup', { method: 'POST' }) + return r.json() + }, } export default api diff --git a/frontend/src/components/PkgListDrawer.vue b/frontend/src/components/PkgListDrawer.vue index 6fd10b1..1e40349 100644 --- a/frontend/src/components/PkgListDrawer.vue +++ b/frontend/src/components/PkgListDrawer.vue @@ -12,7 +12,9 @@ const props = defineProps({ }) const emit = defineEmits(['update:visible', 'pulled']) -const { state, refreshPkgList, addPkgItem, updatePkgItem, deletePkgItem } = useStore() +const { state, refreshPkgList, addPkgItem, updatePkgItem, deletePkgItem, dedupPkgList } = useStore() + +const dedupInProgress = ref(false) // —— 新增 / 编辑弹窗 —— const formOpen = ref(false) @@ -77,6 +79,25 @@ async function doDelete(item) { } } +async function doDedup() { + if (dedupInProgress.value) return + dedupInProgress.value = true + try { + const r = await dedupPkgList() + if (r && r.ok) { + ElMessage.success(r.matched > 0 + ? `去重完成：${r.matched} 条已缓存条目标记为完成，剩余待拉取 ${r.pending} 条` + : '去重完成：无已缓存条目需要标记') + } else { + ElMessage.warning((r && r.message) || '去重失败') + } + } catch (e) { + ElMessage.error(e?.message || '去重请求失败') + } finally { + dedupInProgress.value = false + } +} + function close() { emit('update:visible', false) } @@ -107,6 +128,7 @@ watch(() => props.visible, (v) => {
+
@@ -128,8 +150,8 @@ watch(() => props.visible, (v) => { {{ it.category || '—' }} {{ it.auto ? '内置' : '手工' }} - - {{ it.status === 'done' ? '✅ 已完成' : '⏳ 待拉取' }} + + {{ it.status === 'done' ? (it.cached ? '✅ 已在缓存' : '✅ 已完成') : '⏳ 待拉取' }} @@ -282,6 +304,7 @@ watch(() => props.visible, (v) => { .pld-status { font-size: 11.5px; font-weight: 600; } .pld-status.done { color: var(--green-deep); } .pld-status.pending { color: var(--orange); } +.pld-status.cached { color: #3b82f6; } .pld-ops { white-space: nowrap; } .pld-mini { padding: 4px 10px; diff --git a/frontend/src/store/index.js b/frontend/src/store/index.js index 0845c2a..8f236fe 100644 --- a/frontend/src/store/index.js +++ b/frontend/src/store/index.js @@ -133,6 +133,23 @@ async function doRefresh() { await pollScanProgress() await refreshPackages() // 扫描完成后重新拉取完整数据 } + + // 刷新后检查一键拉取是否仍在运行（浏览器刷新后恢复沙漏旋转） + const sp = await api.getSyncProgress() + if (sp.running) { + state.syncProgress.running = true + state.syncProgress.done = false + state.syncProgress.total = sp.total || 0 + state.syncProgress.processed = sp.processed || 0 + state.syncProgress.success = sp.success || 0 + state.syncProgress.skipped = sp.skipped || 0 + state.syncProgress.failed = sp.failed || 0 + state.syncProgress.failedList = sp.failedList || [] + state.syncProgress.current = sp.current || null + state.syncProgress.error = sp.error || null + pollSyncProgress() + } + await Promise.all([refreshThresholds(), refreshLogs()]) } catch (e) { // 网络/后端异常统一提示一次，避免向上穿透成未处理的 Promise rejection @@ -391,6 +408,16 @@ async function savePkgListConfig(payload) { return r } +async function dedupPkgList() { + const r = await api.dedupPkgList() + if (r && r.ok) { + state.pkgList.items = r.items || state.pkgList.items + state.pkgList.active = countPending() > 0 + if (r.matched > 0) ElMessage.success(`${r.matched} 个已缓存条目标记为完成`) + } + return r +} + export function useStore() { return { state, @@ -415,5 +442,6 @@ scanNow, pollPkgPullProgress, refreshPkgListConfig, savePkgListConfig, + dedupPkgList, } } diff --git a/frontend/src/views/SearchView.vue b/frontend/src/views/SearchView.vue index ff9c9c6..aa62d13 100644 --- a/frontend/src/views/SearchView.vue +++ b/frontend/src/views/SearchView.vue @@ -228,18 +228,20 @@ async function onCleanConfirmed(res) { // —— 一键拉取存量包最新版本 —— const summaryShown = ref(false) -async function doSyncLatest() { - const sp = state.syncProgress - if (sp.running) return - try { - await ElMessageBox.confirm( - '将查询上游 npm registry，为存量包拉取每个包的最新版本（已缓存最新版会自动跳过）。', - '拉取最新版本', - { type: 'info', confirmButtonText: '开始拉取', cancelButtonText: '取消', closeOnClickModal: false }, - ) - } catch { - return // 用户取消 - } +const syncConfirmOpen = ref(false) +const syncResultOpen = ref(false) +const syncResultDetail = ref('') +const syncResultFailed = ref(false) + +function openSyncConfirm() { + if (state.syncProgress.running) return + syncConfirmOpen.value = true +} +function closeSyncConfirm() { + syncConfirmOpen.value = false +} +async function confirmSyncLatest() { + closeSyncConfirm() summaryShown.value = false const res = await syncLatest() if (!res.ok) return @@ -250,6 +252,10 @@ async function doSyncLatest() { // 任务异步执行，完成后由下方 watch 弹出汇总 } +function closeSyncResult() { + syncResultOpen.value = false +} + watch( () => state.syncProgress.running, async (running, prev) => { @@ -261,18 +267,17 @@ watch( detail.push(`❌ 拉取失败 ${sp.failed} 个`) if (sp.failedList.length) detail.push(`失败明细：\n${sp.failedList.map(f => `· ${f}`).join('\n')}`) } - try { - await ElMessageBox.alert(detail.join('\n'), sp.failed ? '拉取完成（部分失败）' : '拉取完成', { - confirmButtonText: '知道了', - type: sp.failed ? 'warning' : 'success', - }) - } catch { - /* 用户关闭弹窗无需处理 */ - } + syncResultDetail.value = detail.join('\n') + syncResultFailed.value = !!sp.failed + syncResultOpen.value = true } }, ) +async function doSyncLatest() { + openSyncConfirm() +} + // —— npm 包清单：查看/编辑入口 + 按清单手动拉取（后台执行，失败自动重试 3 次） —— const pkgListOpen = ref(false) const pullLoading = ref(null) @@ -512,6 +517,33 @@ const pendingCount = computed(() => state.pkgList.items.filter(i => i.status === @confirmed="onCleanConfirmed" /> + + + +
+
+
拉取最新版本
+
将查询上游 npm registry，为存量包拉取每个包的最新版本（已缓存最新版会自动跳过）。
+
+ + +
+
+
+ + + + +
+
+
{{ syncResultFailed ? '拉取完成（部分失败）' : '拉取完成' }}
+ 
{{ syncResultDetail }}
+
+ +
+
+
+