fix: 扫描缓存与真实大小计算,解决页面加载慢与数据虚增

- scanner: 大小改为统计实际 .tgz 文件(兼容 {version}.tgz 与 {pkg}-{version}.tgz 命名),
  不再用 unpackedSize 兜底虚增数倍
- server: 扫描结果内存缓存(首次懒扫描,后续请求秒回),新增 POST /api/scan 手动重扫,
  /api/packages 返回 scannedAt 时间戳,clean/删版本后自动失效缓存
- 前端: 概览页新增「重新扫描」按钮 + 「基于 xxx 扫描」时间展示;Top30 在真实环境
  (无访问统计)降级为「缓存占用 Top 30」按大小排序
- compose: 数据卷改 named volume,修复非 root 容器 SQLITE_CANTOPEN
This commit is contained in:
陈银军
2026-09-19 17:57:58 +08:00
parent 72722ab6bf
commit f84f21574f
8 changed files with 147 additions and 41 deletions
+26 -9
View File
@@ -118,19 +118,36 @@ export function scanStorage(storageDir) {
const timeObj = meta.time || {} const timeObj = meta.time || {}
const distTags = meta['dist-tags'] || {} const distTags = meta['dist-tags'] || {}
// 物理 .tgz 文件:先列目录建立文件名 → 大小索引
// (Verdaccio 各版本命名不同:{version}.tgz 或 {pkgname}-{version}.tgz,不猜、直接扫)
const tgzSizes = new Map()
let tgzTotal = 0
try {
for (const f of fs.readdirSync(pkgDir)) {
if (!f.endsWith('.tgz')) continue
const size = fs.statSync(path.join(pkgDir, f)).size
tgzTotal += size
tgzSizes.set(f.slice(0, -4), size)
}
} catch {}
const versionsDetail = [] const versionsDetail = []
let totalBytes = 0 // 总大小以实际 .tgz 文件为准;unpackedSize 是解压后大小,会数倍虚增,仅作无物理文件时的兜底
const totalBytes = tgzTotal
for (const [v, info] of Object.entries(versionsObj)) { for (const [v, info] of Object.entries(versionsObj)) {
// tgz 物理文件:Verdaccio 约定 {pkgDir}/{version}.tgz let size = tgzSizes.get(v)
const tgz = path.join(pkgDir, `${v}.tgz`) if (typeof size !== 'number') {
let size = 0 // 兼容 {pkgname}-{version}.tgz 命名:用元数据 dist.tarball 的文件名精确匹配
try { const tb = info && info.dist && info.dist.tarball
size = fs.statSync(tgz).size const bn = tb ? String(tb).split('/').pop() : ''
} catch { if (bn.endsWith('.tgz')) {
// 无物理文件:用元数据 dist.tarball 大小兜底(无则 0) const hit = tgzSizes.get(bn.slice(0, -4))
if (typeof hit === 'number') size = hit
}
}
if (typeof size !== 'number') {
size = (info && info.dist && typeof info.dist.unpackedSize === 'number') ? info.dist.unpackedSize : 0 size = (info && info.dist && typeof info.dist.unpackedSize === 'number') ? info.dist.unpackedSize : 0
} }
totalBytes += size
versionsDetail.push({ versionsDetail.push({
version: parseVersion(v), version: parseVersion(v),
size: fmtMb(size), size: fmtMb(size),
+31 -3
View File
@@ -38,14 +38,29 @@ function isDemoMode() {
return process.env.VCM_DEMO === '1' || !isValidStorage(currentStorage()) return process.env.VCM_DEMO === '1' || !isValidStorage(currentStorage())
} }
// —— 扫描结果缓存:真实目录全量扫描 IO 重,缓存避免每次请求/刷新页面重扫 ——
let rawCache = null
let rawCacheAt = null
function getRawPackages() {
if (isDemoMode()) return demoPackages()
if (!rawCache) {
rawCache = scanStorage(currentStorage())
rawCacheAt = new Date().toISOString()
}
return rawCache
}
function invalidateScan() {
rawCache = null
rawCacheAt = null
}
// 构建包列表(统一状态计算 + 排序 + 忽略标记) // 构建包列表(统一状态计算 + 排序 + 忽略标记)
function buildPackages(keyword = '') { function buildPackages(keyword = '') {
const threshold = Number(getSetting('version_threshold', DEFAULT_VERSION_THRESHOLD)) const threshold = Number(getSetting('version_threshold', DEFAULT_VERSION_THRESHOLD))
const ignoredRows = getIgnoredRows() const ignoredRows = getIgnoredRows()
const ignoredSet = new Set(ignoredRows.map(r => r.pkg_name)) const ignoredSet = new Set(ignoredRows.map(r => r.pkg_name))
const ignoredAtMap = Object.fromEntries(ignoredRows.map(r => [r.pkg_name, (r.ignored_at || '').slice(0, 10)])) const ignoredAtMap = Object.fromEntries(ignoredRows.map(r => [r.pkg_name, (r.ignored_at || '').slice(0, 10)]))
const demo = isDemoMode() const raw = getRawPackages()
const raw = demo ? demoPackages() : scanStorage(currentStorage())
let list = raw.map(p => { let list = raw.map(p => {
let status let status
@@ -141,17 +156,28 @@ app.get('/api/packages', (req, res) => {
const demo = isDemoMode() const demo = isDemoMode()
const storage = currentStorage() const storage = currentStorage()
const validation = demo ? { ok: true, message: 'demo' } : validateStorage(storage) const validation = demo ? { ok: true, message: 'demo' } : validateStorage(storage)
const packages = buildPackages(req.query.keyword || '') // 先构建(触发懒扫描并填充缓存)
res.json({ res.json({
demo, demo,
storageDir: storage || null, storageDir: storage || null,
storageValid: validation.ok, storageValid: validation.ok,
storageMessage: validation.message, storageMessage: validation.message,
threshold: Number(getSetting('version_threshold', DEFAULT_VERSION_THRESHOLD)), threshold: Number(getSetting('version_threshold', DEFAULT_VERSION_THRESHOLD)),
packages: buildPackages(req.query.keyword || ''), scannedAt: demo ? new Date().toISOString() : (rawCacheAt || null),
packages,
...(demo ? demoSummary() : {}), ...(demo ? demoSummary() : {}),
}) })
}) })
// —— API:手动重新扫描缓存目录 ——
app.post('/api/scan', (req, res) => {
invalidateScan()
const demo = isDemoMode()
const scannedAt = new Date().toISOString()
if (!demo) getRawPackages() // 同步重扫并填充缓存
res.json({ ok: true, demo, scannedAt })
})
// —— API:清理 —— // —— API:清理 ——
app.post('/api/clean', (req, res) => { app.post('/api/clean', (req, res) => {
const { pkgName, keep } = req.body || {} const { pkgName, keep } = req.body || {}
@@ -168,6 +194,7 @@ app.post('/api/clean', (req, res) => {
deletedCount: result.deletedCount, deletedCount: result.deletedCount,
freedSpace: result.freedSpace, freedSpace: result.freedSpace,
}) })
invalidateScan()
} }
res.json(result) res.json(result)
}) })
@@ -187,6 +214,7 @@ app.delete('/api/packages/:pkgName/versions/:version', (req, res) => {
deletedCount: 1, deletedCount: 1,
freedSpace: result.freedSpace, freedSpace: result.freedSpace,
}) })
invalidateScan()
} }
res.json(result) res.json(result)
}) })
+9 -15
View File
@@ -1,22 +1,16 @@
# Verdaccio 缓存可视化管理面板(§12.2 Docker 部署) # Verdaccio 缓存可视化管理面板(§12.2 Docker 部署)
# 与 Verdaccio 同主机:storage 只读挂载,manager-data 读写(SQLite 数据) # Verdaccio 已独立部署,此处仅运行面板:缓存目录只读挂载,manager-data 读写(SQLite 数据)
services: services:
verdaccio:
image: verdaccio/verdaccio
restart: unless-stopped
volumes:
- "./storage:/verdaccio/storage" # Verdaccio 读写缓存目录
verdaccio-manager: verdaccio-manager:
build: ./ image: verdaccio-cache-manager:latest # 已导入的 amd64 镜像(docker load 后直接可用)
restart: unless-stopped restart: always
environment: environment:
- VERDACCIO_STORAGE=/verdaccio/storage # 只读扫描该目录 - VERDACCIO_STORAGE=/verdaccio/storage # 容器内路径,对应下方只读挂载的右侧
- PORT=3000 - PORT=3000
volumes: volumes:
- "./storage:/verdaccio/storage:ro" # 面板只读挂载 - "/volume2/docker/verdaccio_verdaccio-1/verdaccio_storage:/verdaccio/storage:ro" # Verdaccio 缓存目录(只读)
- "./manager-data:/app/data" # 读写:SQLite 数据 - "vcm-data:/app/data" # 读写:SQLite 数据(named volume,避免宿主目录权限问题)
ports: ports:
- "3000:3000" - "42338:3000"
depends_on: volumes:
- verdaccio vcm-data:
+13 -3
View File
@@ -67,11 +67,21 @@ const api = {
let list = clone(mockState.packages) let list = clone(mockState.packages)
list = list.map(p => ({ ...p, status: computeStatus(p, mockState.versionThreshold) })) list = list.map(p => ({ ...p, status: computeStatus(p, mockState.versionThreshold) }))
if (kw) list = list.filter(p => p.name.toLowerCase().includes(kw)) if (kw) list = list.filter(p => p.name.toLowerCase().includes(kw))
return sortPackages(list) return { packages: sortPackages(list), scannedAt: new Date().toISOString() }
} }
const r = await fetch(`/api/packages?keyword=${encodeURIComponent(keyword)}`) const r = await fetch(`/api/packages?keyword=${encodeURIComponent(keyword)}`)
const { packages } = await r.json() const data = await r.json()
return packages return { packages: data.packages, scannedAt: data.scannedAt || null }
},
/** 手动触发后端重新扫描缓存目录 */
async scanNow() {
if (USE_MOCK) {
await delay(300)
return { ok: true, scannedAt: new Date().toISOString() }
}
const r = await fetch('/api/scan', { method: 'POST' })
return r.json()
}, },
async getThresholds() { async getThresholds() {
+13 -5
View File
@@ -1,12 +1,20 @@
<script setup> <script setup>
// 访问次数 Top 30 排行 // 排行 Top 30:有访问统计按热度,否则降级为缓存占用(真实环境无访问统计)
defineProps({ top30: { type: Array, default: () => [] } }) const props = defineProps({
top30: { type: Array, default: () => [] },
top30By: { type: String, default: 'visits' },
})
const isVisits = props.top30By === 'visits'
function fmtSize(mb) {
if (mb >= 1024) return `${(mb / 1024).toFixed(1)} GB`
return `${mb.toFixed(1)} MB`
}
</script> </script>
<template> <template>
<div class="top30"> <div class="top30">
<div class="top30-head"> <div class="top30-head">
<div class="t"><span class="ico">🏆</span>访问次数 Top 30<span class="tag30">按热度排序</span></div> <div class="t"><span class="ico">🏆</span>{{ isVisits ? '访问次数 Top 30' : '缓存占用 Top 30' }}<span class="tag30">{{ isVisits ? '按热度排序' : '按占用空间排序' }}</span></div>
</div> </div>
<div class="tbl-scroll"> <div class="tbl-scroll">
<table class="rank-tbl"> <table class="rank-tbl">
@@ -14,7 +22,7 @@ defineProps({ top30: { type: Array, default: () => [] } })
<tr> <tr>
<th style="width: 70px">排名</th> <th style="width: 70px">排名</th>
<th>包名</th> <th>包名</th>
<th style="width: 110px">访问次数</th> <th style="width: 130px">{{ isVisits ? '访问次数' : '占用空间' }}</th>
<th>热度</th> <th>热度</th>
</tr> </tr>
</thead> </thead>
@@ -31,7 +39,7 @@ defineProps({ top30: { type: Array, default: () => [] } })
</span> </span>
</span> </span>
</td> </td>
<td><span class="rank-visit">{{ (p.visits || 0).toLocaleString() }}</span></td> <td><span class="rank-visit">{{ isVisits ? (p.visits || 0).toLocaleString() : fmtSize(p.size) }}</span></td>
<td> <td>
<div class="heat"> <div class="heat">
<div class="heat-bar"><i :style="{ width: p.heat + '%' }"></i></div> <div class="heat-bar"><i :style="{ width: p.heat + '%' }"></i></div>
+25 -3
View File
@@ -12,6 +12,7 @@ const state = reactive({
threshold: { versionThreshold: 15, sizeThreshold: 500, exceptions: [] }, threshold: { versionThreshold: 15, sizeThreshold: 500, exceptions: [] },
logs: [], logs: [],
loading: false, loading: false,
scannedAt: null,
storageDir: localStorage.getItem('vcm-storage-dir') || '', storageDir: localStorage.getItem('vcm-storage-dir') || '',
}) })
@@ -23,8 +24,13 @@ const stats = computed(() => {
const cleanable = valid.filter(p => p.status === 'warn').reduce((s, p) => s + p.size, 0) const cleanable = valid.filter(p => p.status === 'warn').reduce((s, p) => s + p.size, 0)
const maxPkg = [...valid].sort((a, b) => b.size - a.size)[0] || null const maxPkg = [...valid].sort((a, b) => b.size - a.size)[0] || null
const oldestPkg = [...valid].sort((a, b) => (a.lastUpdated || '').localeCompare(b.lastUpdated || ''))[0] || null const oldestPkg = [...valid].sort((a, b) => (a.lastUpdated || '').localeCompare(b.lastUpdated || ''))[0] || null
const top30 = [...valid].sort((a, b) => (b.visits || 0) - (a.visits || 0)).slice(0, 30) // 真实环境无访问统计(visits 全 0)时,Top30 降级为「缓存占用 Top 30」
const hasVisits = valid.some(p => (p.visits || 0) > 0)
const top30 = [...valid]
.sort((a, b) => (hasVisits ? (b.visits || 0) - (a.visits || 0) : b.size - a.size))
.slice(0, 30)
const maxVisit = Math.max(...top30.map(p => p.visits || 0), 1) const maxVisit = Math.max(...top30.map(p => p.visits || 0), 1)
const maxSize = Math.max(...top30.map(p => p.size), 1)
return { return {
pkgCount: valid.length, pkgCount: valid.length,
totalSize, totalSize,
@@ -34,7 +40,13 @@ const stats = computed(() => {
cleanableText: fmtSize(cleanable), cleanableText: fmtSize(cleanable),
maxPkg, maxPkg,
oldestPkg, oldestPkg,
top30: top30.map(p => ({ ...p, heat: Math.round(((p.visits || 0) / maxVisit) * 100) })), top30: top30.map(p => ({
...p,
heat: hasVisits
? Math.round(((p.visits || 0) / maxVisit) * 100)
: Math.round((p.size / maxSize) * 100),
})),
top30By: hasVisits ? 'visits' : 'size',
} }
}) })
@@ -46,7 +58,16 @@ function fmtSize(mb) {
const fmtSizeFn = fmtSize const fmtSizeFn = fmtSize
async function refreshPackages(keyword = '') { async function refreshPackages(keyword = '') {
state.packages = await api.getPackages(keyword) const { packages, scannedAt } = await api.getPackages(keyword)
state.packages = packages
if (scannedAt) state.scannedAt = scannedAt
}
/** 手动重新扫描缓存目录 */
async function scanNow() {
const res = await api.scanNow()
if (res.ok) await refreshAll()
return res
} }
async function refreshThresholds() { async function refreshThresholds() {
@@ -113,6 +134,7 @@ export function useStore() {
refreshPackages, refreshPackages,
refreshThresholds, refreshThresholds,
refreshLogs, refreshLogs,
scanNow,
saveThresholds, saveThresholds,
ignorePkg, ignorePkg,
unignorePkg, unignorePkg,
+1
View File
@@ -111,6 +111,7 @@ html[data-theme="dark"] .ambient::after {
user-select: none; user-select: none;
} }
.btn:active { transform: scale(.95); } .btn:active { transform: scale(.95); }
.btn:disabled { background: var(--num-bg); color: var(--ink-faint); cursor: not-allowed; transform: none; box-shadow: none; }
.btn.clean { .btn.clean {
background: var(--grad-main); background: var(--grad-main);
color: #fff; color: #fff;
+29 -3
View File
@@ -11,13 +11,32 @@ import { useStore } from '../store'
import { iconOf } from '../mock/data.js' import { iconOf } from '../mock/data.js'
const router = useRouter() const router = useRouter()
const { state, stats, refreshAll, ignorePkg, unignorePkg, cleanPkg } = useStore() const { state, stats, refreshAll, ignorePkg, unignorePkg, cleanPkg, scanNow } = useStore()
const drawerPkg = ref(null) const drawerPkg = ref(null)
const drawerVisible = ref(false) const drawerVisible = ref(false)
const cleanVisible = ref(false) const cleanVisible = ref(false)
const cleanPkgData = ref(null) const cleanPkgData = ref(null)
const showTop30 = ref(true) const showTop30 = ref(true)
const scanning = ref(false)
// 本次数据基于何时扫描(后端扫描时间,缓存复用)
const fmtScanTime = computed(() => {
const t = state.scannedAt
if (!t) return ''
const d = new Date(t)
const p = n => String(n).padStart(2, '0')
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}`
})
async function rescan() {
scanning.value = true
try {
await scanNow()
} finally {
scanning.value = false
}
}
const warningExcess = computed(() => const warningExcess = computed(() =>
stats.value.warnCount stats.value.warnCount
@@ -94,13 +113,17 @@ function quickClean() {
</div> </div>
<!-- Top 30 排行(可收起) --> <!-- Top 30 排行(可收起) -->
<Top30Panel v-if="showTop30" :top30="top30WithIcon" /> <Top30Panel v-if="showTop30" :top30="top30WithIcon" :top30-by="stats.top30By" />
<!-- 包卡片列表:仅需关注项 --> <!-- 包卡片列表:仅需关注项 -->
<div class="sec-head"> <div class="sec-head">
<span class="sec-title"><span class="emo">⚠️</span>需要关注</span> <span class="sec-title"><span class="emo">⚠️</span>需要关注</span>
<span class="sec-sub">共 {{ attentionPackages.length }} 个需关注 · 预警置顶</span> <span class="sec-sub">共 {{ attentionPackages.length }} 个需关注 · 预警置顶</span>
<button class="link-all" @click="router.push('/search')">查看全部 →</button> <span class="sec-actions">
<span v-if="state.scannedAt" class="scan-stamp">🕐 基于 {{ fmtScanTime }} 扫描</span>
<button class="btn ghost mini" :disabled="scanning" @click="rescan">{{ scanning ? '⏳ 扫描中…' : '🔄 重新扫描' }}</button>
<button class="link-all" @click="router.push('/search')">查看全部 →</button>
</span>
</div> </div>
<div v-if="state.loading" class="loading-tip">⏳ 加载中…</div> <div v-if="state.loading" class="loading-tip">⏳ 加载中…</div>
@@ -151,6 +174,9 @@ function quickClean() {
.sec-title { font-size: 17px; font-weight: 700; display: flex; align-items: center; gap: 10px; } .sec-title { font-size: 17px; font-weight: 700; display: flex; align-items: center; gap: 10px; }
.sec-title .emo { font-size: 19px; } .sec-title .emo { font-size: 19px; }
.sec-sub { font-family: var(--num); font-size: 12px; color: var(--ink-faint); font-weight: 600; } .sec-sub { font-family: var(--num); font-size: 12px; color: var(--ink-faint); font-weight: 600; }
.sec-actions { display: flex; align-items: center; gap: 10px; margin-left: auto; }
.scan-stamp { font-family: var(--num); font-size: 11.5px; color: var(--ink-faint); font-weight: 600; }
.btn.mini { padding: 6px 13px; font-size: 12.5px; border-radius: 999px; }
.link-all { padding: 6px 14px; border: none; border-radius: 999px; background: var(--panel-soft); color: var(--orange); font-size: 13px; font-weight: 600; cursor: pointer; transition: transform .2s, box-shadow .2s; } .link-all { padding: 6px 14px; border: none; border-radius: 999px; background: var(--panel-soft); color: var(--orange); font-size: 13px; font-weight: 600; cursor: pointer; transition: transform .2s, box-shadow .2s; }
.link-all:hover { transform: translateY(-1px); box-shadow: var(--shadow); } .link-all:hover { transform: translateY(-1px); box-shadow: var(--shadow); }
.pkg-grid { display: grid; grid-template-columns: repeat(3, 1fr); gap: 16px; } .pkg-grid { display: grid; grid-template-columns: repeat(3, 1fr); gap: 16px; }