verdaccio-cache-manager/backend/scanner.js

/**
 * Verdaccio 缓存目录扫描（§12.1）
 * - storage 目录解析优先级：VERDACCIO_STORAGE 环境变量 → UI 设置（settings.storage_dir）
 * - 递归扫描目录内所有 package.json，解析 versions/time，匹配 .tgz 计算占用，支持 scoped 包
 * - 未配置或无效时进入 demo 模式（VCM_DEMO=1 或首次无配置）
 */
import fs from 'node:fs'
import path from 'node:path'
import { getSetting } from './db.js'

// —— storage 目录解析 ——
export function resolveStorageDir() {
  return process.env.VERDACCIO_STORAGE || getSetting('storage_dir') || null
}

export function isValidStorage(dir) {
  if (!dir) return false
  try {
    const st = fs.statSync(dir)
    if (!st.isDirectory()) return false
    fs.accessSync(dir, fs.constants.R_OK)
    return true
  } catch {
    return false
  }
}

export function validateStorage(dir) {
  if (!isValidStorage(dir)) return { ok: false, message: '目录不存在或不可读' }
  // 至少包含 package.json 或 .tgz 之一，才算有效缓存目录
  const files = listFiles(dir, 1)
  const hasPkg = files.some(f => f.endsWith('package.json'))
  const hasTgz = files.some(f => f.endsWith('.tgz'))
  if (!hasPkg && !hasTgz) {
    return { ok: false, message: '目录中未找到 package.json / .tgz，不是有效的 Verdaccio 缓存目录' }
  }
  return { ok: true, message: 'ok' }
}

function listFiles(dir, depth = Infinity, out = []) {
  let entries
  try {
    entries = fs.readdirSync(dir, { withFileTypes: true })
  } catch {
    return out
  }
  for (const e of entries) {
    if (e.name === 'node_modules' || e.name.startsWith('.') || e.name === '.DS_Store') continue
    const full = path.join(dir, e.name)
    if (e.isDirectory()) {
      if (depth > 0) listFiles(full, depth - 1, out)
    } else {
      out.push(full)
    }
  }
  return out
}

function readJson(file) {
  try {
    return JSON.parse(fs.readFileSync(file, 'utf8'))
  } catch {
    return null
  }
}

function fmtMb(bytes) {
  // 保留到 KB 级精度：toFixed(1) 会把 <0.05MB 的轻量包（几 KB~几十 KB）截成 0
  return Math.round((bytes / 1024 / 1024) * 1024) / 1024
}

// 目录名 → 包名（支持 scoped：@scope/name）
function dirToPkgName(segments) {
  if (segments.length >= 2 && segments[0].startsWith('@')) {
    return `${segments[0]}/${segments[1]}`
  }
  return segments[segments.length - 1]
}

// 从 package.json 所在目录向上推导包名
function pkgNameFromDir(pkgDir, storageDir) {
  const rel = path.relative(storageDir, pkgDir).split(path.sep)
  return dirToPkgName(rel)
}

function parseVersion(v) {
  // 去前缀 v、prerelease 后缀仅比较主体
  return (String(v).replace(/^v/, '') || '').trim()
}

// —— 日期口径（项目统一北京时间，避免 UTC slice 跨日偏差） ——
const SH_DATE = new Intl.DateTimeFormat('en-CA', {
  timeZone: 'Asia/Shanghai', year: 'numeric', month: '2-digit', day: '2-digit',
})
/** ISO 字符串 → 北京时间日期（YYYY-MM-DD）；time 字段为 UTC，直接 slice 会与本地跨日 */
function shDate(iso) {
  if (!iso) return null
  const d = new Date(iso)
  return Number.isNaN(d.getTime()) ? null : SH_DATE.format(d)
}
/** 本地时区日期（YYYY-MM-DD）：.tgz 文件 mtime 为本地时间，与前端 todayStr 同口径 */
function localDate(d) {
  const p = n => String(n).padStart(2, '0')
  return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}`
}
/** 最近更新 = max(元数据最新版本发布日, 最新 .tgz 缓存日) —— 今天拉取缓存也算「今天更新」 */
function latestDate(metaUpdated, tgzMtime) {
  const tgzUpdated = tgzMtime ? localDate(new Date(tgzMtime)) : null
  return [metaUpdated, tgzUpdated].filter(Boolean).sort().pop() || null
}

/**
 * 扫描 storage 目录
 * @returns {Array} 包列表（versionsDetail: [{version, size(MB), date}]）
 */
export function scanStorage(storageDir) {
  const packages = []
  if (!isValidStorage(storageDir)) return packages

  const allFiles = listFiles(storageDir)
  // 包目录 = 含 package.json 的目录 ∪ 含 .tgz 的目录（避免只缓存了 tgz、缺元数据的目录被漏算）
  const pkgDirs = new Set()
  for (const f of allFiles) {
    if (f.endsWith('package.json') || f.endsWith('.tgz')) pkgDirs.add(path.dirname(f))
  }
  const storageRoot = path.resolve(storageDir)

  for (const pkgDir of pkgDirs) {
    const pkgFile = path.join(pkgDir, 'package.json')

    // 物理 .tgz 文件：先列目录建立文件名 → 大小索引
    // （Verdaccio 各版本命名不同：{version}.tgz 或 {pkgname}-{version}.tgz，不猜、直接扫）
    // 先统计再读元数据：即使元数据损坏也能算出真实占用
    const tgzSizes = new Map()
    let tgzTotal = 0
    let tgzLatest = 0 // 最新 .tgz 文件 mtime（毫秒）：今天缓存了新版本也算「今天更新」
    try {
      for (const f of fs.readdirSync(pkgDir)) {
        if (!f.endsWith('.tgz')) continue
        const st = fs.statSync(path.join(pkgDir, f))
        const size = st.size
        tgzTotal += size
        tgzSizes.set(f.slice(0, -4), size)
        if (st.mtimeMs > tgzLatest) tgzLatest = st.mtimeMs
      }
    } catch {}

    const meta = readJson(pkgFile)
    if (!meta) {
      // 元数据缺失/损坏：标记异常包（真实占用按物理 .tgz 计算），不影响其他包
      packages.push({
        name: pkgNameFromDir(pkgDir, storageRoot),
        status: 'broken',
        versions: 0,
        size: fmtMb(tgzTotal),
        lastUpdated: latestDate(null, tgzLatest),
        versionsDetail: [],
      })
      continue
    }

    const versionsObj = meta.versions || {}
    const timeObj = meta.time || {}
    const distTags = meta['dist-tags'] || {}
    // 包信息（来自本地元数据，离线可用）：介绍/主页/仓库/作者/许可
    const description = typeof meta.description === 'string' ? meta.description : null
    const homepage = typeof meta.homepage === 'string' && meta.homepage ? meta.homepage : null
    const repository = typeof meta.repository === 'string'
      ? meta.repository
      : (meta.repository && typeof meta.repository.url === 'string' ? meta.repository.url : null)
    const author = typeof meta.author === 'string'
      ? meta.author
      : (meta.author && typeof meta.author.name === 'string' ? meta.author.name : null)
    const license = typeof meta.license === 'string' ? meta.license : null

    const versionsDetail = []
    // 总大小以实际 .tgz 文件为准
    const totalBytes = tgzTotal
    for (const [v, info] of Object.entries(versionsObj)) {
      let size = tgzSizes.get(v)
      if (typeof size !== 'number') {
        // 兼容 {pkgname}-{version}.tgz 命名：用元数据 dist.tarball 的文件名精确匹配
        const tb = info && info.dist && info.dist.tarball
        const bn = tb ? String(tb).split('/').pop() : ''
        if (bn.endsWith('.tgz')) {
          const hit = tgzSizes.get(bn.slice(0, -4))
          if (typeof hit === 'number') size = hit
        }
      }
      // 无物理 .tgz 文件的版本是元数据残留（实体已被清理），不计数，避免版本数虚高
      if (typeof size !== 'number') continue
      versionsDetail.push({
        version: parseVersion(v),
        size: fmtMb(size),
        date: timeObj[v] ? shDate(timeObj[v]) : null,
      })
    }
    // 按 semver 数值排序（旧 → 新）
    versionsDetail.sort((a, b) => a.version.localeCompare(b.version, undefined, { numeric: true }))

    // 最近更新：取「元数据最新版本发布日」与「最新 .tgz 缓存日」中较新者（今天拉缓存也算今天更新）
    const metaUpdated = Object.keys(timeObj).length
      ? Object.entries(timeObj).map(([, iso]) => shDate(iso)).filter(Boolean).sort().pop()
      : null
    const lastUpdated = latestDate(metaUpdated, tgzLatest)

    packages.push({
      name: pkgNameFromDir(pkgDir, storageRoot),
      versions: versionsDetail.length,
      size: fmtMb(totalBytes),
      lastUpdated,
      visits: 0, // 真实环境无访问统计，界面不依赖真实值
      distTags,
      description,
      homepage,
      repository,
      author,
      license,
      versionsDetail,
    })
  }

  return packages
}

/**
 * 异步扫描（分片让出事件循环 + 进度回调）
 * 用于后台重扫：扫描期间仍能响应 /api/scan/progress 等请求，前端可实时展示进度
 */
export async function scanStorageAsync(storageDir, onProgress) {
  const packages = []
  if (!isValidStorage(storageDir)) return packages

  const allFiles = listFiles(storageDir)
  // 包目录 = 含 package.json 的目录 ∪ 含 .tgz 的目录（避免只缓存了 tgz、缺元数据的目录被漏算）
  const pkgDirs = new Set()
  for (const f of allFiles) {
    if (f.endsWith('package.json') || f.endsWith('.tgz')) pkgDirs.add(path.dirname(f))
  }
  const storageRoot = path.resolve(storageDir)
  let scanned = 0
  let scannedBytes = 0 // 所有文件字节和（总占用，与 du 同口径）
  let tgzBytes = 0     // 仅 .tgz 字节和（包粒度用）

  for (const pkgDir of pkgDirs) {
    // 每处理完一个包目录就让出事件循环，保证进度接口可实时响应
    await new Promise(r => setImmediate(r))
    const pkgFile = path.join(pkgDir, 'package.json')

    const tgzSizes = new Map()
    let tgzTotal = 0
    let tgzLatest = 0 // 最新 .tgz 文件 mtime（毫秒）：今天缓存了新版本也算「今天更新」
    try {
      for (const f of fs.readdirSync(pkgDir)) {
        const st = fs.statSync(path.join(pkgDir, f))
        const size = st.size
        scannedBytes += size
        if (f.endsWith('.tgz')) {
          tgzTotal += size
          tgzSizes.set(f.slice(0, -4), size)
          if (st.mtimeMs > tgzLatest) tgzLatest = st.mtimeMs
        }
      }
    } catch {}
    tgzBytes += tgzTotal

    const meta = readJson(pkgFile)
    if (!meta) {
      packages.push({
        name: pkgNameFromDir(pkgDir, storageRoot),
        status: 'broken',
        versions: 0,
        size: fmtMb(tgzTotal),
        lastUpdated: latestDate(null, tgzLatest),
        versionsDetail: [],
      })
    } else {
      const versionsObj = meta.versions || {}
      const timeObj = meta.time || {}
      const distTags = meta['dist-tags'] || {}
      // 包信息（来自本地元数据，离线可用）：介绍/主页/仓库/作者/许可
      const description = typeof meta.description === 'string' ? meta.description : null
      const homepage = typeof meta.homepage === 'string' && meta.homepage ? meta.homepage : null
      const repository = typeof meta.repository === 'string'
        ? meta.repository
        : (meta.repository && typeof meta.repository.url === 'string' ? meta.repository.url : null)
      const author = typeof meta.author === 'string'
        ? meta.author
        : (meta.author && typeof meta.author.name === 'string' ? meta.author.name : null)
      const license = typeof meta.license === 'string' ? meta.license : null
      const versionsDetail = []
      const totalBytes = tgzTotal
      for (const [v, info] of Object.entries(versionsObj)) {
        let size = tgzSizes.get(v)
        if (typeof size !== 'number') {
          const tb = info && info.dist && info.dist.tarball
          const bn = tb ? String(tb).split('/').pop() : ''
          if (bn.endsWith('.tgz')) {
            const hit = tgzSizes.get(bn.slice(0, -4))
            if (typeof hit === 'number') size = hit
          }
        }
        // 无物理 .tgz 文件的版本是元数据残留（实体已被清理），不计数，避免版本数虚高
        if (typeof size !== 'number') continue
        versionsDetail.push({
          version: parseVersion(v),
          size: fmtMb(size),
          date: timeObj[v] ? shDate(timeObj[v]) : null,
        })
      }
      versionsDetail.sort((a, b) => a.version.localeCompare(b.version, undefined, { numeric: true }))
      // 最近更新：取「元数据最新版本发布日」与「最新 .tgz 缓存日」中较新者（今天拉缓存也算今天更新）
      const metaUpdated = Object.keys(timeObj).length
        ? Object.entries(timeObj).map(([, iso]) => shDate(iso)).filter(Boolean).sort().pop()
        : null
      const lastUpdated = latestDate(metaUpdated, tgzLatest)
      packages.push({
        name: pkgNameFromDir(pkgDir, storageRoot),
        versions: versionsDetail.length,
        size: fmtMb(totalBytes),
        lastUpdated,
        visits: 0,
        distTags,
        description,
        homepage,
        repository,
        author,
        license,
        versionsDetail,
      })
    }

    scanned += 1
    if (onProgress) onProgress({ packages: scanned, bytes: scannedBytes, tgzBytes })
  }

  return packages
}

// —— demo 模式数据（无真实目录时的演示） ——
const DEMO_SEED = [
  ['lodash', 25, 4, 12456], ['@antv/g2', 32, 12, 7654], ['axios', 18, 1.2, 9832],
  ['react', 14, 10, 6543], ['vue', 12, 9, 5218], ['express', 16, 0.8, 4376],
  ['tensorflow', 24, 90, 2103], ['typescript', 20, 22, 3421], ['webpack', 19, 18, 2876],
  ['echarts', 15, 16, 1987], ['dayjs', 8, 0.6, 1789], ['uuid', 11, 0.4, 1654],
  ['vite', 9, 14, 1432], ['prisma', 7, 28, 1120], ['pg', 6, 5, 986],
  ['three', 10, 30, 876], ['sharp', 5, 24, 654], ['chalk', 6, 0.3, 743],
  ['semver', 4, 0.3, 532], ['zod', 5, 0.5, 421],
  ['prettier', 3, 1, 388], ['rollup', 8, 6, 356], ['eslint', 13, 8, 342],
  ['moment', 20, 2, 312], ['mocha', 7, 1.4, 301], ['cheerio', 9, 3, 287],
  ['@vue/compiler-sfc', 11, 5, 265], ['ioredis', 10, 4, 254], ['graphql', 12, 7, 243],
  ['socket.io', 6, 2, 231], ['xlsx', 14, 20, 218], ['d3', 9, 12, 205],
  ['puppeteer', 4, 15, 187], ['sharp', 5, 24, 176],
]

function pad(n) { return String(n).padStart(2, '0') }
function dateStr(offset) {
  const d = new Date(Date.now() - offset * 86400000)
  return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())}`
}
function rnd(min, max) { return Math.floor(Math.random() * (max - min + 1)) + min }

function genVersions(count) {
  const list = []
  for (let i = 0; i < count; i++) {
    list.push({
      version: `1.${rnd(0, 9)}.${rnd(0, 20)}`,
      size: Number((rnd(5, 400) / 10).toFixed(1)),
      date: dateStr((count - i) * rnd(3, 9)),
    })
  }
  list.sort((a, b) => a.version.localeCompare(b.version, undefined, { numeric: true }))
  return list
}

// demo 包信息（演示模式展示用，真实环境来自缓存元数据）
const DEMO_META = {
  lodash: { description: 'A modern JavaScript utility library delivering modularity, performance & extras.', homepage: 'https://lodash.com', repository: 'git+https://github.com/lodash/lodash.git', license: 'MIT' },
  react: { description: 'React is a JavaScript library for building user interfaces.', homepage: 'https://react.dev', repository: 'git+https://github.com/facebook/react.git', license: 'MIT' },
  vue: { description: 'The progressive JavaScript framework for building modern web UI.', homepage: 'https://vuejs.org', repository: 'git+https://github.com/vuejs/core.git', license: 'MIT' },
  axios: { description: 'Promise based HTTP client for the browser and node.js.', homepage: 'https://axios-http.com', repository: 'git+https://github.com/axios/axios.git', license: 'MIT' },
  typescript: { description: 'TypeScript is a language for application-scale JavaScript.', homepage: 'https://www.typescriptlang.org', repository: 'git+https://github.com/microsoft/TypeScript.git', license: 'Apache-2.0' },
  express: { description: 'Fast, unopinionated, minimalist web framework for node.', homepage: 'https://expressjs.com', repository: 'git+https://github.com/expressjs/express.git', license: 'MIT' },
  vite: { description: 'Native-ESM powered web dev build tool.', homepage: 'https://vitejs.dev', repository: 'git+https://github.com/vitejs/vite.git', license: 'MIT' },
  webpack: { description: 'Packs CommonJs/AMD modules for the browser.', homepage: 'https://webpack.js.org', repository: 'git+https://github.com/webpack/webpack.git', license: 'MIT' },
  echarts: { description: 'Apache ECharts is a powerful, interactive charting and visualization library for browser.', homepage: 'https://echarts.apache.org', repository: 'git+https://github.com/apache/echarts.git', license: 'Apache-2.0' },
  dayjs: { description: 'Day.js is a minimalist JavaScript library that parses, validates, manipulates, and displays dates and times for modern browsers.', homepage: 'https://day.js.org', repository: 'git+https://github.com/iamkun/dayjs.git', license: 'MIT' },
  prisma: { description: 'Prisma is a next-generation ORM that helps you build and query databases.', homepage: 'https://www.prisma.io', repository: 'git+https://github.com/prisma/prisma.git', license: 'Apache-2.0' },
  three: { description: 'JavaScript 3D library.', homepage: 'https://threejs.org', repository: 'git+https://github.com/mrdoob/three.js.git', license: 'MIT' },
  uuid: { description: 'RFC4122 (v1, v4, and v5) UUIDs.', homepage: 'https://github.com/uuidjs/uuid', repository: 'git+https://github.com/uuidjs/uuid.git', license: 'MIT' },
  zod: { description: 'TypeScript-first schema validation with static type inference.', homepage: 'https://zod.dev', repository: 'git+https://github.com/colinhacks/zod.git', license: 'MIT' },
}

export function demoPackages() {
  const seen = new Set()
  return DEMO_SEED.filter(([n]) => !seen.has(n) && seen.add(n)).map(([name, verCount, sizeBase, visits]) => {
    const detail = genVersions(verCount)
    const meta = DEMO_META[name] || {}
    return {
      name,
      versions: detail.length,
      size: Number((detail.reduce((s, v) => s + v.size, 0)).toFixed(1)),
      lastUpdated: detail.length ? detail[detail.length - 1].date : null,
      visits,
      distTags: { latest: detail.length ? detail[detail.length - 1].version : '1.0.0' },
      description: meta.description || null,
      homepage: meta.homepage || null,
      repository: meta.repository || null,
      author: meta.author || null,
      license: meta.license || null,
      versionsDetail: detail,
    }
  })
}

export function demoSummary() {
  return {
    demo: true,
    todayDownloads: 1234,
    todayTrend: '+18%',
    note: '演示模式：未配置 Verdaccio 缓存目录，展示内置演示数据。',
  }
}