verdaccio-cache-manager/backend/pkglist.js

/**
 * npm 包清单（热门包缓存扩充）—— 数据层 + 配置 + 定时调度
 *
 * 背景：前端 npm 包层出不穷，存量缓存覆盖不全。本模块维护一份「存量之外」的
 * 热门 npm 包清单（vue / react / nodejs 生态为主），定时更新收录最新热门包，
 * 并以私服 VERDACCIO_BASE_URL 为镜像源按清单拉取，尽可能多地缓存 npm 包，
 * 提高依赖下载速度。
 *
 * 清单语义：
 *   - status='pending' 待拉取（含新写入 / 拉取失败未成功）
 *   - status='done'    已成功拉取到私服缓存
 *   - 全部条目 done ⇔ 清单失效（当前无可用 npm 包）：手动拉取提示、定时拉取自动停止，
 *     直到清单写入新的 pending 条目（手工新增 / 定时更新收录新热门包）后恢复。
 *
 * 配置存储（settings KV，JSON）：
 *   pkglist_schedule_update  { enabled, cron }  定时更新清单（默认每天 03:00）
 *   pkglist_schedule_pull    { enabled, cron }  定时按清单拉取（默认每天 06:00）
 *   pkglist_seeded           '1'                内置清单是否已初始化
 */
import fs from 'node:fs'
import path from 'node:path'
import db, { getSetting, setSetting, addLog, nowIso } from './db.js'
import { resolveStorageDir, isValidStorage } from './scanner.js'

// ================= 轻量：读取 storage 中已缓存的包名集合（仅列目录，不读文件） =================
function listCachedPkgNames() {
  const storageDir = resolveStorageDir()
  if (!isValidStorage(storageDir)) return new Set()
  const out = new Set()
  let entries
  try {
    entries = fs.readdirSync(storageDir, { withFileTypes: true })
  } catch { return out }
  for (const e of entries) {
    if (!e.isDirectory() || e.name === 'node_modules' || e.name.startsWith('.')) continue
    if (e.name.startsWith('@')) {
      // scoped 包目录：@scope 下每个子目录是一个包
      try {
        const subs = fs.readdirSync(path.join(storageDir, e.name), { withFileTypes: true })
        for (const s of subs) {
          if (s.isDirectory()) out.add(`${e.name}/${s.name}`)
        }
      } catch { /* 忽略不可读目录 */ }
    } else {
      out.add(e.name)
    }
  }
  return out
}

// ================= 内置热门清单（存量之外，按品类收录） =================
// version 全部留空 = 拉取时解析私服 dist-tags.latest（永远最新版）
const HOT_POOL = [
  // —— 代码编辑器 ——
  { name: 'monaco-editor', category: '代码编辑器' },
  { name: '@monaco-editor/react', category: '代码编辑器' },
  { name: 'codemirror', category: '代码编辑器' },
  { name: '@codemirror/view', category: '代码编辑器' },
  { name: '@codemirror/state', category: '代码编辑器' },
  { name: '@codemirror/lang-javascript', category: '代码编辑器' },
  { name: '@codemirror/lang-html', category: '代码编辑器' },
  { name: '@codemirror/lang-css', category: '代码编辑器' },
  { name: '@codemirror/lang-json', category: '代码编辑器' },
  { name: '@uiw/react-codemirror', category: '代码编辑器' },
  { name: 'ace-builds', category: '代码编辑器' },
  { name: '@toast-ui/editor', category: '代码编辑器' },
  { name: 'highlight.js', category: '代码编辑器' },
  { name: 'prismjs', category: '代码编辑器' },
  { name: 'shiki', category: '代码编辑器' },
  { name: 'marked', category: '代码编辑器' },
  { name: 'markdown-it', category: '代码编辑器' },
  { name: 'dompurify', category: '代码编辑器' },
  // —— JSON 编辑器 ——
  { name: 'vanilla-jsoneditor', category: 'JSON 编辑器' },
  { name: '@json-editor/json-editor', category: 'JSON 编辑器' },
  { name: 'jsoneditor', category: 'JSON 编辑器' },
  { name: 'react-json-view', category: 'JSON 编辑器' },
  { name: 'react-json-view-lite', category: 'JSON 编辑器' },
  { name: '@uiw/react-json-view', category: 'JSON 编辑器' },
  { name: 'jsonc-parser', category: 'JSON 编辑器' },
  // —— 可视化图表 / 数据可视化 ——
  { name: 'echarts', category: '可视化图表' },
  { name: 'echarts-for-react', category: '可视化图表' },
  { name: 'vue-echarts', category: '可视化图表' },
  { name: '@antv/g2', category: '可视化图表' },
  { name: '@antv/g2plot', category: '可视化图表' },
  { name: '@ant-design/plots', category: '可视化图表' },
  { name: '@ant-design/charts', category: '可视化图表' },
  { name: 'bizcharts', category: '可视化图表' },
  { name: 'chart.js', category: '可视化图表' },
  { name: 'react-chartjs-2', category: '可视化图表' },
  { name: 'd3', category: '可视化图表' },
  { name: 'd3-scale', category: '可视化图表' },
  { name: 'd3-shape', category: '可视化图表' },
  { name: 'three', category: '可视化图表' },
  { name: 'plotly.js', category: '可视化图表' },
  { name: 'recharts', category: '可视化图表' },
  { name: 'apexcharts', category: '可视化图表' },
  { name: 'lightweight-charts', category: '可视化图表' },
  { name: 'zrender', category: '可视化图表' },
  { name: 'konva', category: '可视化图表' },
  { name: 'vega', category: '可视化图表' },
  { name: 'vega-lite', category: '可视化图表' },
  { name: 'leaflet', category: '可视化图表' },
  { name: 'maptalks', category: '可视化图表' },
  { name: 'openlayers', category: '可视化图表' },
  // —— 流程图 / 关系图 / 思维导图 ——
  { name: '@antv/x6', category: '流程图' },
  { name: '@antv/g6', category: '流程图' },
  { name: 'dagre', category: '流程图' },
  { name: 'dagre-d3', category: '流程图' },
  { name: '@xyflow/react', category: '流程图' },
  { name: 'reactflow', category: '流程图' },
  { name: '@vue-flow/core', category: '流程图' },
  { name: 'gojs', category: '流程图' },
  { name: 'jointjs', category: '流程图' },
  { name: 'cytoscape', category: '流程图' },
  { name: 'vis-network', category: '流程图' },
  { name: 'elkjs', category: '流程图' },
  { name: '@logicflow/core', category: '流程图' },
  { name: 'mermaid', category: '流程图' },
  { name: 'markmap-lib', category: '思维导图' },
  { name: 'markmap-view', category: '思维导图' },
  { name: 'mind-elixir', category: '思维导图' },
  // —— 文档在线预览 / 文档处理 ——
  { name: 'pdfjs-dist', category: '文档预览' },
  { name: 'pdf-lib', category: '文档预览' },
  { name: 'pdfmake', category: '文档预览' },
  { name: 'jspdf', category: '文档预览' },
  { name: 'docx', category: '文档预览' },
  { name: 'pptxgenjs', category: '文档预览' },
  { name: 'exceljs', category: '文档预览' },
  { name: 'xlsx', category: '文档预览' },
  { name: 'file-saver', category: '文档预览' },
  { name: 'mammoth', category: '文档预览' },
  { name: 'html2canvas', category: '文档预览' },
  { name: 'html2pdf.js', category: '文档预览' },
  { name: 'docx-preview', category: '文档预览' },
  { name: '@vue-office/docx', category: '文档预览' },
  { name: '@vue-office/pdf', category: '文档预览' },
  { name: '@vue-office/excel', category: '文档预览' },
  { name: '@react-pdf/renderer', category: '文档预览' },
  { name: 'viewerjs', category: '文档预览' },
  // —— 工具函数 / 基础库 ——
  { name: 'lodash', category: '工具函数' },
  { name: 'lodash-es', category: '工具函数' },
  { name: 'dayjs', category: '工具函数' },
  { name: 'moment', category: '工具函数' },
  { name: 'date-fns', category: '工具函数' },
  { name: 'rxjs', category: '工具函数' },
  { name: 'ramda', category: '工具函数' },
  { name: 'immer', category: '工具函数' },
  { name: 'nanoid', category: '工具函数' },
  { name: 'uuid', category: '工具函数' },
  { name: 'clsx', category: '工具函数' },
  { name: 'classnames', category: '工具函数' },
  { name: 'fast-deep-equal', category: '工具函数' },
  { name: 'deepmerge', category: '工具函数' },
  { name: 'qs', category: '工具函数' },
  { name: 'semver', category: '工具函数' },
  { name: 'zod', category: '工具函数' },
  { name: 'ajv', category: '工具函数' },
  { name: 'yup', category: '工具函数' },
  { name: 'joi', category: '工具函数' },
  { name: 'mitt', category: '工具函数' },
  { name: 'eventemitter3', category: '工具函数' },
  { name: 'p-limit', category: '工具函数' },
  { name: 'p-map', category: '工具函数' },
  { name: 'p-queue', category: '工具函数' },
  { name: 'fast-glob', category: '工具函数' },
  { name: 'picomatch', category: '工具函数' },
  { name: 'minimatch', category: '工具函数' },
  { name: 'js-yaml', category: '工具函数' },
  { name: 'yaml', category: '工具函数' },
  { name: 'dotenv', category: '工具函数' },
  { name: 'cross-env', category: '工具函数' },
  { name: 'execa', category: '工具函数' },
  { name: 'chalk', category: '工具函数' },
  { name: 'ora', category: '工具函数' },
  { name: 'commander', category: '工具函数' },
  { name: 'minimist', category: '工具函数' },
  { name: 'chokidar', category: '工具函数' },
  { name: 'fs-extra', category: '工具函数' },
  { name: 'rimraf', category: '工具函数' },
  { name: 'glob', category: '工具函数' },
  { name: 'archiver', category: '工具函数' },
  { name: 'adm-zip', category: '工具函数' },
  { name: 'tar', category: '工具函数' },
  { name: 'qrcode', category: '工具函数' },
  { name: 'js-cookie', category: '工具函数' },
  { name: 'js-base64', category: '工具函数' },
  { name: 'crypto-js', category: '工具函数' },
  { name: 'he', category: '工具函数' },
  // —— Vue 生态 ——
  { name: 'vue', category: 'Vue 生态' },
  { name: 'vue-router', category: 'Vue 生态' },
  { name: 'pinia', category: 'Vue 生态' },
  { name: 'vuex', category: 'Vue 生态' },
  { name: '@vueuse/core', category: 'Vue 生态' },
  { name: 'vue-i18n', category: 'Vue 生态' },
  { name: 'element-plus', category: 'Vue 生态' },
  { name: '@element-plus/icons-vue', category: 'Vue 生态' },
  { name: 'ant-design-vue', category: 'Vue 生态' },
  { name: 'naive-ui', category: 'Vue 生态' },
  { name: 'vant', category: 'Vue 生态' },
  { name: 'quasar', category: 'Vue 生态' },
  { name: 'vite', category: 'Vue 生态' },
  { name: '@vitejs/plugin-vue', category: 'Vue 生态' },
  { name: '@vue/compiler-sfc', category: 'Vue 生态' },
  { name: 'vue-loader', category: 'Vue 生态' },
  { name: '@vue/test-utils', category: 'Vue 生态' },
  { name: 'async-validator', category: 'Vue 生态' },
  { name: 'sortablejs', category: 'Vue 生态' },
  { name: 'vuedraggable', category: 'Vue 生态' },
  { name: '@tiptap/core', category: '富文本编辑器' },
  { name: '@tiptap/starter-kit', category: '富文本编辑器' },
  { name: '@tiptap/vue-3', category: '富文本编辑器' },
  { name: '@tiptap/react', category: '富文本编辑器' },
  { name: 'slate', category: '富文本编辑器' },
  { name: 'slate-react', category: '富文本编辑器' },
  { name: 'prosemirror-model', category: '富文本编辑器' },
  { name: 'prosemirror-view', category: '富文本编辑器' },
  { name: 'quill', category: '富文本编辑器' },
  // —— React 生态 ——
  { name: 'react', category: 'React 生态' },
  { name: 'react-dom', category: 'React 生态' },
  { name: 'react-router-dom', category: 'React 生态' },
  { name: '@reduxjs/toolkit', category: 'React 生态' },
  { name: 'react-redux', category: 'React 生态' },
  { name: 'zustand', category: 'React 生态' },
  { name: 'jotai', category: 'React 生态' },
  { name: 'mobx', category: 'React 生态' },
  { name: '@tanstack/react-query', category: 'React 生态' },
  { name: '@tanstack/react-table', category: 'React 生态' },
  { name: 'next', category: 'React 生态' },
  { name: 'antd', category: 'React 生态' },
  { name: '@ant-design/icons', category: 'React 生态' },
  { name: '@mui/material', category: 'React 生态' },
  { name: '@emotion/react', category: 'React 生态' },
  { name: 'styled-components', category: 'React 生态' },
  { name: 'tailwindcss', category: 'React 生态' },
  { name: '@headlessui/react', category: 'React 生态' },
  { name: 'framer-motion', category: 'React 生态' },
  { name: 'react-hook-form', category: 'React 生态' },
  { name: 'formik', category: 'React 生态' },
  { name: 'react-dropzone', category: 'React 生态' },
  { name: 'react-select', category: 'React 生态' },
  { name: 'react-window', category: 'React 生态' },
  { name: 'prop-types', category: 'React 生态' },
  { name: '@vitejs/plugin-react', category: 'React 生态' },
  // —— Node 生态 ——
  { name: 'express', category: 'Node 生态' },
  { name: 'koa', category: 'Node 生态' },
  { name: 'fastify', category: 'Node 生态' },
  { name: '@nestjs/core', category: 'Node 生态' },
  { name: '@nestjs/common', category: 'Node 生态' },
  { name: 'socket.io', category: 'Node 生态' },
  { name: 'ws', category: 'Node 生态' },
  { name: 'node-fetch', category: 'Node 生态' },
  { name: 'undici', category: 'Node 生态' },
  { name: 'axios', category: 'Node 生态' },
  { name: 'got', category: 'Node 生态' },
  { name: 'mongoose', category: 'Node 生态' },
  { name: 'pg', category: 'Node 生态' },
  { name: 'mysql2', category: 'Node 生态' },
  { name: 'better-sqlite3', category: 'Node 生态' },
  { name: 'prisma', category: 'Node 生态' },
  { name: '@prisma/client', category: 'Node 生态' },
  { name: 'sequelize', category: 'Node 生态' },
  { name: 'knex', category: 'Node 生态' },
  { name: 'typeorm', category: 'Node 生态' },
  { name: 'redis', category: 'Node 生态' },
  { name: 'ioredis', category: 'Node 生态' },
  { name: 'bullmq', category: 'Node 生态' },
  { name: 'node-cron', category: 'Node 生态' },
  { name: 'nodemailer', category: 'Node 生态' },
  { name: 'jsonwebtoken', category: 'Node 生态' },
  { name: 'bcryptjs', category: 'Node 生态' },
  { name: 'jose', category: 'Node 生态' },
  { name: 'helmet', category: 'Node 生态' },
  { name: 'cors', category: 'Node 生态' },
  { name: 'morgan', category: 'Node 生态' },
  { name: 'pino', category: 'Node 生态' },
  { name: 'winston', category: 'Node 生态' },
  { name: 'typescript', category: 'Node 生态' },
  { name: 'ts-node', category: 'Node 生态' },
  { name: 'tsx', category: 'Node 生态' },
  { name: 'eslint', category: 'Node 生态' },
  { name: 'prettier', category: 'Node 生态' },
  { name: 'vitest', category: 'Node 生态' },
  { name: 'jest', category: 'Node 生态' },
  { name: 'mocha', category: 'Node 生态' },
  { name: 'supertest', category: 'Node 生态' },
  { name: 'chai', category: 'Node 生态' },
  { name: 'esbuild', category: 'Node 生态' },
  { name: 'rollup', category: 'Node 生态' },
  { name: 'webpack', category: 'Node 生态' },
  { name: 'terser', category: 'Node 生态' },
  { name: 'postcss', category: 'Node 生态' },
  { name: 'autoprefixer', category: 'Node 生态' },
  { name: 'sass', category: 'Node 生态' },
  { name: 'less', category: 'Node 生态' },
  // —— 动画 / 视觉辅助 ——
  { name: 'gsap', category: '动画' },
  { name: 'lottie-web', category: '动画' },
  { name: 'canvas-confetti', category: '动画' },
  { name: 'nprogress', category: '动画' },
  { name: 'animejs', category: '动画' },
  { name: 'intro.js', category: '动画' },
  { name: 'driver.js', category: '动画' },
  { name: 'swiper', category: '动画' },
]

// ================= 数据访问 =================

export const HOT_POOL_NAMES = HOT_POOL.map(p => p.name)

const HOT_BY_NAME = new Map(HOT_POOL.map(p => [p.name, p.category]))

/** 首次初始化：把内置热门清单写入 DB（过滤已缓存包，直接置 done；幂等） */
export function seedIfNeeded() {
  if (getSetting('pkglist_seeded', '') === '1') return 0
  const cached = listCachedPkgNames()
  const ins = db.prepare(
    'INSERT OR IGNORE INTO pkglist (pkg_name, version, category, status, auto, cached, created_at) VALUES (?, NULL, ?, ?, 1, ?, ?)',
  )
  let n = 0, skipped = 0
  db.transaction(() => {
    for (const p of HOT_POOL) {
      if (cached.has(p.name)) {
        // 已缓存：直接插入并置 done（auto=1），不计入待拉取
        ins.run(p.name, p.category, 'done', 1, nowIso())
        skipped++
      } else {
        n += ins.run(p.name, p.category, 'pending', 0, nowIso()).changes
      }
    }
    setSetting('pkglist_seeded', '1')
  })()
  console.log(`[pkglist] 内置热门清单初始化完成：写入 ${n} 条，跳过（已缓存）${skipped} 条`)
  return n
}

/** 全部清单条目（id desc） */
export function getAllRows() {
  return db.prepare('SELECT id, pkg_name, version, category, status, auto, cached, created_at FROM pkglist ORDER BY id DESC').all()
}

/** pending 条目（待拉取） */
export function getPendingRows() {
  return db.prepare("SELECT id, pkg_name, version, category FROM pkglist WHERE status = 'pending' ORDER BY id ASC").all()
}

export function countPending() {
  return db.prepare("SELECT COUNT(*) AS c FROM pkglist WHERE status = 'pending'").get().c
}

/** 标记某包已成功拉取（清单侧置 done；文件落盘由拉取模块完成） */
export function markDone(id) {
  return db.prepare("UPDATE pkglist SET status = 'done' WHERE id = ?").run(id).changes > 0
}

/** 重置为待拉取（清单有新的变更写入时调用，使定时拉取恢复） */
export function resetPending(id) {
  db.prepare("UPDATE pkglist SET status = 'pending' WHERE id = ?").run(id)
}

/** 清单对外视图：items + 是否还有可用包（active = 存在 pending） */
export function getPkgList() {
  const items = getAllRows().map(r => ({
    id: r.id,
    pkgName: r.pkg_name,
    version: r.version || '', // 空 = 最新版
    category: r.category || '',
    status: r.status,
    cached: !!r.cached,
    auto: !!r.auto,
    createdAt: r.created_at,
  }))
  return { items, active: countPending() > 0 }
}

/** 新增条目（包名必填且唯一；version 可空 = 最新版；自动来源 auto=0） */
export function addPkg({ pkgName, version = '', category = '' }) {
  const name = String(pkgName || '').trim()
  if (!name) return { ok: false, message: '包名不能为空' }
  const exists = db.prepare('SELECT 1 FROM pkglist WHERE pkg_name = ?').get(name)
  if (exists) return { ok: false, message: `「${name}」已在清单中` }
  const cat = String(category || '').trim() || HOT_BY_NAME.get(name) || '其他'
  // 插入时自动检测：storage 中已有则直接置 done，否则 pending
  const cached = listCachedPkgNames()
  const status = cached.has(name) ? 'done' : 'pending'
  db.prepare('INSERT INTO pkglist (pkg_name, version, category, status, auto, cached, created_at) VALUES (?, ?, ?, ?, 0, ?, ?)')
    .run(name, String(version || '').trim() || null, cat, status, status === 'done' ? 1 : 0, nowIso())
  addLog({ action: 'list-add', pkgName: name, deletedVersions: JSON.stringify({ version: version || null, category: cat, preCached: status === 'done' }) })
  return { ok: true, status }
}

/** 编辑条目（改包名/版本/品类；内容变更后重置为 pending，相当于清单写入新内容） */
export function editPkg(id, { pkgName, version, category }) {
  const row = db.prepare('SELECT * FROM pkglist WHERE id = ?').get(Number(id))
  if (!row) return { ok: false, message: '清单条目不存在' }
  const name = pkgName != null ? String(pkgName).trim() : row.pkg_name
  if (!name) return { ok: false, message: '包名不能为空' }
  if (name !== row.pkg_name) {
    const dup = db.prepare('SELECT 1 FROM pkglist WHERE pkg_name = ? AND id != ?').get(name, row.id)
    if (dup) return { ok: false, message: `「${name}」已在清单中` }
  }
  const ver = version != null ? String(version).trim() || null : row.version
  const cat = category != null && String(category).trim() ? String(category).trim() : row.category
  // 内容有变化 → 重置 pending（新目标需要重新拉取；否则保持原状态）
  const changed = name !== row.pkg_name || ver !== row.version || cat !== row.category
  const newStatus = changed ? 'pending' : row.status
  const newCached = newStatus === 'done' ? (row.cached || 0) : 0
  db.prepare('UPDATE pkglist SET pkg_name = ?, version = ?, category = ?, status = ?, cached = ? WHERE id = ?')
    .run(name, ver, cat, newStatus, newCached, row.id)
  if (changed) addLog({ action: 'list-edit', pkgName: name, deletedVersions: JSON.stringify({ from: row.pkg_name, version: ver, category: cat }) })
  return { ok: true }
}

/** 删除条目 */
export function removePkg(id) {
  const row = db.prepare('SELECT * FROM pkglist WHERE id = ?').get(Number(id))
  if (!row) return { ok: false, message: '清单条目不存在' }
  db.prepare('DELETE FROM pkglist WHERE id = ?').run(row.id)
  addLog({ action: 'list-del', pkgName: row.pkg_name, deletedVersions: JSON.stringify({ version: row.version }) })
  return { ok: true }
}

/** 定时更新清单：把内置热门池中尚未收录的包批量加入（pending，auto 来源），返回新增条数 */
export function addFromHotPool() {
  const ins = db.prepare(
    'INSERT OR IGNORE INTO pkglist (pkg_name, version, category, status, auto, cached, created_at) VALUES (?, NULL, ?, ?, 1, 0, ?)',
  )
  let n = 0
  db.transaction(() => {
    for (const p of HOT_POOL) {
      n += ins.run(p.name, p.category, 'pending', nowIso()).changes
    }
  })()
  if (n > 0) console.log(`[pkglist] 定时收录 ${n} 个新热门包`)
  return n
}

// ================= npm-search 热门榜单抓取（并入定时更新，动态收录） =================
// 与「最新最热门」内置池合并为同一个「定时更新清单」能力：内置池 ∪ 榜单，按包名去重。
// 榜单源优先级：私服 search 端点 → 官方 registry npm-search；全部不可用时降级为仅内置池。

const VERDACCIO_BASE = (process.env.VERDACCIO_BASE_URL || 'http://127.0.0.1:4873').replace(/\/+$/, '')
const NPMJS_SEARCH_BASE = 'https://registry.npmjs.org'

// 关键词 → 品类（命中同一包时首个关键词的品类生效；未命中的包归为「热门榜单」）
const SEARCH_KEYWORDS = [
  ['vue', 'Vue 生态'],
  ['react', 'React 生态'],
  ['nodejs', 'Node 生态'],
  ['typescript', 'Node 生态'],
  ['chart', '可视化图表'],
  ['code editor', '代码编辑器'],
  ['json editor', 'JSON 编辑器'],
  ['flowchart', '流程图'],
  ['pdf', '文档预览'],
  ['utility', '工具函数'],
  ['form', '表单'],
  ['animation', '动画'],
  ['rich text', '富文本编辑器'],
  ['state management', '状态管理'],
]

const SEARCH_SIZE = 50          // 每关键词取前 50（按搜索综合分）
const SEARCH_TIMEOUT_MS = 6000  // 单次请求超时，避免私服 search 挂起阻塞定时任务
const SEARCH_BATCH = 4          // 并发请求数，避免触发 registry 限流
const RANK_LIMIT = 300          // 榜单合并去重后最多收录条数

const batchOf = (arr, size) => {
  const out = []
  for (let i = 0; i < arr.length; i += size) out.push(arr.slice(i, i + size))
  return out
}

async function fetchSearchJson(base, keyword) {
  const url = `${base}/-/v1/search?text=${encodeURIComponent(keyword)}&size=${SEARCH_SIZE}`
  const ctrl = new AbortController()
  const timer = setTimeout(() => ctrl.abort(), SEARCH_TIMEOUT_MS)
  try {
    const r = await fetch(url, { signal: ctrl.signal, headers: { Accept: 'application/json' } })
    if (!r.ok) throw new Error(`search 响应 ${r.status}`)
    return await r.json()
  } finally {
    clearTimeout(timer)
  }
}

/** 从单个来源分批并发轮询关键词抓取热门包（内部按包名去重，品类随首个命中关键词），按热度降序裁剪到 RANK_LIMIT；源不可用返回 [] */
export async function fetchRankFromSource(base, keywords = SEARCH_KEYWORDS) {
  const byName = new Map()
  const kwCat = new Map(keywords) // kw -> category
  let gotAny = false
  for (const batch of batchOf(keywords, SEARCH_BATCH)) {
    const settled = await Promise.allSettled(
      batch.map(async ([kw]) => ({ kw, data: await fetchSearchJson(base, kw) })),
    )
    for (const s of settled) {
      if (s.status !== 'fulfilled') continue
      gotAny = true
      const category = kwCat.get(s.value.kw) || '热门榜单'
      const objs = s.value.data && Array.isArray(s.value.data.objects) ? s.value.data.objects : []
      for (const o of objs) {
        const name = o && o.package && typeof o.package.name === 'string' ? o.package.name.trim() : ''
        if (!name || name.length > 100 || byName.has(name)) continue
        const popular = (o.score && o.score.detail && o.score.detail.popularity) || 0
        byName.set(name, { name, popular, category })
      }
    }
  }
  // 一个关键词都没有成功 → 该源不可用（如 search 端点未启用 / 挂起），返回空让上层回退
  if (!gotAny) return []
  return [...byName.values()]
    .sort((a, b) => b.popular - a.popular)
    .slice(0, RANK_LIMIT)
    .map(p => ({ name: p.name, category: p.category }))
}

/**
 * 榜单抓取：私服 search 端点优先，不可用时回退官方 registry npm-search。
 * 全部失败返回 null（调用方据此降级为仅内置池）。onSource(base) 用于记录实际来源。
 */
export async function fetchNpmSearchRank({ onSource } = {}) {
  for (const base of [VERDACCIO_BASE, NPMJS_SEARCH_BASE]) {
    try {
      const items = await fetchRankFromSource(base)
      if (items && items.length > 0) {
        if (onSource) onSource(base)
        return { source: base, items }
      }
    } catch (e) { /* 尝试下一来源 */ }
  }
  return null
}

/**
 * 定时更新清单（唯一入口，不新增配置开关）：
 * 内置「最新最热门」热门池 ∪ npm-search 榜单，按包名去重后增量写入（pending，auto 来源）。
 * 榜单抓取失败时降级为仅内置池，不影响收录。
 * @returns {{ added: number, rankSource: string }} added 本次新增条数；rankSource 榜单实际来源（'' = 榜单不可用）
 */
export async function updateFromAllSources() {
  const merged = new Map() // name -> category（内置池优先，保证品类与唯一性）
  for (const p of HOT_POOL) {
    if (!merged.has(p.name)) merged.set(p.name, p.category)
  }
  let rankSource = ''
  try {
    const rank = await fetchNpmSearchRank()
    if (rank) {
      rankSource = rank.source
      for (const p of rank.items) {
        if (!merged.has(p.name)) merged.set(p.name, p.category || '热门榜单')
      }
    }
  } catch (e) {
    // 榜单解析异常 → 仅内置池
  }
  // 定时新增前同样做存量去重：storage 已缓存的包直接置 done（不重复拉取），其余置 pending
  const cached = listCachedPkgNames()
  const ins = db.prepare(
    'INSERT OR IGNORE INTO pkglist (pkg_name, version, category, status, auto, cached, created_at) VALUES (?, NULL, ?, ?, 1, ?, ?)',
  )
  let added = 0, skipped = 0
  try {
    db.transaction(() => {
      for (const [name, category] of merged) {
        const isCached = cached.has(name)
        const status = isCached ? 'done' : 'pending'
        const changes = ins.run(name, category, status, isCached ? 1 : 0, nowIso()).changes
        if (changes) {
          if (status === 'done') skipped++
          else added++
        }
      }
    })()
  } catch (e) {
    console.error('[pkglist] 定时收录写入失败:', e.message)
    return { added: 0, rankSource: rankSource || '' }
  }
  const note = rankSource
    ? `定时收录热门 npm 包：内置池+榜单 新增 ${added} 个、已缓存 ${skipped} 个（榜单源 ${rankSource}）`
    : `定时收录热门 npm 包：内置池 新增 ${added} 个、已缓存 ${skipped} 个（npm-search 榜单不可用，已降级）`
  if (added > 0 || skipped > 0 || rankSource) console.log(`[pkglist] ${note}`)
  return { added, rankSource: rankSource || '' }
}

/** 按包名将清单条目标记为 done（手动去重时用） */
export function markDoneByName(pkgName) {
  return db.prepare("UPDATE pkglist SET status = 'done', cached = 1 WHERE pkg_name = ? AND status != 'done'").run(pkgName).changes
}

/**
 * 手动去重：把清单中「storage 已缓存」的 pending 条目直接置 done，
 * 避免对存量包重复拉取。返回处理条数与剩余 pending 数。
 */
export function dedupPkgList() {
  const cached = listCachedPkgNames()
  const rows = db.prepare("SELECT id, pkg_name FROM pkglist WHERE status = 'pending'").all()
  const upd = db.prepare("UPDATE pkglist SET status = 'done', cached = 1 WHERE id = ?")
  let matched = 0
  db.transaction(() => {
    for (const r of rows) {
      if (cached.has(r.pkg_name)) {
        upd.run(r.id)
        matched++
      }
    }
  })()
  console.log(`[pkglist] 手动去重：${matched} 条已缓存条目置 done，剩余 pending ${rows.length - matched} 条`)
  if (matched > 0) addLog({ action: 'list-dedup', pkgName: null, deletedCount: matched })
  return { matched, pending: rows.length - matched }
}

// ================= 定时配置 =================

const DEFAULT_CONFIG = {
  update: { enabled: false, cron: '0 3 * * *' }, // 每天 03:00 收录最新热门包
  pull: { enabled: false, cron: '0 6 * * *' },   // 每天 06:00 按清单拉取
}

function readJson(key, fallback) {
  try {
    const v = JSON.parse(getSetting(key, ''))
    if (v && typeof v === 'object') return v
  } catch { /* fallthrough */ }
  return fallback
}

function writeCfg(key, cfg) {
  setSetting(key, JSON.stringify(cfg))
}

export function getConfig() {
  const update = { ...DEFAULT_CONFIG.update, ...readJson('pkglist_schedule_update', {}) }
  const pull = { ...DEFAULT_CONFIG.pull, ...readJson('pkglist_schedule_pull', {}) }
  return { update, pull }
}

/** 保存配置：{ update?: {enabled, cron}, pull?: {enabled, cron} }；cron 校验 5 段 */
export function saveConfig(patch = {}) {
  const cur = getConfig()
  for (const key of ['update', 'pull']) {
    const p = patch[key]
    if (!p || typeof p !== 'object') continue
    const suffix = key === 'update' ? 'pkglist_schedule_update' : 'pkglist_schedule_pull'
    const next = { ...cur[key], enabled: !!p.enabled }
    if (p.cron != null) {
      const cron = String(p.cron).trim()
      if (!/^(\S+ ){4}\S+$/.test(cron)) return { ok: false, message: `定时${key === 'update' ? '更新' : '拉取'}的 cron 表达式格式应为 5 段（分 时 日 月 周）` }
      next.cron = cron
    }
    writeCfg(suffix, next)
  }
  return { ok: true, config: getConfig() }
}

// ================= 定时调度（依赖注入，避免与 server 循环引用） =================

const pad2 = n => String(n).padStart(2, '0')
const cronKey = d => `${d.getFullYear()}${pad2(d.getMonth() + 1)}${pad2(d.getDate())}${pad2(d.getHours())}${pad2(d.getMinutes())}`

/** 简化 cron 匹配：分 时 日 月 周（0/7=周日），支持 * / , - */
export function cronMatch(expr, date) {
  const parts = String(expr || '').trim().split(/\s+/)
  if (parts.length !== 5) return false
  const [min, hour, day, mon, dow] = parts
  const d = date || new Date()
  const mins = expandCron(min, 0, 59)
  const hours = expandCron(hour, 0, 23)
  const days = expandCron(day, 1, 31)
  const months = expandCron(mon, 1, 12)
  const dows = expandCron(dow, 0, 7) // 7 → 0（周日）
  const dowsNorm = new Set([...dows].map(x => (x === 7 ? 0 : x)))
  return mins.has(d.getMinutes())
    && hours.has(d.getHours())
    && days.has(d.getDate())
    && months.has(d.getMonth() + 1)
    && dowsNorm.has(d.getDay())
}

function expandCron(field, lo, hi) {
  const out = new Set()
  for (const part of String(field).split(',')) {
    if (part === '*') { for (let v = lo; v <= hi; v++) out.add(v); continue }
    let step = 1
    let base = part
    const slash = part.indexOf('/')
    if (slash !== -1) { base = part.slice(0, slash); step = parseInt(part.slice(slash + 1), 10) || 1 }
    if (base === '*') { for (let v = lo; v <= hi; v += step) out.add(v); continue }
    const dash = base.indexOf('-')
    if (dash !== -1) {
      const a = parseInt(base.slice(0, dash), 10)
      const b = parseInt(base.slice(dash + 1), 10)
      for (let v = a; v <= b; v += step) out.add(v)
      continue
    }
    const n = parseInt(base, 10)
    if (!Number.isNaN(n)) out.add(n)
  }
  return out
}

let lastRunMinute = ''

/**
 * 启动清单守护（每分钟判定一次 cron；同一分钟只执行一次，幂等）
 * @param {{ addLog, invalidateScan, runPull }} services
 *    runPull(source) 由外部注入拉取执行器（pkglist-pull.js）
 */
export function startPkgListDaemon({ runAutoUpdate, runAutoPull }) {
  setInterval(() => {
    try {
      const now = new Date()
      const key = cronKey(now)
      if (key === lastRunMinute) return
      lastRunMinute = key
      const cfg = getConfig()
      if (cfg.update.enabled && cronMatch(cfg.update.cron, now)) runAutoUpdate()
      if (cfg.pull.enabled && cronMatch(cfg.pull.cron, now)) runAutoPull()
    } catch (e) {
      console.error('[pkglist] 定时守护执行异常:', e.message)
    }
  }, 30_000).unref()
  console.log('[pkglist] 定时守护已启动（每分钟检查 update/pull cron）')
}