feat(skills): curated GitHub catalog redesign (#3016)
* feat(skills): remove ClawHub catalog integration Drop the ClawHub registry as a skills catalog source across web server, shared UI, VS Code, docs, and locales. The catalog now serves git-based sources only: the curated Anthropic repo and user-defined repositories. Also removes the now-unused adm-zip dependency. * feat(skills): redesign catalog around curated GitHub repositories Replace the single-source dropdown with a card grid of curated GitHub repositories (Anthropic, OpenAI, Cursor pstack/skills, Matt Pocock) plus user-defined sources. Source cards show skill counts, GitHub stars, and last-updated time; a global search covers all loaded sources. Server: curated sources gain GitHub repo metadata (stars, pushed_at) fetched best-effort with a 3-hour in-memory and on-disk cache; scans run through a concurrency-limited, deduplicated cache with 3-hour TTL persisted across restarts. Refresh still bypasses the cache. Shared UI: source cards, global search with clear button, per-skill GitHub links, install/installed states. VS Code curated list updated to match. All new copy translated across 12 locales. * fix(skills): address catalog review findings - GitHub metadata fetch timeout drops to 1.5s (under the catalog client's 3s deadline) and failed lookups cache briefly (5 min) so repeated catalog loads do not re-hit a failing API. - Disk cache files are written with owner-only permissions (0o600); rename preserves the mode. - loadSource deduplicates concurrent in-flight requests per source and the shared isLoadingSource flag now clears only when the last active source load finishes.
This commit is contained in:
committed by
GitHub
parent
90d8868bfc
commit
1ed3f1f575
@@ -1,6 +1,58 @@
|
||||
const DEFAULT_TTL_MS = 30 * 60 * 1000;
|
||||
import { readDiskCache, writeDiskCache } from './disk-cache.js';
|
||||
|
||||
const DEFAULT_TTL_MS = 3 * 60 * 60 * 1000;
|
||||
const DISK_CACHE_FILE = 'skills-catalog-cache.json';
|
||||
const MAX_CONCURRENT_SCANS = 2;
|
||||
|
||||
const cache = new Map();
|
||||
const inFlight = new Map();
|
||||
|
||||
let diskLoaded = false;
|
||||
let diskWriteTimer = null;
|
||||
|
||||
const loadDiskEntries = () => {
|
||||
if (diskLoaded) {
|
||||
return;
|
||||
}
|
||||
diskLoaded = true;
|
||||
const persisted = readDiskCache(DISK_CACHE_FILE);
|
||||
if (!persisted) {
|
||||
return;
|
||||
}
|
||||
const now = Date.now();
|
||||
for (const [key, entry] of Object.entries(persisted)) {
|
||||
if (
|
||||
entry
|
||||
&& typeof entry === 'object'
|
||||
&& typeof entry.expiresAt === 'number'
|
||||
&& entry.expiresAt > now
|
||||
&& entry.value
|
||||
&& typeof entry.value === 'object'
|
||||
) {
|
||||
cache.set(key, entry);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const scheduleDiskWrite = () => {
|
||||
if (diskWriteTimer) {
|
||||
return;
|
||||
}
|
||||
diskWriteTimer = setTimeout(() => {
|
||||
diskWriteTimer = null;
|
||||
const now = Date.now();
|
||||
const persisted = {};
|
||||
for (const [key, entry] of cache.entries()) {
|
||||
if (entry.expiresAt > now) {
|
||||
persisted[key] = entry;
|
||||
}
|
||||
}
|
||||
writeDiskCache(DISK_CACHE_FILE, persisted);
|
||||
}, 1000);
|
||||
if (typeof diskWriteTimer.unref === 'function') {
|
||||
diskWriteTimer.unref();
|
||||
}
|
||||
};
|
||||
|
||||
export function getCacheKey({ normalizedRepo, subpath, identityId }) {
|
||||
const safeRepo = String(normalizedRepo || '').trim();
|
||||
@@ -10,6 +62,7 @@ export function getCacheKey({ normalizedRepo, subpath, identityId }) {
|
||||
}
|
||||
|
||||
export function getCachedScan(key) {
|
||||
loadDiskEntries();
|
||||
const entry = cache.get(key);
|
||||
if (!entry) return null;
|
||||
if (Date.now() >= entry.expiresAt) {
|
||||
@@ -22,4 +75,70 @@ export function getCachedScan(key) {
|
||||
export function setCachedScan(key, value, ttlMs = DEFAULT_TTL_MS) {
|
||||
const ttl = Number.isFinite(ttlMs) ? ttlMs : DEFAULT_TTL_MS;
|
||||
cache.set(key, { expiresAt: Date.now() + ttl, value });
|
||||
scheduleDiskWrite();
|
||||
}
|
||||
|
||||
export function clearCache() {
|
||||
cache.clear();
|
||||
inFlight.clear();
|
||||
}
|
||||
|
||||
// ─── Concurrency-limited scan orchestration ───
|
||||
|
||||
let activeScans = 0;
|
||||
const scanQueue = [];
|
||||
|
||||
const acquireScanSlot = () => new Promise((resolve) => {
|
||||
scanQueue.push(resolve);
|
||||
pumpScanQueue();
|
||||
});
|
||||
|
||||
const releaseScanSlot = () => {
|
||||
activeScans -= 1;
|
||||
pumpScanQueue();
|
||||
};
|
||||
|
||||
const pumpScanQueue = () => {
|
||||
while (activeScans < MAX_CONCURRENT_SCANS && scanQueue.length > 0) {
|
||||
const resolve = scanQueue.shift();
|
||||
activeScans += 1;
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Run `loader` for a scan cache key with deduplication and a global
|
||||
* concurrency limit. Concurrent callers for the same key share one loader
|
||||
* run; at most MAX_CONCURRENT_SCANS loaders run at once. Only successful
|
||||
* (`ok: true`) results are cached.
|
||||
*/
|
||||
export async function scanWithCache(key, loader, { refresh = false } = {}) {
|
||||
if (!refresh) {
|
||||
const cached = getCachedScan(key);
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
}
|
||||
|
||||
const existing = inFlight.get(key);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
const run = (async () => {
|
||||
await acquireScanSlot();
|
||||
try {
|
||||
const result = await loader();
|
||||
if (result && result.ok) {
|
||||
setCachedScan(key, result);
|
||||
}
|
||||
return result;
|
||||
} finally {
|
||||
releaseScanSlot();
|
||||
inFlight.delete(key);
|
||||
}
|
||||
})();
|
||||
|
||||
inFlight.set(key, run);
|
||||
return run;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user