feat(skills): curated GitHub catalog redesign (#3016)

* feat(skills): remove ClawHub catalog integration Drop the ClawHub registry as a skills catalog source across web server, shared UI, VS Code, docs, and locales. The catalog now serves git-based sources only: the curated Anthropic repo and user-defined repositories. Also removes the now-unused adm-zip dependency. * feat(skills): redesign catalog around curated GitHub repositories Replace the single-source dropdown with a card grid of curated GitHub repositories (Anthropic, OpenAI, Cursor pstack/skills, Matt Pocock) plus user-defined sources. Source cards show skill counts, GitHub stars, and last-updated time; a global search covers all loaded sources. Server: curated sources gain GitHub repo metadata (stars, pushed_at) fetched best-effort with a 3-hour in-memory and on-disk cache; scans run through a concurrency-limited, deduplicated cache with 3-hour TTL persisted across restarts. Refresh still bypasses the cache. Shared UI: source cards, global search with clear button, per-skill GitHub links, install/installed states. VS Code curated list updated to match. All new copy translated across 12 locales. * fix(skills): address catalog review findings - GitHub metadata fetch timeout drops to 1.5s (under the catalog client's 3s deadline) and failed lookups cache briefly (5 min) so repeated catalog loads do not re-hit a failing API. - Disk cache files are written with owner-only permissions (0o600); rename preserves the mode. - loadSource deduplicates concurrent in-flight requests per source and the shared isLoadingSource flag now clears only when the last active source load finishes.
2026-08-20 01:40:10 +03:00
parent 90d8868bfc
commit 1ed3f1f575
45 changed files with 1143 additions and 1286 deletions
@@ -0,0 +1,139 @@
+import { readDiskCache, writeDiskCache } from './disk-cache.js';
+
+const GITHUB_API_BASE = 'https://api.github.com';
+const CACHE_TTL_MS = 3 * 60 * 60 * 1000;
+const FAILURE_CACHE_TTL_MS = 5 * 60 * 1000;
+// Keep well under the catalog route's client request deadline so optional
+// metadata enrichment can never abort catalog loading.
+const FETCH_TIMEOUT_MS = 1500;
+const DISK_CACHE_FILE = 'skills-github-meta.json';
+
+const metaCache = new Map();
+const inFlight = new Map();
+
+let diskLoaded = false;
+let diskWriteTimer = null;
+
+const loadDiskEntries = () => {
+  if (diskLoaded) {
+    return;
+  }
+  diskLoaded = true;
+  const persisted = readDiskCache(DISK_CACHE_FILE);
+  if (!persisted) {
+    return;
+  }
+  const now = Date.now();
+  for (const [repo, entry] of Object.entries(persisted)) {
+    if (
+      entry
+      && typeof entry === 'object'
+      && typeof entry.expiresAt === 'number'
+      && entry.expiresAt > now
+      && entry.value
+      && typeof entry.value === 'object'
+    ) {
+      metaCache.set(repo, entry);
+    }
+  }
+};
+
+const scheduleDiskWrite = () => {
+  if (diskWriteTimer) {
+    return;
+  }
+  diskWriteTimer = setTimeout(() => {
+    diskWriteTimer = null;
+    const now = Date.now();
+    const persisted = {};
+    for (const [repo, entry] of metaCache.entries()) {
+      if (entry.expiresAt > now) {
+        persisted[repo] = entry;
+      }
+    }
+    writeDiskCache(DISK_CACHE_FILE, persisted);
+  }, 1000);
+  if (typeof diskWriteTimer.unref === 'function') {
+    diskWriteTimer.unref();
+  }
+};
+
+const parseMeta = (payload) => {
+  if (!payload || typeof payload !== 'object') {
+    return null;
+  }
+  const pushedAt = payload.pushed_at;
+  return {
+    stars: Number.isFinite(payload.stargazers_count) ? payload.stargazers_count : null,
+    repoUpdatedAt: typeof pushedAt === 'string' && pushedAt ? pushedAt : null,
+  };
+};
+
+const fetchRepoMeta = async (normalizedRepo) => {
+  loadDiskEntries();
+  const cached = metaCache.get(normalizedRepo);
+  if (cached && Date.now() < cached.expiresAt) {
+    return cached.value;
+  }
+
+  const existing = inFlight.get(normalizedRepo);
+  if (existing) {
+    return existing;
+  }
+
+  const run = (async () => {
+    try {
+      const response = await fetch(`${GITHUB_API_BASE}/repos/${normalizedRepo}`, {
+        headers: { Accept: 'application/vnd.github+json' },
+        signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+      });
+      if (!response.ok) {
+        // Cache failures briefly so repeated catalog loads do not re-hit a
+        // rate-limited or failing API for the same repository.
+        metaCache.set(normalizedRepo, {
+          expiresAt: Date.now() + FAILURE_CACHE_TTL_MS,
+          value: { stars: null, repoUpdatedAt: null },
+        });
+        scheduleDiskWrite();
+        return null;
+      }
+
+      const value = parseMeta(await response.json());
+      if (value) {
+        metaCache.set(normalizedRepo, { expiresAt: Date.now() + CACHE_TTL_MS, value });
+        scheduleDiskWrite();
+      }
+      return value;
+    } catch {
+      metaCache.set(normalizedRepo, {
+        expiresAt: Date.now() + FAILURE_CACHE_TTL_MS,
+        value: { stars: null, repoUpdatedAt: null },
+      });
+      scheduleDiskWrite();
+      return null;
+    } finally {
+      inFlight.delete(normalizedRepo);
+    }
+  })();
+
+  inFlight.set(normalizedRepo, run);
+  return run;
+};
+
+/**
+ * Fetch GitHub repository metadata (stars, last push) for a list of
+ * `owner/repo` strings. Best-effort: failed lookups resolve to null and
+ * never block the catalog response.
+ */
+export async function fetchGitHubRepoMetas(normalizedRepos) {
+  const unique = [...new Set(normalizedRepos.filter(Boolean))];
+  const entries = await Promise.all(unique.map(async (repo) => [repo, await fetchRepoMeta(repo)]));
+  return Object.fromEntries(entries);
+}
+
+/** For tests only: clear the in-memory repository metadata cache. */
+export function clearGitHubMetaCache() {
+  metaCache.clear();
+  inFlight.clear();
+  diskLoaded = true;
+}