feat(skills): curated GitHub catalog redesign (#3016)
* feat(skills): remove ClawHub catalog integration Drop the ClawHub registry as a skills catalog source across web server, shared UI, VS Code, docs, and locales. The catalog now serves git-based sources only: the curated Anthropic repo and user-defined repositories. Also removes the now-unused adm-zip dependency. * feat(skills): redesign catalog around curated GitHub repositories Replace the single-source dropdown with a card grid of curated GitHub repositories (Anthropic, OpenAI, Cursor pstack/skills, Matt Pocock) plus user-defined sources. Source cards show skill counts, GitHub stars, and last-updated time; a global search covers all loaded sources. Server: curated sources gain GitHub repo metadata (stars, pushed_at) fetched best-effort with a 3-hour in-memory and on-disk cache; scans run through a concurrency-limited, deduplicated cache with 3-hour TTL persisted across restarts. Refresh still bypasses the cache. Shared UI: source cards, global search with clear button, per-skill GitHub links, install/installed states. VS Code curated list updated to match. All new copy translated across 12 locales. * fix(skills): address catalog review findings - GitHub metadata fetch timeout drops to 1.5s (under the catalog client's 3s deadline) and failed lookups cache briefly (5 min) so repeated catalog loads do not re-hit a failing API. - Disk cache files are written with owner-only permissions (0o600); rename preserves the mode. - loadSource deduplicates concurrent in-flight requests per source and the shared isLoadingSource flag now clears only when the last active source load finishes.
This commit is contained in:
Bohdan Triapitsyn
committed by
GitHub
GitHub
parent
90d8868bfc
commit
1ed3f1f575
@@ -45,12 +45,11 @@ import {
|
||||
import { SKILL_DIR, SKILL_SCOPE, readSkillSupportingFile, writeSkillSupportingFile, deleteSkillSupportingFile } from './shared.js';
|
||||
import { getSkillSources, discoverSkills, mergeDiscoveredSkills, createSkill, updateSkill, deleteSkill, renameSkill, isManagedSkillPath } from './skills.js';
|
||||
import { getCuratedSkillsSources } from '../skills-catalog/curated-sources.js';
|
||||
import { getCacheKey, getCachedScan, setCachedScan } from '../skills-catalog/cache.js';
|
||||
import { isClawdHubSource, parseSkillRepoSource } from '../skills-catalog/source.js';
|
||||
import { getCacheKey, scanWithCache } from '../skills-catalog/cache.js';
|
||||
import { parseSkillRepoSource } from '../skills-catalog/source.js';
|
||||
import { scanSkillsRepository } from '../skills-catalog/scan.js';
|
||||
import { installSkillsFromRepository } from '../skills-catalog/install.js';
|
||||
import { scanClawdHubPage } from '../skills-catalog/clawdhub/scan.js';
|
||||
import { installSkillsFromClawdHub } from '../skills-catalog/clawdhub/install.js';
|
||||
import { fetchGitHubRepoMetas } from '../skills-catalog/github-meta.js';
|
||||
|
||||
export const createFeatureRoutesRuntime = (dependencies) => {
|
||||
const {
|
||||
@@ -287,14 +286,11 @@ export const createFeatureRoutesRuntime = (dependencies) => {
|
||||
SKILL_DIR,
|
||||
getCuratedSkillsSources,
|
||||
getCacheKey,
|
||||
getCachedScan,
|
||||
setCachedScan,
|
||||
scanWithCache,
|
||||
parseSkillRepoSource,
|
||||
scanSkillsRepository,
|
||||
installSkillsFromRepository,
|
||||
scanClawdHubPage,
|
||||
installSkillsFromClawdHub,
|
||||
isClawdHubSource,
|
||||
fetchGitHubRepoMetas,
|
||||
getProfiles,
|
||||
getProfile,
|
||||
});
|
||||
|
||||
@@ -40,14 +40,11 @@ export const registerSkillRoutes = (app, dependencies) => {
|
||||
SKILL_DIR,
|
||||
getCuratedSkillsSources,
|
||||
getCacheKey,
|
||||
getCachedScan,
|
||||
setCachedScan,
|
||||
scanWithCache,
|
||||
parseSkillRepoSource,
|
||||
scanSkillsRepository,
|
||||
installSkillsFromRepository,
|
||||
scanClawdHubPage,
|
||||
installSkillsFromClawdHub,
|
||||
isClawdHubSource,
|
||||
fetchGitHubRepoMetas,
|
||||
getProfiles,
|
||||
getProfile,
|
||||
} = dependencies;
|
||||
@@ -305,9 +302,26 @@ export const registerSkillRoutes = (app, dependencies) => {
|
||||
}));
|
||||
|
||||
const sources = [...curatedSources, ...customSources];
|
||||
const sourcesForUi = sources.map(({ gitIdentityId, ...rest }) => rest);
|
||||
|
||||
res.json({ ok: true, sources: sourcesForUi, itemsBySource: {}, pageInfoBySource: {} });
|
||||
const githubRepos = sources
|
||||
.map((src) => parseSkillRepoSource(src.source))
|
||||
.filter((parsed) => parsed.ok && parsed.host === 'github.com')
|
||||
.map((parsed) => parsed.normalizedRepo);
|
||||
const repoMetas = await fetchGitHubRepoMetas(githubRepos);
|
||||
|
||||
const sourcesForUi = sources.map(({ gitIdentityId, ...rest }) => {
|
||||
const parsed = parseSkillRepoSource(rest.source);
|
||||
const meta = parsed.ok && parsed.host === 'github.com'
|
||||
? repoMetas[parsed.normalizedRepo] || {}
|
||||
: {};
|
||||
return {
|
||||
...rest,
|
||||
stars: typeof meta.stars === 'number' ? meta.stars : null,
|
||||
repoUpdatedAt: typeof meta.repoUpdatedAt === 'string' ? meta.repoUpdatedAt : null,
|
||||
};
|
||||
});
|
||||
|
||||
res.json({ ok: true, sources: sourcesForUi, itemsBySource: {} });
|
||||
} catch (error) {
|
||||
console.error('Failed to load skills catalog:', error);
|
||||
res.status(500).json({ ok: false, error: { kind: 'unknown', message: error.message || 'Failed to load catalog' } });
|
||||
@@ -327,7 +341,6 @@ export const registerSkillRoutes = (app, dependencies) => {
|
||||
}
|
||||
|
||||
const refresh = String(req.query.refresh || '').toLowerCase() === 'true';
|
||||
const cursor = typeof req.query.cursor === 'string' ? req.query.cursor : null;
|
||||
|
||||
const curatedSources = getCuratedSkillsSources();
|
||||
const settings = await readSettingsFromDisk();
|
||||
@@ -355,26 +368,6 @@ export const registerSkillRoutes = (app, dependencies) => {
|
||||
);
|
||||
const installedByName = new Map(resolvedDiscovered.map((s) => [s.name, s]));
|
||||
|
||||
if (src.sourceType === 'clawdhub' || isClawdHubSource(src.source)) {
|
||||
const scanned = await scanClawdHubPage({ cursor: cursor || null });
|
||||
if (!scanned.ok) {
|
||||
return res.status(500).json({ ok: false, error: scanned.error });
|
||||
}
|
||||
|
||||
const items = (scanned.items || []).map((item) => {
|
||||
const installed = installedByName.get(item.skillName);
|
||||
return {
|
||||
...item,
|
||||
sourceId: src.id,
|
||||
installed: installed
|
||||
? { isInstalled: true, scope: installed.scope, source: installed.source }
|
||||
: { isInstalled: false },
|
||||
};
|
||||
});
|
||||
|
||||
return res.json({ ok: true, items, nextCursor: scanned.nextCursor || null });
|
||||
}
|
||||
|
||||
const parsed = parseSkillRepoSource(src.source);
|
||||
if (!parsed.ok) {
|
||||
return res.status(400).json({ ok: false, error: parsed.error });
|
||||
@@ -387,21 +380,19 @@ export const registerSkillRoutes = (app, dependencies) => {
|
||||
identityId: src.gitIdentityId || '',
|
||||
});
|
||||
|
||||
let scanResult = !refresh ? getCachedScan(cacheKey) : null;
|
||||
if (!scanResult) {
|
||||
const scanned = await scanSkillsRepository({
|
||||
const scanResult = await scanWithCache(
|
||||
cacheKey,
|
||||
() => scanSkillsRepository({
|
||||
source: src.source,
|
||||
subpath: src.defaultSubpath,
|
||||
defaultSubpath: src.defaultSubpath,
|
||||
identity: resolveGitIdentity(src.gitIdentityId),
|
||||
});
|
||||
}),
|
||||
{ refresh },
|
||||
);
|
||||
|
||||
if (!scanned.ok) {
|
||||
return res.status(500).json({ ok: false, error: scanned.error });
|
||||
}
|
||||
|
||||
scanResult = scanned;
|
||||
setCachedScan(cacheKey, scanResult);
|
||||
if (!scanResult.ok) {
|
||||
return res.status(500).json({ ok: false, error: scanResult.error });
|
||||
}
|
||||
|
||||
const items = (scanResult.items || []).map((item) => {
|
||||
@@ -483,41 +474,6 @@ export const registerSkillRoutes = (app, dependencies) => {
|
||||
workingDirectory = resolved.directory;
|
||||
}
|
||||
|
||||
if (isClawdHubSource(source)) {
|
||||
const result = await installSkillsFromClawdHub({
|
||||
scope,
|
||||
targetSource,
|
||||
workingDirectory,
|
||||
userSkillDir: SKILL_DIR,
|
||||
selections,
|
||||
conflictPolicy,
|
||||
conflictDecisions,
|
||||
});
|
||||
|
||||
if (!result.ok) {
|
||||
if (result.error?.kind === 'conflicts') {
|
||||
return res.status(409).json({ ok: false, error: result.error });
|
||||
}
|
||||
return res.status(400).json({ ok: false, error: result.error });
|
||||
}
|
||||
|
||||
const installed = result.installed || [];
|
||||
const skipped = result.skipped || [];
|
||||
const requiresRestart = installed.length > 0;
|
||||
|
||||
return res.json({
|
||||
ok: true,
|
||||
installed,
|
||||
skipped,
|
||||
...(requiresRestart
|
||||
? buildDeferredRestartResponse('Skills installed successfully. Restart OpenCode to apply.')
|
||||
: {
|
||||
requiresReload: false,
|
||||
message: 'No skills were installed',
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
const identity = resolveGitIdentity(gitIdentityId);
|
||||
|
||||
const result = await installSkillsFromRepository({
|
||||
|
||||
@@ -69,14 +69,11 @@ const startSkillsApp = ({ projectRoot }) => {
|
||||
SKILL_DIR,
|
||||
getCuratedSkillsSources: () => [],
|
||||
getCacheKey: () => 'k',
|
||||
getCachedScan: () => null,
|
||||
setCachedScan: () => {},
|
||||
scanWithCache: async (_key, loader) => loader(),
|
||||
parseSkillRepoSource: () => ({ ok: false }),
|
||||
scanSkillsRepository: async () => ({ ok: false }),
|
||||
installSkillsFromRepository: async () => ({ ok: false }),
|
||||
scanClawdHubPage: async () => ({ ok: false }),
|
||||
installSkillsFromClawdHub: async () => ({ ok: false }),
|
||||
isClawdHubSource: () => false,
|
||||
fetchGitHubRepoMetas: async () => ({}),
|
||||
getProfiles: () => [],
|
||||
getProfile: () => null,
|
||||
});
|
||||
|
||||
@@ -1,21 +1,17 @@
|
||||
# Skills Catalog Module Documentation
|
||||
|
||||
## Purpose
|
||||
This module provides skill discovery, scanning, and installation capabilities for OpenCode. It supports multiple skill sources including git repositories and the ClawHub registry, with caching and conflict resolution for skill installation.
|
||||
This module provides skill discovery, scanning, and installation capabilities for OpenCode. It supports skill sources backed by git repositories, with caching and conflict resolution for skill installation.
|
||||
|
||||
## Entrypoints and structure
|
||||
- `packages/web/server/lib/skills-catalog/`: Skills catalog module directory containing all skill-related functionality.
|
||||
- `cache.js`: In-memory cache for scan results with TTL support.
|
||||
- `curated-sources.js`: Predefined skill sources (Anthropic, ClawHub).
|
||||
- `curated-sources.js`: Predefined skill sources (Anthropic, OpenAI, Cursor, Matt Pocock).
|
||||
- `github-meta.js`: Best-effort GitHub repository metadata (stars, last push) with in-memory TTL cache.
|
||||
- `git.js`: Git operations helpers for cloning and auth error detection.
|
||||
- `install.js`: Skills installation from git repositories.
|
||||
- `scan.js`: Skills scanning from git repositories.
|
||||
- `source.js`: Source string parsing for git repositories.
|
||||
- `clawdhub/`: ClawHub registry integration.
|
||||
- `index.js`: Public API exports for ClawHub.
|
||||
- `scan.js`: Scanning ClawHub registry with pagination.
|
||||
- `install.js`: Installation from ClawHub (ZIP download).
|
||||
- `api.js`: ClawHub API client with rate limiting.
|
||||
|
||||
## Public API
|
||||
|
||||
@@ -24,13 +20,19 @@ The following functions are exported and used by the web server:
|
||||
### Cache (`cache.js`)
|
||||
- `getCacheKey({ normalizedRepo, subpath, identityId })`: Generate cache key for scan results.
|
||||
- `getCachedScan(key)`: Retrieve cached scan result if not expired.
|
||||
- `setCachedScan(key, value, ttlMs)`: Store scan result with TTL (default 30 minutes).
|
||||
- `setCachedScan(key, value, ttlMs)`: Store scan result with TTL (default 3 hours).
|
||||
- `scanWithCache(key, loader, { refresh })`: Run a scan loader with cache lookup, in-flight deduplication, and a global concurrency limit (2 concurrent scans); only `ok: true` results are cached.
|
||||
- `clearCache()`: Clear all cached scan results.
|
||||
- Scan results persist to `skills-catalog-cache.json` in the OpenChamber data dir (debounced, atomic rename) and survive server restarts within the TTL.
|
||||
|
||||
### Curated Sources (`curated-sources.js`)
|
||||
- `getCuratedSkillsSources()`: Return list of curated skill sources (Anthropic, ClawHub).
|
||||
- `getCuratedSkillsSources()`: Return list of curated skill sources (Anthropic, OpenAI, Cursor, Matt Pocock).
|
||||
- `CURATED_SKILLS_SOURCES`: Constant array of predefined sources.
|
||||
|
||||
### GitHub Repository Metadata (`github-meta.js`)
|
||||
- `fetchGitHubRepoMetas(normalizedRepos)`: Fetch `{ stars, repoUpdatedAt }` for GitHub `owner/repo` strings. Best-effort: failures resolve to `null`; in-flight requests deduplicate; results cached in memory and on disk (`skills-github-meta.json`) for three hours.
|
||||
- `clearGitHubMetaCache()`: Test-only cache reset.
|
||||
|
||||
### Source Parsing (`source.js`)
|
||||
- `parseSkillRepoSource(source, { subpath })`: Parse git repository source string into structured object with SSH/HTTPS clone URLs, normalized repo, and effective subpath. Supports SSH URLs, HTTPS URLs, and shorthand `owner/repo[/subpath]` format.
|
||||
|
||||
@@ -40,20 +42,6 @@ The following functions are exported and used by the web server:
|
||||
### Git Repository Installation (`install.js`)
|
||||
- `installSkillsFromRepository({ source, subpath, defaultSubpath, identity, scope, targetSource, workingDirectory, userSkillDir, selections, conflictPolicy, conflictDecisions })`: Install skills from git repository. Supports user/project scopes, opencode/agents targets, conflict resolution (prompt/skipAll/overwriteAll), and sparse checkout for efficiency.
|
||||
|
||||
### ClawHub Integration (`clawdhub/index.js`)
|
||||
- `isClawdHubSource(source)`: Check if source string refers to ClawHub.
|
||||
- `scanClawdHub()`: Scan entire ClawHub registry for all skills (paginated, max 20 pages).
|
||||
- `scanClawdHubPage({ cursor })`: Scan a single page of ClawHub results with cursor-based pagination.
|
||||
- `installSkillsFromClawdHub({ scope, targetSource, workingDirectory, userSkillDir, selections, conflictPolicy, conflictDecisions })`: Install skills from ClawHub by downloading ZIP files.
|
||||
- `fetchClawdHubSkills({ cursor })`: Fetch paginated skills list from ClawHub API.
|
||||
- `fetchClawdHubSkillVersion(slug, version)`: Fetch specific skill version details.
|
||||
- `fetchClawdHubSkillInfo(slug)`: Fetch skill metadata without version details.
|
||||
- `downloadClawdHubSkill(slug, version)`: Download skill package as ZIP buffer.
|
||||
|
||||
### ClawHub Constants (`clawdhub/index.js`)
|
||||
- `CLAWDHUB_SOURCE_ID`: Source identifier for curated sources.
|
||||
- `CLAWDHUB_SOURCE_STRING`: Source string format.
|
||||
|
||||
## Internal Helpers
|
||||
|
||||
The following functions are internal helpers used by exported functions:
|
||||
@@ -63,10 +51,10 @@ The following functions are internal helpers used by exported functions:
|
||||
- `looksLikeAuthError(message)`: Detect if error message indicates authentication failure (permission denied, publickey, etc.).
|
||||
- `assertGitAvailable()`: Check if git is available in PATH.
|
||||
|
||||
### Skill Name Validation (used in `install.js`, `scan.js`, `clawdhub/install.js`)
|
||||
### Skill Name Validation (used in `install.js`, `scan.js`)
|
||||
- `validateSkillName(skillName)`: Validate skill name against pattern `/^[a-z0-9][a-z0-9-]*[a-z0-9]$|^[a-z0-9]$/` (1-64 chars, lowercase alphanumeric with hyphens).
|
||||
|
||||
### File System Helpers (`install.js`, `scan.js`, `clawdhub/install.js`)
|
||||
### File System Helpers (`install.js`, `scan.js`)
|
||||
- `safeRm(dir)`: Safely remove directory recursively (ignores errors).
|
||||
- `ensureDir(dirPath)`: Ensure directory exists with recursive creation.
|
||||
- `copyDirectoryNoSymlinks(srcDir, dstDir)`: Copy directory contents without symlinks, with path traversal protection.
|
||||
@@ -82,10 +70,6 @@ The following functions are internal helpers used by exported functions:
|
||||
- `toFsPath(repoDir, repoRelPosixPath)`: Convert POSIX path to filesystem path.
|
||||
- `getTargetSkillDir({ scope, targetSource, workingDirectory, userSkillDir, skillName })`: Determine target installation directory based on scope (user/project), targetSource (opencode/agents), and skill name.
|
||||
|
||||
### ClawHub API Helpers (`clawdhub/api.js`)
|
||||
- `rateLimitedFetch(url, options)`: Fetch with rate limiting (120 req/min limit, 100ms delay between requests, exponential backoff on 429/500 errors).
|
||||
- `mapClawdHubItem(item)`: Transform ClawHub API response to SkillsCatalogItem format.
|
||||
|
||||
## Response Contracts
|
||||
|
||||
### Scan Skills Repository Response
|
||||
@@ -101,12 +85,6 @@ The following functions are internal helpers used by exported functions:
|
||||
- `skipped`: Array of skipped skills with `{ skillName, reason }`.
|
||||
- `error`: Error object with `{ kind, message, conflicts? }` on failure. Kinds: `authRequired`, `networkError`, `conflicts`, `invalidSource`, `unknown`.
|
||||
|
||||
### ClawHub Scan Response
|
||||
- `ok`: Boolean indicating success.
|
||||
- `items`: Array of skill items with ClawHub-specific metadata in `clawdhub` property.
|
||||
- `nextCursor`: Pagination cursor for next page (only for `scanClawdHubPage`).
|
||||
- `error`: Error object with `{ kind, message }` on failure.
|
||||
|
||||
### Parse Source Response
|
||||
- `ok`: Boolean indicating success.
|
||||
- `host`: Git host (e.g., `github.com`, `gitlab.com`).
|
||||
@@ -129,7 +107,7 @@ The following functions are internal helpers used by exported functions:
|
||||
|
||||
### Skill Name Validation
|
||||
- All skill names must match `/^[a-z0-9][a-z0-9-]*[a-z0-9]$|^[a-z0-9]$/` (1-64 chars).
|
||||
- Skill names are derived from directory basenames for git repos and slugs for ClawHub.
|
||||
- Skill names are derived from directory basenames for git repos.
|
||||
- Invalid names result in non-installable skills with appropriate warnings.
|
||||
|
||||
### Git Cloning Strategy
|
||||
@@ -144,17 +122,12 @@ The following functions are internal helpers used by exported functions:
|
||||
- Per-skill decisions override global policy via `conflictDecisions` map.
|
||||
- Conflict response includes `{ skillName, scope, source }` for each conflict.
|
||||
|
||||
### ClawHub Integration
|
||||
- ClawHub API base URL: `https://clawdhub.com/api/v1`.
|
||||
- Pagination uses cursor-based approach with `MAX_PAGES=20` safety limit.
|
||||
- Rate limiting: 120 req/min with 100ms delay between requests.
|
||||
- Downloaded skills are extracted from ZIP files using `adm-zip`.
|
||||
- Always validate `SKILL.md` exists before installation.
|
||||
|
||||
### Cache Management
|
||||
- Cache keys include `normalizedRepo`, `subpath`, and `identityId` for isolation.
|
||||
- Default TTL is 30 minutes; can be overridden via `ttlMs` parameter.
|
||||
- Cache is in-memory (not persisted across restarts).
|
||||
- Default TTL is 3 hours for both scan results and GitHub repository metadata.
|
||||
- Scan and GitHub metadata caches persist to JSON files in the OpenChamber data dir, so app restarts and page refreshes reuse previous results instead of re-hitting GitHub.
|
||||
- Scans run through a global concurrency limiter (2 at a time) with per-key in-flight deduplication.
|
||||
- The refresh button passes `refresh: true` and bypasses the cache.
|
||||
|
||||
### Security Considerations
|
||||
- Path traversal protection in `copyDirectoryNoSymlinks`: resolves real paths and checks containment.
|
||||
|
||||
@@ -1,6 +1,58 @@
|
||||
const DEFAULT_TTL_MS = 30 * 60 * 1000;
|
||||
import { readDiskCache, writeDiskCache } from './disk-cache.js';
|
||||
|
||||
const DEFAULT_TTL_MS = 3 * 60 * 60 * 1000;
|
||||
const DISK_CACHE_FILE = 'skills-catalog-cache.json';
|
||||
const MAX_CONCURRENT_SCANS = 2;
|
||||
|
||||
const cache = new Map();
|
||||
const inFlight = new Map();
|
||||
|
||||
let diskLoaded = false;
|
||||
let diskWriteTimer = null;
|
||||
|
||||
const loadDiskEntries = () => {
|
||||
if (diskLoaded) {
|
||||
return;
|
||||
}
|
||||
diskLoaded = true;
|
||||
const persisted = readDiskCache(DISK_CACHE_FILE);
|
||||
if (!persisted) {
|
||||
return;
|
||||
}
|
||||
const now = Date.now();
|
||||
for (const [key, entry] of Object.entries(persisted)) {
|
||||
if (
|
||||
entry
|
||||
&& typeof entry === 'object'
|
||||
&& typeof entry.expiresAt === 'number'
|
||||
&& entry.expiresAt > now
|
||||
&& entry.value
|
||||
&& typeof entry.value === 'object'
|
||||
) {
|
||||
cache.set(key, entry);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const scheduleDiskWrite = () => {
|
||||
if (diskWriteTimer) {
|
||||
return;
|
||||
}
|
||||
diskWriteTimer = setTimeout(() => {
|
||||
diskWriteTimer = null;
|
||||
const now = Date.now();
|
||||
const persisted = {};
|
||||
for (const [key, entry] of cache.entries()) {
|
||||
if (entry.expiresAt > now) {
|
||||
persisted[key] = entry;
|
||||
}
|
||||
}
|
||||
writeDiskCache(DISK_CACHE_FILE, persisted);
|
||||
}, 1000);
|
||||
if (typeof diskWriteTimer.unref === 'function') {
|
||||
diskWriteTimer.unref();
|
||||
}
|
||||
};
|
||||
|
||||
export function getCacheKey({ normalizedRepo, subpath, identityId }) {
|
||||
const safeRepo = String(normalizedRepo || '').trim();
|
||||
@@ -10,6 +62,7 @@ export function getCacheKey({ normalizedRepo, subpath, identityId }) {
|
||||
}
|
||||
|
||||
export function getCachedScan(key) {
|
||||
loadDiskEntries();
|
||||
const entry = cache.get(key);
|
||||
if (!entry) return null;
|
||||
if (Date.now() >= entry.expiresAt) {
|
||||
@@ -22,4 +75,70 @@ export function getCachedScan(key) {
|
||||
export function setCachedScan(key, value, ttlMs = DEFAULT_TTL_MS) {
|
||||
const ttl = Number.isFinite(ttlMs) ? ttlMs : DEFAULT_TTL_MS;
|
||||
cache.set(key, { expiresAt: Date.now() + ttl, value });
|
||||
scheduleDiskWrite();
|
||||
}
|
||||
|
||||
export function clearCache() {
|
||||
cache.clear();
|
||||
inFlight.clear();
|
||||
}
|
||||
|
||||
// ─── Concurrency-limited scan orchestration ───
|
||||
|
||||
let activeScans = 0;
|
||||
const scanQueue = [];
|
||||
|
||||
const acquireScanSlot = () => new Promise((resolve) => {
|
||||
scanQueue.push(resolve);
|
||||
pumpScanQueue();
|
||||
});
|
||||
|
||||
const releaseScanSlot = () => {
|
||||
activeScans -= 1;
|
||||
pumpScanQueue();
|
||||
};
|
||||
|
||||
const pumpScanQueue = () => {
|
||||
while (activeScans < MAX_CONCURRENT_SCANS && scanQueue.length > 0) {
|
||||
const resolve = scanQueue.shift();
|
||||
activeScans += 1;
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Run `loader` for a scan cache key with deduplication and a global
|
||||
* concurrency limit. Concurrent callers for the same key share one loader
|
||||
* run; at most MAX_CONCURRENT_SCANS loaders run at once. Only successful
|
||||
* (`ok: true`) results are cached.
|
||||
*/
|
||||
export async function scanWithCache(key, loader, { refresh = false } = {}) {
|
||||
if (!refresh) {
|
||||
const cached = getCachedScan(key);
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
}
|
||||
|
||||
const existing = inFlight.get(key);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
const run = (async () => {
|
||||
await acquireScanSlot();
|
||||
try {
|
||||
const result = await loader();
|
||||
if (result && result.ok) {
|
||||
setCachedScan(key, result);
|
||||
}
|
||||
return result;
|
||||
} finally {
|
||||
releaseScanSlot();
|
||||
inFlight.delete(key);
|
||||
}
|
||||
})();
|
||||
|
||||
inFlight.set(key, run);
|
||||
return run;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { clearCache, scanWithCache, setCachedScan, getCachedScan } from './cache.js';
|
||||
|
||||
let tempDataDir;
|
||||
|
||||
beforeEach(() => {
|
||||
tempDataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skills-cache-test-'));
|
||||
process.env.OPENCHAMBER_DATA_DIR = tempDataDir;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
delete process.env.OPENCHAMBER_DATA_DIR;
|
||||
clearCache();
|
||||
vi.restoreAllMocks();
|
||||
fs.rmSync(tempDataDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const flushDiskWrites = async () => new Promise((resolve) => setTimeout(resolve, 1200));
|
||||
|
||||
describe('scanWithCache', () => {
|
||||
it('deduplicates concurrent loaders for the same key', async () => {
|
||||
const loader = vi.fn(async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
return { ok: true, items: [] };
|
||||
});
|
||||
|
||||
const [a, b] = await Promise.all([
|
||||
scanWithCache('k', loader),
|
||||
scanWithCache('k', loader),
|
||||
]);
|
||||
|
||||
expect(loader).toHaveBeenCalledTimes(1);
|
||||
expect(a).toEqual(b);
|
||||
});
|
||||
|
||||
it('limits concurrent scans across different keys', async () => {
|
||||
let running = 0;
|
||||
let peak = 0;
|
||||
const loader = async () => {
|
||||
running += 1;
|
||||
peak = Math.max(peak, running);
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
running -= 1;
|
||||
return { ok: true, items: [] };
|
||||
};
|
||||
|
||||
await Promise.all(Array.from({ length: 6 }, (_, i) => scanWithCache(`key-${i}`, loader)));
|
||||
|
||||
expect(peak).toBeLessThanOrEqual(2);
|
||||
});
|
||||
|
||||
it('does not cache failed scans', async () => {
|
||||
await scanWithCache('bad', async () => ({ ok: false, error: { kind: 'networkError', message: 'x' } }));
|
||||
|
||||
expect(getCachedScan('bad')).toBeNull();
|
||||
});
|
||||
|
||||
it('refresh bypasses the cache', async () => {
|
||||
setCachedScan('fresh', { ok: true, items: ['cached'] });
|
||||
|
||||
const result = await scanWithCache('fresh', async () => ({ ok: true, items: ['reloaded'] }), { refresh: true });
|
||||
|
||||
expect(result.items).toEqual(['reloaded']);
|
||||
expect(getCachedScan('fresh').items).toEqual(['reloaded']);
|
||||
});
|
||||
|
||||
it('persists successful scans to disk for later processes', async () => {
|
||||
await scanWithCache('persisted', async () => ({ ok: true, items: [{ skillName: 'x' }] }));
|
||||
await flushDiskWrites();
|
||||
|
||||
const onDisk = JSON.parse(fs.readFileSync(path.join(tempDataDir, 'skills-catalog-cache.json'), 'utf8'));
|
||||
expect(onDisk.persisted.value.items).toEqual([{ skillName: 'x' }]);
|
||||
});
|
||||
});
|
||||
@@ -1,126 +0,0 @@
|
||||
/**
|
||||
* ClawdHub API client
|
||||
*
|
||||
* ClawdHub is a public skill registry at https://clawdhub.com
|
||||
* This client provides methods to fetch skills list and download skill packages.
|
||||
*/
|
||||
|
||||
const CLAWDHUB_API_BASE = 'https://clawdhub.com/api/v1';
|
||||
const CLAWDHUB_PAGE_LIMIT = 25;
|
||||
|
||||
// Rate limiting: ClawdHub allows 120 requests/minute
|
||||
const RATE_LIMIT_DELAY_MS = 100;
|
||||
let lastRequestTime = 0;
|
||||
|
||||
async function rateLimitedFetch(url, options = {}) {
|
||||
const maxAttempts = 10;
|
||||
|
||||
let lastResponse = null;
|
||||
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
||||
const now = Date.now();
|
||||
const elapsed = now - lastRequestTime;
|
||||
if (elapsed < RATE_LIMIT_DELAY_MS) {
|
||||
await new Promise((resolve) => setTimeout(resolve, RATE_LIMIT_DELAY_MS - elapsed));
|
||||
}
|
||||
lastRequestTime = Date.now();
|
||||
|
||||
const response = await fetch(url, {
|
||||
...options,
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
'User-Agent': 'OpenChamber/1.0',
|
||||
...options.headers,
|
||||
},
|
||||
});
|
||||
|
||||
lastResponse = response;
|
||||
|
||||
if (response.status === 429 || response.status >= 500) {
|
||||
if (attempt < maxAttempts - 1) {
|
||||
const waitMs = 50 * (attempt + 1);
|
||||
await new Promise((resolve) => setTimeout(resolve, waitMs));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
return lastResponse;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch paginated list of skills from ClawdHub
|
||||
* @param {Object} options
|
||||
* @param {string} [options.cursor] - Pagination cursor from previous response
|
||||
* @returns {Promise<{ items: Array, nextCursor?: string }>}
|
||||
*/
|
||||
export async function fetchClawdHubSkills({ cursor } = {}) {
|
||||
const url = cursor
|
||||
? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}&limit=${CLAWDHUB_PAGE_LIMIT}`
|
||||
: `${CLAWDHUB_API_BASE}/skills?limit=${CLAWDHUB_PAGE_LIMIT}`;
|
||||
|
||||
const response = await rateLimitedFetch(url);
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text().catch(() => '');
|
||||
throw new Error(`ClawdHub API error (${response.status}): ${text || response.statusText}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const nextCursor =
|
||||
(typeof data.nextCursor === 'string' && data.nextCursor) ||
|
||||
(typeof data.next_cursor === 'string' && data.next_cursor) ||
|
||||
(typeof data.next === 'string' && data.next) ||
|
||||
(typeof data.cursor === 'string' && data.cursor) ||
|
||||
null;
|
||||
|
||||
return {
|
||||
items: data.items || [],
|
||||
nextCursor,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a skill package as a ZIP buffer
|
||||
* @param {string} slug - Skill slug/identifier
|
||||
* @param {string} version - Specific version string
|
||||
* @returns {Promise<ArrayBuffer>} - ZIP file contents
|
||||
*/
|
||||
export async function downloadClawdHubSkill(slug, version) {
|
||||
const versionParam = typeof version === 'string' && version !== 'latest'
|
||||
? `&version=${encodeURIComponent(version)}`
|
||||
: '&tag=latest';
|
||||
const url = `${CLAWDHUB_API_BASE}/download?slug=${encodeURIComponent(slug)}${versionParam}`;
|
||||
|
||||
const response = await rateLimitedFetch(url, {
|
||||
headers: {
|
||||
Accept: 'application/zip',
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text().catch(() => '');
|
||||
throw new Error(`ClawdHub download error (${response.status}): ${text || response.statusText}`);
|
||||
}
|
||||
|
||||
return response.arrayBuffer();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get skill metadata without version details
|
||||
* @param {string} slug - Skill slug/identifier
|
||||
* @returns {Promise<Object>}
|
||||
*/
|
||||
export async function fetchClawdHubSkillInfo(slug) {
|
||||
const url = `${CLAWDHUB_API_BASE}/skills/${encodeURIComponent(slug)}`;
|
||||
const response = await rateLimitedFetch(url);
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text().catch(() => '');
|
||||
throw new Error(`ClawdHub skill error (${response.status}): ${text || response.statusText}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
@@ -1,238 +0,0 @@
|
||||
/**
|
||||
* ClawdHub skill installation
|
||||
*
|
||||
* Downloads skills from ClawdHub as ZIP files and extracts them
|
||||
* to the appropriate skill directory.
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import AdmZip from 'adm-zip';
|
||||
|
||||
import { downloadClawdHubSkill, fetchClawdHubSkillInfo } from './api.js';
|
||||
|
||||
const SKILL_NAME_PATTERN = /^[a-z0-9][a-z0-9-]*[a-z0-9]$|^[a-z0-9]$/;
|
||||
|
||||
function normalizeUserSkillDir(userSkillDir) {
|
||||
if (!userSkillDir) return null;
|
||||
const legacySkillDir = path.join(os.homedir(), '.config', 'opencode', 'skill');
|
||||
const pluralSkillDir = path.join(os.homedir(), '.config', 'opencode', 'skills');
|
||||
if (userSkillDir === legacySkillDir) {
|
||||
if (fs.existsSync(legacySkillDir) && !fs.existsSync(pluralSkillDir)) return legacySkillDir;
|
||||
return pluralSkillDir;
|
||||
}
|
||||
return userSkillDir;
|
||||
}
|
||||
|
||||
function validateSkillName(skillName) {
|
||||
if (typeof skillName !== 'string') return false;
|
||||
if (skillName.length < 1 || skillName.length > 64) return false;
|
||||
return SKILL_NAME_PATTERN.test(skillName);
|
||||
}
|
||||
|
||||
async function safeRm(dir) {
|
||||
try {
|
||||
await fs.promises.rm(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureDir(dirPath) {
|
||||
await fs.promises.mkdir(dirPath, { recursive: true });
|
||||
}
|
||||
|
||||
function getTargetSkillDir({ scope, targetSource, workingDirectory, userSkillDir, skillName }) {
|
||||
const source = targetSource === 'agents' ? 'agents' : 'opencode';
|
||||
|
||||
if (scope === 'user') {
|
||||
if (source === 'agents') {
|
||||
return path.join(os.homedir(), '.agents', 'skills', skillName);
|
||||
}
|
||||
return path.join(userSkillDir, skillName);
|
||||
}
|
||||
|
||||
if (!workingDirectory) {
|
||||
throw new Error('workingDirectory is required for project installs');
|
||||
}
|
||||
|
||||
if (source === 'agents') {
|
||||
return path.join(workingDirectory, '.agents', 'skills', skillName);
|
||||
}
|
||||
|
||||
return path.join(workingDirectory, '.opencode', 'skills', skillName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Install skills from ClawdHub registry
|
||||
* @param {Object} options
|
||||
* @param {string} options.scope - 'user' or 'project'
|
||||
* @param {string} [options.targetSource] - 'opencode' or 'agents'
|
||||
* @param {string} [options.workingDirectory] - Required for project scope
|
||||
* @param {string} options.userSkillDir - User skills directory
|
||||
* @param {Array} options.selections - Array of { skillDir, clawdhub: { slug, version } }
|
||||
* @param {string} [options.conflictPolicy] - 'prompt', 'skipAll', or 'overwriteAll'
|
||||
* @param {Object} [options.conflictDecisions] - Per-skill conflict decisions
|
||||
* @returns {Promise<{ ok: boolean, installed?: Array, skipped?: Array, error?: Object }>}
|
||||
*/
|
||||
export async function installSkillsFromClawdHub({
|
||||
scope,
|
||||
targetSource,
|
||||
workingDirectory,
|
||||
userSkillDir,
|
||||
selections,
|
||||
conflictPolicy,
|
||||
conflictDecisions,
|
||||
} = {}) {
|
||||
if (scope !== 'user' && scope !== 'project') {
|
||||
return { ok: false, error: { kind: 'invalidSource', message: 'Invalid scope' } };
|
||||
}
|
||||
|
||||
if (targetSource !== undefined && targetSource !== 'opencode' && targetSource !== 'agents') {
|
||||
return { ok: false, error: { kind: 'invalidSource', message: 'Invalid target source' } };
|
||||
}
|
||||
|
||||
if (!userSkillDir) {
|
||||
return { ok: false, error: { kind: 'unknown', message: 'userSkillDir is required' } };
|
||||
}
|
||||
|
||||
const normalizedUserSkillDir = normalizeUserSkillDir(userSkillDir);
|
||||
if (normalizedUserSkillDir) {
|
||||
userSkillDir = normalizedUserSkillDir;
|
||||
}
|
||||
|
||||
if (scope === 'project' && !workingDirectory) {
|
||||
return { ok: false, error: { kind: 'invalidSource', message: 'Project installs require a directory parameter' } };
|
||||
}
|
||||
|
||||
const requestedSkills = Array.isArray(selections) ? selections : [];
|
||||
if (requestedSkills.length === 0) {
|
||||
return { ok: false, error: { kind: 'invalidSource', message: 'No skills selected for installation' } };
|
||||
}
|
||||
|
||||
// Build installation plans
|
||||
const skillPlans = requestedSkills.map((sel) => {
|
||||
const slug = sel.clawdhub?.slug || sel.skillDir;
|
||||
const version = sel.clawdhub?.version || 'latest';
|
||||
return {
|
||||
slug,
|
||||
version,
|
||||
installable: validateSkillName(slug),
|
||||
};
|
||||
});
|
||||
|
||||
// Check for conflicts before downloading
|
||||
const conflicts = [];
|
||||
for (const plan of skillPlans) {
|
||||
if (!plan.installable) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const targetDir = getTargetSkillDir({ scope, targetSource, workingDirectory, userSkillDir, skillName: plan.slug });
|
||||
if (fs.existsSync(targetDir)) {
|
||||
const decision = conflictDecisions?.[plan.slug];
|
||||
const hasAutoPolicy = conflictPolicy === 'skipAll' || conflictPolicy === 'overwriteAll';
|
||||
if (!decision && !hasAutoPolicy) {
|
||||
conflicts.push({ skillName: plan.slug, scope, source: targetSource === 'agents' ? 'agents' : 'opencode' });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (conflicts.length > 0) {
|
||||
return {
|
||||
ok: false,
|
||||
error: {
|
||||
kind: 'conflicts',
|
||||
message: 'Some skills already exist in the selected scope',
|
||||
conflicts,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const installed = [];
|
||||
const skipped = [];
|
||||
|
||||
for (const plan of skillPlans) {
|
||||
if (!plan.installable) {
|
||||
skipped.push({ skillName: plan.slug, reason: 'Invalid skill name' });
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
// Resolve 'latest' version if needed
|
||||
let resolvedVersion = plan.version;
|
||||
if (resolvedVersion === 'latest') {
|
||||
try {
|
||||
const info = await fetchClawdHubSkillInfo(plan.slug);
|
||||
const latest = info.skill?.tags?.latest || info.latestVersion?.version || null;
|
||||
if (latest) {
|
||||
resolvedVersion = latest;
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
|
||||
if (resolvedVersion === 'latest') {
|
||||
skipped.push({ skillName: plan.slug, reason: 'Unable to resolve latest version' });
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const targetDir = getTargetSkillDir({ scope, targetSource, workingDirectory, userSkillDir, skillName: plan.slug });
|
||||
const exists = fs.existsSync(targetDir);
|
||||
|
||||
// Determine conflict resolution
|
||||
let decision = conflictDecisions?.[plan.slug] || null;
|
||||
if (!decision) {
|
||||
if (exists && conflictPolicy === 'skipAll') decision = 'skip';
|
||||
if (exists && conflictPolicy === 'overwriteAll') decision = 'overwrite';
|
||||
if (!exists) decision = 'overwrite'; // No conflict, proceed
|
||||
}
|
||||
|
||||
if (exists && decision === 'skip') {
|
||||
skipped.push({ skillName: plan.slug, reason: 'Already installed (skipped)' });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (exists && decision === 'overwrite') {
|
||||
await safeRm(targetDir);
|
||||
}
|
||||
|
||||
// Download the skill ZIP
|
||||
const zipBuffer = await downloadClawdHubSkill(plan.slug, resolvedVersion);
|
||||
|
||||
// Extract to a temp directory first for validation
|
||||
const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), `clawdhub-${plan.slug}-`));
|
||||
|
||||
try {
|
||||
const zip = new AdmZip(Buffer.from(zipBuffer));
|
||||
zip.extractAllTo(tempDir, true);
|
||||
|
||||
// Verify SKILL.md exists
|
||||
const skillMdPath = path.join(tempDir, 'SKILL.md');
|
||||
if (!fs.existsSync(skillMdPath)) {
|
||||
skipped.push({ skillName: plan.slug, reason: 'SKILL.md not found in downloaded package' });
|
||||
continue;
|
||||
}
|
||||
|
||||
// Move to target directory
|
||||
await ensureDir(path.dirname(targetDir));
|
||||
await fs.promises.rename(tempDir, targetDir);
|
||||
|
||||
installed.push({ skillName: plan.slug, scope, source: targetSource === 'agents' ? 'agents' : 'opencode' });
|
||||
} catch (extractError) {
|
||||
await safeRm(tempDir);
|
||||
throw extractError;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to install ClawdHub skill "${plan.slug}":`, error);
|
||||
skipped.push({
|
||||
skillName: plan.slug,
|
||||
reason: error instanceof Error ? error.message : 'Failed to download or extract skill',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { ok: true, installed, skipped };
|
||||
}
|
||||
@@ -1,100 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import AdmZip from 'adm-zip';
|
||||
|
||||
// Mock the ClawdHub network client so no real HTTP happens. The download
|
||||
// function is what feeds the ZIP buffer into adm-zip inside install.js.
|
||||
vi.mock('./api.js', () => ({
|
||||
downloadClawdHubSkill: vi.fn(),
|
||||
fetchClawdHubSkillInfo: vi.fn(),
|
||||
}));
|
||||
|
||||
const { downloadClawdHubSkill } = await import('./api.js');
|
||||
const { installSkillsFromClawdHub } = await import('./install.js');
|
||||
|
||||
/**
|
||||
* Build a real ZIP archive with adm-zip (the dependency under test).
|
||||
* Returns the raw Buffer, mirroring what downloadClawdHubSkill resolves to.
|
||||
*/
|
||||
function buildSkillZip(entries) {
|
||||
const zip = new AdmZip();
|
||||
for (const [entryName, content] of Object.entries(entries)) {
|
||||
zip.addFile(entryName, Buffer.from(content, 'utf8'));
|
||||
}
|
||||
return zip.toBuffer();
|
||||
}
|
||||
|
||||
describe('installSkillsFromClawdHub (adm-zip extraction path)', () => {
|
||||
let userSkillDir;
|
||||
|
||||
beforeEach(async () => {
|
||||
// Keep the target dir under os.tmpdir() so the temp->target rename in
|
||||
// install.js stays on one filesystem (avoids EXDEV cross-device errors).
|
||||
userSkillDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'clawdhub-test-skills-'));
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fs.promises.rm(userSkillDir, { recursive: true, force: true }).catch(() => {});
|
||||
});
|
||||
|
||||
it('extracts a real ZIP (incl. nested subdirectories) into the target skill dir', async () => {
|
||||
const skillMd = 'name: demo-skill\ndescription: adm-zip extraction regression guard\n';
|
||||
const nested = 'nested file content for subdirectory extraction check\n';
|
||||
downloadClawdHubSkill.mockResolvedValue(
|
||||
buildSkillZip({ 'SKILL.md': skillMd, 'nested/data.txt': nested }),
|
||||
);
|
||||
|
||||
const result = await installSkillsFromClawdHub({
|
||||
scope: 'user',
|
||||
targetSource: 'opencode',
|
||||
userSkillDir,
|
||||
// Non-'latest' version avoids the fetchClawdHubSkillInfo resolve branch.
|
||||
selections: [{ clawdhub: { slug: 'demo-skill', version: '1.0.0' } }],
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
expect(result.installed).toEqual([
|
||||
{ skillName: 'demo-skill', scope: 'user', source: 'opencode' },
|
||||
]);
|
||||
expect(result.skipped).toEqual([]);
|
||||
|
||||
// downloadClawdHubSkill received the resolved (non-latest) version.
|
||||
expect(downloadClawdHubSkill).toHaveBeenCalledWith('demo-skill', '1.0.0');
|
||||
|
||||
// adm-zip actually wrote the files, preserving the nested subdirectory.
|
||||
const targetDir = path.join(userSkillDir, 'demo-skill');
|
||||
const skillMdPath = path.join(targetDir, 'SKILL.md');
|
||||
const nestedPath = path.join(targetDir, 'nested', 'data.txt');
|
||||
|
||||
expect(fs.existsSync(skillMdPath)).toBe(true);
|
||||
expect(fs.existsSync(nestedPath)).toBe(true);
|
||||
expect(fs.readFileSync(skillMdPath, 'utf8')).toBe(skillMd);
|
||||
expect(fs.readFileSync(nestedPath, 'utf8')).toBe(nested);
|
||||
});
|
||||
|
||||
it('skips a package whose extracted contents lack SKILL.md', async () => {
|
||||
// Valid ZIP, but no SKILL.md at the root -> install.js must skip it and
|
||||
// must NOT create the target dir. This exercises the extractAllTo path
|
||||
// followed by the post-extraction validation.
|
||||
downloadClawdHubSkill.mockResolvedValue(
|
||||
buildSkillZip({ 'README.md': 'no skill manifest here\n' }),
|
||||
);
|
||||
|
||||
const result = await installSkillsFromClawdHub({
|
||||
scope: 'user',
|
||||
targetSource: 'opencode',
|
||||
userSkillDir,
|
||||
selections: [{ clawdhub: { slug: 'broken-skill', version: '1.0.0' } }],
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
expect(result.installed).toEqual([]);
|
||||
expect(result.skipped).toEqual([
|
||||
{ skillName: 'broken-skill', reason: 'SKILL.md not found in downloaded package' },
|
||||
]);
|
||||
expect(fs.existsSync(path.join(userSkillDir, 'broken-skill'))).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,61 +0,0 @@
|
||||
/**
|
||||
* ClawdHub skill scanning
|
||||
*
|
||||
* Fetches all available skills from the ClawdHub registry
|
||||
* and transforms them into SkillsCatalogItem format.
|
||||
*/
|
||||
|
||||
import { fetchClawdHubSkills } from './api.js';
|
||||
|
||||
const CLAWDHUB_PAGE_LIMIT = 25;
|
||||
|
||||
const mapClawdHubItem = (item) => {
|
||||
const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0';
|
||||
|
||||
return {
|
||||
sourceId: 'clawdhub',
|
||||
repoSource: 'clawdhub:registry',
|
||||
repoSubpath: null,
|
||||
gitIdentityId: null,
|
||||
skillDir: item.slug,
|
||||
skillName: item.slug,
|
||||
frontmatterName: item.displayName || item.slug,
|
||||
description: item.summary || null,
|
||||
installable: true,
|
||||
warnings: [],
|
||||
// ClawdHub-specific metadata
|
||||
clawdhub: {
|
||||
slug: item.slug,
|
||||
version: latestVersion,
|
||||
displayName: item.displayName,
|
||||
owner: item.owner?.handle || null,
|
||||
downloads: item.stats?.downloads || 0,
|
||||
stars: item.stats?.stars || 0,
|
||||
versionsCount: item.stats?.versions || 1,
|
||||
createdAt: item.createdAt,
|
||||
updatedAt: item.updatedAt,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Scan a single ClawdHub page (cursor-based)
|
||||
* @returns {Promise<{ ok: boolean, items?: Array, nextCursor?: string | null, error?: Object }>}
|
||||
*/
|
||||
export async function scanClawdHubPage({ cursor } = {}) {
|
||||
try {
|
||||
const { items, nextCursor } = await fetchClawdHubSkills({ cursor });
|
||||
const mapped = (items || []).map(mapClawdHubItem).slice(0, CLAWDHUB_PAGE_LIMIT);
|
||||
mapped.sort((a, b) => (b.clawdhub?.downloads || 0) - (a.clawdhub?.downloads || 0));
|
||||
return { ok: true, items: mapped, nextCursor: nextCursor || null };
|
||||
} catch (error) {
|
||||
console.error('ClawdHub page scan error:', error);
|
||||
return {
|
||||
ok: false,
|
||||
error: {
|
||||
kind: 'networkError',
|
||||
message: error instanceof Error ? error.message : 'Failed to fetch skills from ClawdHub',
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -8,11 +8,27 @@ const CURATED_SKILLS_SOURCES = [
|
||||
sourceType: 'github',
|
||||
},
|
||||
{
|
||||
id: 'clawdhub',
|
||||
label: 'ClawHub',
|
||||
description: 'Community skill registry with vector search',
|
||||
source: 'clawdhub:registry',
|
||||
sourceType: 'clawdhub',
|
||||
id: 'openai',
|
||||
label: 'OpenAI',
|
||||
description: "OpenAI's curated skills",
|
||||
source: 'openai/skills',
|
||||
defaultSubpath: 'skills/.curated',
|
||||
sourceType: 'github',
|
||||
},
|
||||
{
|
||||
id: 'cursor',
|
||||
label: 'Cursor',
|
||||
description: "Cursor's plugin skills",
|
||||
source: 'cursor/plugins',
|
||||
defaultSubpath: 'pstack/skills',
|
||||
sourceType: 'github',
|
||||
},
|
||||
{
|
||||
id: 'mattpocock',
|
||||
label: 'Matt Pocock',
|
||||
description: 'Matt Pocock skills collection',
|
||||
source: 'mattpocock/skills',
|
||||
sourceType: 'github',
|
||||
},
|
||||
];
|
||||
|
||||
|
||||
@@ -2,9 +2,9 @@ import { describe, expect, it } from 'vitest';
|
||||
import { getCuratedSkillsSources } from './curated-sources.js';
|
||||
|
||||
describe('getCuratedSkillsSources', () => {
|
||||
it('labels the ClawHub curated source as ClawHub', () => {
|
||||
const clawhub = getCuratedSkillsSources().find((source) => source.id === 'clawdhub');
|
||||
expect(clawhub).toBeDefined();
|
||||
expect(clawhub.label).toBe('ClawHub');
|
||||
it('includes the Anthropic curated source', () => {
|
||||
const anthropic = getCuratedSkillsSources().find((source) => source.id === 'anthropic');
|
||||
expect(anthropic).toBeDefined();
|
||||
expect(anthropic.label).toBe('Anthropic');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
|
||||
const resolveDataDir = () => (process.env.OPENCHAMBER_DATA_DIR
|
||||
? path.resolve(process.env.OPENCHAMBER_DATA_DIR)
|
||||
: path.join(os.homedir(), '.config', 'openchamber'));
|
||||
|
||||
const readJsonFile = (filePath) => {
|
||||
try {
|
||||
const raw = fs.readFileSync(filePath, 'utf8');
|
||||
const parsed = JSON.parse(raw);
|
||||
return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Read a persisted cache object from the OpenChamber data directory.
|
||||
* Returns null when the file is missing, unreadable, or malformed.
|
||||
*/
|
||||
export const readDiskCache = (fileName) => {
|
||||
try {
|
||||
return readJsonFile(path.join(resolveDataDir(), fileName));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Persist a cache object to the OpenChamber data directory with an atomic
|
||||
* temp-file rename. Failures are ignored: the in-memory cache stays
|
||||
* authoritative and the next successful write retries persistence.
|
||||
*/
|
||||
export const writeDiskCache = (fileName, data) => {
|
||||
const filePath = path.join(resolveDataDir(), fileName);
|
||||
const tempPath = `${filePath}.${process.pid}.${Date.now()}.tmp`;
|
||||
try {
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||
fs.writeFileSync(tempPath, JSON.stringify(data), { encoding: 'utf8', mode: 0o600 });
|
||||
fs.renameSync(tempPath, filePath);
|
||||
return true;
|
||||
} catch {
|
||||
try {
|
||||
fs.unlinkSync(tempPath);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,139 @@
|
||||
import { readDiskCache, writeDiskCache } from './disk-cache.js';
|
||||
|
||||
const GITHUB_API_BASE = 'https://api.github.com';
|
||||
const CACHE_TTL_MS = 3 * 60 * 60 * 1000;
|
||||
const FAILURE_CACHE_TTL_MS = 5 * 60 * 1000;
|
||||
// Keep well under the catalog route's client request deadline so optional
|
||||
// metadata enrichment can never abort catalog loading.
|
||||
const FETCH_TIMEOUT_MS = 1500;
|
||||
const DISK_CACHE_FILE = 'skills-github-meta.json';
|
||||
|
||||
const metaCache = new Map();
|
||||
const inFlight = new Map();
|
||||
|
||||
let diskLoaded = false;
|
||||
let diskWriteTimer = null;
|
||||
|
||||
const loadDiskEntries = () => {
|
||||
if (diskLoaded) {
|
||||
return;
|
||||
}
|
||||
diskLoaded = true;
|
||||
const persisted = readDiskCache(DISK_CACHE_FILE);
|
||||
if (!persisted) {
|
||||
return;
|
||||
}
|
||||
const now = Date.now();
|
||||
for (const [repo, entry] of Object.entries(persisted)) {
|
||||
if (
|
||||
entry
|
||||
&& typeof entry === 'object'
|
||||
&& typeof entry.expiresAt === 'number'
|
||||
&& entry.expiresAt > now
|
||||
&& entry.value
|
||||
&& typeof entry.value === 'object'
|
||||
) {
|
||||
metaCache.set(repo, entry);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const scheduleDiskWrite = () => {
|
||||
if (diskWriteTimer) {
|
||||
return;
|
||||
}
|
||||
diskWriteTimer = setTimeout(() => {
|
||||
diskWriteTimer = null;
|
||||
const now = Date.now();
|
||||
const persisted = {};
|
||||
for (const [repo, entry] of metaCache.entries()) {
|
||||
if (entry.expiresAt > now) {
|
||||
persisted[repo] = entry;
|
||||
}
|
||||
}
|
||||
writeDiskCache(DISK_CACHE_FILE, persisted);
|
||||
}, 1000);
|
||||
if (typeof diskWriteTimer.unref === 'function') {
|
||||
diskWriteTimer.unref();
|
||||
}
|
||||
};
|
||||
|
||||
const parseMeta = (payload) => {
|
||||
if (!payload || typeof payload !== 'object') {
|
||||
return null;
|
||||
}
|
||||
const pushedAt = payload.pushed_at;
|
||||
return {
|
||||
stars: Number.isFinite(payload.stargazers_count) ? payload.stargazers_count : null,
|
||||
repoUpdatedAt: typeof pushedAt === 'string' && pushedAt ? pushedAt : null,
|
||||
};
|
||||
};
|
||||
|
||||
const fetchRepoMeta = async (normalizedRepo) => {
|
||||
loadDiskEntries();
|
||||
const cached = metaCache.get(normalizedRepo);
|
||||
if (cached && Date.now() < cached.expiresAt) {
|
||||
return cached.value;
|
||||
}
|
||||
|
||||
const existing = inFlight.get(normalizedRepo);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
const run = (async () => {
|
||||
try {
|
||||
const response = await fetch(`${GITHUB_API_BASE}/repos/${normalizedRepo}`, {
|
||||
headers: { Accept: 'application/vnd.github+json' },
|
||||
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
|
||||
});
|
||||
if (!response.ok) {
|
||||
// Cache failures briefly so repeated catalog loads do not re-hit a
|
||||
// rate-limited or failing API for the same repository.
|
||||
metaCache.set(normalizedRepo, {
|
||||
expiresAt: Date.now() + FAILURE_CACHE_TTL_MS,
|
||||
value: { stars: null, repoUpdatedAt: null },
|
||||
});
|
||||
scheduleDiskWrite();
|
||||
return null;
|
||||
}
|
||||
|
||||
const value = parseMeta(await response.json());
|
||||
if (value) {
|
||||
metaCache.set(normalizedRepo, { expiresAt: Date.now() + CACHE_TTL_MS, value });
|
||||
scheduleDiskWrite();
|
||||
}
|
||||
return value;
|
||||
} catch {
|
||||
metaCache.set(normalizedRepo, {
|
||||
expiresAt: Date.now() + FAILURE_CACHE_TTL_MS,
|
||||
value: { stars: null, repoUpdatedAt: null },
|
||||
});
|
||||
scheduleDiskWrite();
|
||||
return null;
|
||||
} finally {
|
||||
inFlight.delete(normalizedRepo);
|
||||
}
|
||||
})();
|
||||
|
||||
inFlight.set(normalizedRepo, run);
|
||||
return run;
|
||||
};
|
||||
|
||||
/**
|
||||
* Fetch GitHub repository metadata (stars, last push) for a list of
|
||||
* `owner/repo` strings. Best-effort: failed lookups resolve to null and
|
||||
* never block the catalog response.
|
||||
*/
|
||||
export async function fetchGitHubRepoMetas(normalizedRepos) {
|
||||
const unique = [...new Set(normalizedRepos.filter(Boolean))];
|
||||
const entries = await Promise.all(unique.map(async (repo) => [repo, await fetchRepoMeta(repo)]));
|
||||
return Object.fromEntries(entries);
|
||||
}
|
||||
|
||||
/** For tests only: clear the in-memory repository metadata cache. */
|
||||
export function clearGitHubMetaCache() {
|
||||
metaCache.clear();
|
||||
inFlight.clear();
|
||||
diskLoaded = true;
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { clearGitHubMetaCache, fetchGitHubRepoMetas } from './github-meta.js';
|
||||
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
let tempDataDir;
|
||||
|
||||
beforeEach(() => {
|
||||
tempDataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'github-meta-test-'));
|
||||
process.env.OPENCHAMBER_DATA_DIR = tempDataDir;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
delete process.env.OPENCHAMBER_DATA_DIR;
|
||||
globalThis.fetch = originalFetch;
|
||||
clearGitHubMetaCache();
|
||||
vi.restoreAllMocks();
|
||||
fs.rmSync(tempDataDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('fetchGitHubRepoMetas', () => {
|
||||
it('returns stars and pushed_at from the GitHub API', async () => {
|
||||
const fetchMock = vi.fn(async () => new Response(
|
||||
JSON.stringify({ stargazers_count: 42, pushed_at: '2026-08-01T00:00:00Z' }),
|
||||
{ status: 200 },
|
||||
));
|
||||
globalThis.fetch = fetchMock;
|
||||
|
||||
const metas = await fetchGitHubRepoMetas(['anthropics/skills']);
|
||||
|
||||
expect(metas).toEqual({
|
||||
'anthropics/skills': { stars: 42, repoUpdatedAt: '2026-08-01T00:00:00Z' },
|
||||
});
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('resolves failed lookups to null without throwing', async () => {
|
||||
globalThis.fetch = vi.fn(async () => new Response('rate limited', { status: 403 }));
|
||||
|
||||
const metas = await fetchGitHubRepoMetas(['anthropics/skills']);
|
||||
|
||||
expect(metas).toEqual({ 'anthropics/skills': null });
|
||||
});
|
||||
|
||||
it('caches failed lookups briefly to avoid repeat hits', async () => {
|
||||
const fetchMock = vi.fn(async () => new Response('rate limited', { status: 403 }));
|
||||
globalThis.fetch = fetchMock;
|
||||
|
||||
await fetchGitHubRepoMetas(['anthropics/skills']);
|
||||
const second = await fetchGitHubRepoMetas(['anthropics/skills']);
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(second).toEqual({ 'anthropics/skills': { stars: null, repoUpdatedAt: null } });
|
||||
});
|
||||
|
||||
it('deduplicates repositories', async () => {
|
||||
const fetchMock = vi.fn(async () => new Response(
|
||||
JSON.stringify({ stargazers_count: 1, pushed_at: null }),
|
||||
{ status: 200 },
|
||||
));
|
||||
globalThis.fetch = fetchMock;
|
||||
|
||||
const metas = await fetchGitHubRepoMetas(['a/b', 'a/b', null]);
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(metas['a/b']).toEqual({ stars: 1, repoUpdatedAt: null });
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,4 @@
|
||||
const GITHUB_HOST = 'github.com';
|
||||
const CLAWDHUB_SOURCE_PREFIX = 'clawdhub:';
|
||||
|
||||
|
||||
function normalizeGitOwnerRepo(owner, repo) {
|
||||
@@ -86,7 +85,3 @@ export function parseSkillRepoSource(input, options = {}) {
|
||||
|
||||
return { ok: false, error: { kind: 'invalidSource', message: 'Unsupported repository source format' } };
|
||||
}
|
||||
|
||||
export function isClawdHubSource(input) {
|
||||
return typeof input === 'string' && input.trim().toLowerCase().startsWith(CLAWDHUB_SOURCE_PREFIX);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user