From 25525d5b1bfea9b310f5852758891a2485bac425 Mon Sep 17 00:00:00 2001 From: Bohdan Triapitsyn Date: Mon, 26 Jan 2026 19:50:11 +0200 Subject: [PATCH] feat: enhance ClawdHub integration with paging and retry Add paging support when loading ClawdHub skills Retry on API errors and throttle for ClawdHub requests Load source content on source change and show loading indicators --- .../desktop/src-tauri/src/skills_catalog.rs | 64 ++++- .../components/sections/agents/AgentsPage.tsx | 52 ---- .../skills/catalog/SkillsCatalogPage.tsx | 173 +++++++----- packages/ui/src/lib/api/types.ts | 8 + .../ui/src/stores/useSkillsCatalogStore.ts | 256 +++++++++++++++--- packages/vscode/src/skillsCatalog.ts | 91 +++++-- packages/web/server/index.js | 198 ++++++++------ .../server/lib/skills-catalog/clawdhub/api.js | 67 +++-- .../lib/skills-catalog/clawdhub/index.js | 2 +- .../lib/skills-catalog/clawdhub/install.js | 13 +- .../lib/skills-catalog/clawdhub/scan.js | 94 +++++-- 11 files changed, 699 insertions(+), 319 deletions(-) diff --git a/packages/desktop/src-tauri/src/skills_catalog.rs b/packages/desktop/src-tauri/src/skills_catalog.rs index eec8b531..54f1ded0 100644 --- a/packages/desktop/src-tauri/src/skills_catalog.rs +++ b/packages/desktop/src-tauri/src/skills_catalog.rs @@ -721,6 +721,7 @@ fn cache_key(normalized_repo: &str, subpath: Option<&str>, identity_id: Option<& // ============== ClawdHub API ============== const CLAWDHUB_API_BASE: &str = "https://clawdhub.com/api/v1"; +const CLAWDHUB_PAGE_LIMIT: usize = 25; fn is_clawdhub_source(source: &str) -> bool { source.starts_with("clawdhub:") @@ -789,23 +790,56 @@ async fn scan_clawdhub() -> Result> { let mut cursor: Option = None; let max_pages = 20; - for _ in 0..max_pages { + for page in 0..max_pages { let url = match &cursor { Some(c) => format!( - "{}{}?cursor={}", + "{}{}?cursor={}&limit={}", CLAWDHUB_API_BASE, "/skills", - urlencoding::encode(c) + urlencoding::encode(c), + CLAWDHUB_PAGE_LIMIT ), - None => format!("{}/skills", CLAWDHUB_API_BASE), + None => format!("{}/skills?limit={}", CLAWDHUB_API_BASE, CLAWDHUB_PAGE_LIMIT), }; - let response = client.get(&url).send().await?; - if !response.status().is_success() { - return Err(anyhow!("ClawdHub API error: {}", response.status())); + let mut response: Option = None; + let max_attempts = 10; + + for attempt in 0..max_attempts { + let resp = client.get(&url).send().await?; + if resp.status().is_success() { + response = Some(resp); + break; + } + + let should_retry = (resp.status() == reqwest::StatusCode::TOO_MANY_REQUESTS + || resp.status().is_server_error()) + && attempt + 1 < max_attempts; + + if should_retry { + tokio::time::sleep(Duration::from_millis(50 * (attempt + 1) as u64)).await; + continue; + } + + if page > 0 && !all_items.is_empty() { + break; + } + return Err(anyhow!("ClawdHub API error: {}", resp.status())); } - let data: ClawdHubSkillsResponse = response.json().await?; + let Some(response) = response else { + break; + }; + + let data: ClawdHubSkillsResponse = match response.json().await { + Ok(parsed) => parsed, + Err(err) => { + if page > 0 && !all_items.is_empty() { + break; + } + return Err(err.into()); + } + }; for item in data.items { let latest_version = item @@ -1510,12 +1544,22 @@ async fn install_skills_from_clawdhub( // Resolve 'latest' version if version == "latest" { if let Ok(info) = fetch_clawdhub_skill_info(slug).await { - version = info + if let Some(latest) = info .skill .and_then(|s| s.tags) .and_then(|t| t.latest) .or_else(|| info.latest_version.and_then(|v| v.version)) - .unwrap_or_else(|| "latest".to_string()); + { + version = latest; + } + } + + if version == "latest" { + skipped.push(SkippedSkill { + skill_name: slug.to_string(), + reason: "Unable to resolve latest version".to_string(), + }); + continue; } } diff --git a/packages/ui/src/components/sections/agents/AgentsPage.tsx b/packages/ui/src/components/sections/agents/AgentsPage.tsx index 9d72b022..5a7fa1f9 100644 --- a/packages/ui/src/components/sections/agents/AgentsPage.tsx +++ b/packages/ui/src/components/sections/agents/AgentsPage.tsx @@ -147,29 +147,6 @@ const permissionConfigToRuleset = (value: unknown): PermissionRule[] => { return rules; }; -const buildPermissionConfigFromRules = (ruleset: PermissionRule[]): AgentConfig['permission'] | undefined => { - const normalized = normalizeRuleset(ruleset); - if (normalized.length === 0) { - return undefined; - } - - const grouped: Record> = {}; - for (const rule of normalized) { - (grouped[rule.permission] ||= {})[rule.pattern] = rule.action; - } - - const result: Record = {}; - for (const [permissionName, patterns] of Object.entries(grouped)) { - if (Object.keys(patterns).length === 1 && patterns['*']) { - result[permissionName] = patterns['*']; - } else { - result[permissionName] = patterns; - } - } - - return result as AgentConfig['permission']; -}; - const buildPermissionConfigWithGlobal = ( globalAction: PermissionAction, ruleset: PermissionRule[], @@ -201,23 +178,6 @@ const buildPermissionConfigWithGlobal = ( return result as AgentConfig['permission']; }; -const buildPermissionDiffConfig = ( - baselineRules: PermissionRule[], - currentRules: PermissionRule[], -): AgentConfig['permission'] | undefined => { - const baselineMap = buildRuleMap(baselineRules); - const currentMap = buildRuleMap(currentRules); - const changedRules: PermissionRule[] = []; - - for (const [key, rule] of currentMap.entries()) { - const baselineRule = baselineMap.get(key); - if (!baselineRule || baselineRule.action !== rule.action) { - changedRules.push(rule); - } - } - - return buildPermissionConfigFromRules(changedRules); -}; export const AgentsPage: React.FC = () => { const { selectedAgentName, getAgentByName, createAgent, updateAgent, agents, agentDraft, setAgentDraft } = useAgentsStore(); @@ -324,12 +284,6 @@ export const AgentsPage: React.FC = () => { currentRuleMap.get(buildRuleKey(permissionName, '*'))?.action ), [currentRuleMap]); - const getEffectiveWildcardAction = React.useCallback((permissionName: string): PermissionAction => { - if (permissionName === '*') { - return globalPermission; - } - return getWildcardOverride(permissionName) ?? globalPermission; - }, [getWildcardOverride, globalPermission]); const getPatternRules = React.useCallback((permissionName: string): PermissionRule[] => ( permissionRules @@ -348,12 +302,6 @@ export const AgentsPage: React.FC = () => { return Array.from(names).sort((a, b) => a.localeCompare(b)); }, [knownPermissionNames]); - const getFallbackDefaultAction = React.useCallback((permissionName: string): PermissionAction => { - if (permissionName === 'doom_loop' || permissionName === 'external_directory') { - return 'ask'; - } - return 'allow'; - }, []); const getPermissionSummary = React.useCallback((permissionName: string) => { const defaultAction = permissionName === '*' diff --git a/packages/ui/src/components/sections/skills/catalog/SkillsCatalogPage.tsx b/packages/ui/src/components/sections/skills/catalog/SkillsCatalogPage.tsx index 09b63e57..a825b56e 100644 --- a/packages/ui/src/components/sections/skills/catalog/SkillsCatalogPage.tsx +++ b/packages/ui/src/components/sections/skills/catalog/SkillsCatalogPage.tsx @@ -65,7 +65,13 @@ export const SkillsCatalogPage: React.FC = ({ mode, onMo selectedSourceId, setSelectedSource, loadCatalog, + loadSource, + loadMoreClawdHub, isLoadingCatalog, + isLoadingSource, + isLoadingMore, + loadedSourceIds, + clawdhubHasMoreBySource, lastCatalogError, } = useSkillsCatalogStore(); @@ -79,6 +85,15 @@ export const SkillsCatalogPage: React.FC = ({ mode, onMo void loadCatalog(); }, [loadCatalog]); + React.useEffect(() => { + if (!selectedSourceId) { + return; + } + if (!loadedSourceIds[selectedSourceId]) { + void loadSource(selectedSourceId); + } + }, [selectedSourceId, loadedSourceIds, loadSource]); + const items = React.useMemo(() => { if (!selectedSourceId) return []; return itemsBySource[selectedSourceId] || []; @@ -98,6 +113,10 @@ export const SkillsCatalogPage: React.FC = ({ mode, onMo const selectedSource = React.useMemo(() => sources.find((s) => s.id === selectedSourceId) || null, [sources, selectedSourceId]); const isCustomSource = Boolean(selectedSourceId && selectedSourceId.startsWith('custom:')); + const isClawdHubSource = selectedSource?.source === 'clawdhub:registry' || selectedSource?.sourceType === 'clawdhub'; + const hasMoreClawdHub = Boolean( + selectedSourceId && (clawdhubHasMoreBySource[selectedSourceId] ?? true) + ); const removeSelectedCatalog = async () => { if (!selectedSourceId || !isCustomSource) { @@ -167,8 +186,14 @@ export const SkillsCatalogPage: React.FC = ({ mode, onMo + + + ); + })} + {isClawdHubSource && hasMoreClawdHub ? ( +
+
- ); - }) + ) : null} + )} diff --git a/packages/ui/src/lib/api/types.ts b/packages/ui/src/lib/api/types.ts index aadc49ff..21057395 100644 --- a/packages/ui/src/lib/api/types.ts +++ b/packages/ui/src/lib/api/types.ts @@ -828,6 +828,14 @@ export interface SkillsCatalogResponse { ok: boolean; sources?: SkillsCatalogSource[]; itemsBySource?: Record; + pageInfoBySource?: Record; + error?: { kind: string; message: string }; +} + +export interface SkillsCatalogSourceResponse { + ok: boolean; + items?: SkillsCatalogItem[]; + nextCursor?: string | null; error?: { kind: string; message: string }; } diff --git a/packages/ui/src/stores/useSkillsCatalogStore.ts b/packages/ui/src/stores/useSkillsCatalogStore.ts index d7ffc5a4..0c64007b 100644 --- a/packages/ui/src/stores/useSkillsCatalogStore.ts +++ b/packages/ui/src/stores/useSkillsCatalogStore.ts @@ -10,11 +10,30 @@ import type { SkillsInstallRequest, SkillsInstallResponse, SkillsInstallError, + SkillsCatalogSourceResponse, } from '@/lib/api/types'; import { useSkillsStore } from '@/stores/useSkillsStore'; import { opencodeClient } from '@/lib/opencode/client'; +const FALLBACK_SOURCES: SkillsCatalogSource[] = [ + { + id: 'anthropic', + label: 'Anthropic', + description: "Anthropic's public skills repository", + source: 'anthropics/skills', + defaultSubpath: 'skills', + sourceType: 'github', + }, + { + id: 'clawdhub', + label: 'ClawdHub', + description: 'Community skill registry with vector search', + source: 'clawdhub:registry', + sourceType: 'clawdhub', + }, +]; + const getCurrentDirectory = (): string | null => { const opencodeDirectory = opencodeClient.getDirectory(); if (typeof opencodeDirectory === 'string' && opencodeDirectory.trim().length > 0) { @@ -38,8 +57,13 @@ export interface SkillsCatalogState { sources: SkillsCatalogSource[]; itemsBySource: Record; selectedSourceId: string | null; + pageInfoBySource: Record; + loadedSourceIds: Record; + clawdhubHasMoreBySource: Record; isLoadingCatalog: boolean; + isLoadingSource: boolean; + isLoadingMore: boolean; isScanning: boolean; isInstalling: boolean; @@ -52,6 +76,8 @@ export interface SkillsCatalogState { setSelectedSource: (id: string | null) => void; loadCatalog: (options?: { refresh?: boolean }) => Promise; + loadSource: (sourceId: string, options?: { refresh?: boolean }) => Promise; + loadMoreClawdHub: () => Promise; scanRepo: (request: SkillsRepoScanRequest) => Promise; installSkills: (request: SkillsInstallRequest) => Promise; } @@ -59,11 +85,16 @@ export interface SkillsCatalogState { export const useSkillsCatalogStore = create()( devtools( (set, get) => ({ - sources: [], + sources: FALLBACK_SOURCES, itemsBySource: {}, - selectedSourceId: null, + selectedSourceId: FALLBACK_SOURCES[0]?.id ?? null, + pageInfoBySource: {}, + loadedSourceIds: {}, + clawdhubHasMoreBySource: {}, isLoadingCatalog: false, + isLoadingSource: false, + isLoadingMore: false, isScanning: false, isInstalling: false, @@ -81,55 +112,64 @@ export const useSkillsCatalogStore = create()( const previous = { sources: get().sources, itemsBySource: get().itemsBySource, + pageInfoBySource: get().pageInfoBySource, + loadedSourceIds: get().loadedSourceIds, + clawdhubHasMoreBySource: get().clawdhubHasMoreBySource, }; let lastError: SkillsCatalogResponse['error'] | null = null; try { - for (let attempt = 0; attempt < 3; attempt++) { - try { - const currentDirectory = getCurrentDirectory(); - const refresh = options?.refresh ? '&refresh=true' : ''; - const queryParams = currentDirectory - ? `?directory=${encodeURIComponent(currentDirectory)}${refresh}` - : refresh - ? `?refresh=true` - : ''; + const refresh = options?.refresh ? '?refresh=true' : ''; + const controller = new AbortController(); + const timeoutId = window.setTimeout(() => controller.abort(), 3000); - const response = await fetch(`/api/config/skills/catalog${queryParams}`, { - method: 'GET', - headers: { Accept: 'application/json' }, - }); + try { + const response = await fetch(`/api/config/skills/catalog${refresh}`, { + method: 'GET', + headers: { Accept: 'application/json' }, + signal: controller.signal, + }); - const payload = (await response.json().catch(() => null)) as SkillsCatalogResponse | null; - if (!response.ok || !payload?.ok) { - lastError = payload?.error || { kind: 'unknown', message: `Failed to load catalog (${response.status})` }; - const waitMs = 200 * (attempt + 1); - await new Promise((resolve) => setTimeout(resolve, waitMs)); - continue; - } - - const sources = payload.sources || []; - const itemsBySource = payload.itemsBySource || {}; - const currentSelected = get().selectedSourceId; - const selectedSourceId = - (currentSelected && sources.some((s) => s.id === currentSelected)) - ? currentSelected - : (sources[0]?.id ?? null); - - set({ sources, itemsBySource, selectedSourceId }); - return true; - - } catch (error) { - lastError = { kind: 'unknown', message: error instanceof Error ? error.message : String(error) }; - const waitMs = 200 * (attempt + 1); - await new Promise((resolve) => setTimeout(resolve, waitMs)); + const payload = (await response.json().catch(() => null)) as SkillsCatalogResponse | null; + if (!response.ok || !payload?.ok) { + lastError = payload?.error || { kind: 'unknown', message: `Failed to load catalog (${response.status})` }; + throw new Error(lastError.message); } - } + const sources = (payload.sources && payload.sources.length > 0) ? payload.sources : previous.sources; + const itemsBySource = options?.refresh ? {} : (get().itemsBySource || {}); + const pageInfoBySource = options?.refresh ? {} : (get().pageInfoBySource || {}); + const loadedSourceIds = options?.refresh ? {} : (get().loadedSourceIds || {}); + const clawdhubHasMoreBySource = options?.refresh ? {} : (get().clawdhubHasMoreBySource || {}); + const currentSelected = get().selectedSourceId; + const selectedSourceId = + (currentSelected && sources.some((s) => s.id === currentSelected)) + ? currentSelected + : (sources[0]?.id ?? null); + + set({ + sources, + itemsBySource, + pageInfoBySource, + loadedSourceIds, + clawdhubHasMoreBySource, + selectedSourceId, + }); + + return true; + } finally { + window.clearTimeout(timeoutId); + } + } catch (error) { + lastError = lastError || { kind: 'unknown', message: error instanceof Error ? error.message : String(error) }; + set({ sources: previous.sources, itemsBySource: previous.itemsBySource, + pageInfoBySource: previous.pageInfoBySource, + loadedSourceIds: previous.loadedSourceIds, + clawdhubHasMoreBySource: previous.clawdhubHasMoreBySource, lastCatalogError: lastError || { kind: 'unknown', message: 'Failed to load catalog' }, }); @@ -139,6 +179,144 @@ export const useSkillsCatalogStore = create()( } }, + loadSource: async (sourceId, options) => { + if (!sourceId) { + return false; + } + + set({ isLoadingSource: true, lastCatalogError: null }); + + try { + const currentDirectory = getCurrentDirectory(); + const refresh = options?.refresh ? '&refresh=true' : ''; + const queryParams = currentDirectory + ? `?directory=${encodeURIComponent(currentDirectory)}&sourceId=${encodeURIComponent(sourceId)}${refresh}` + : `?sourceId=${encodeURIComponent(sourceId)}${refresh}`; + + const response = await fetch(`/api/config/skills/catalog/source${queryParams}`, { + method: 'GET', + headers: { Accept: 'application/json' }, + }); + + const payload = (await response.json().catch(() => null)) as SkillsCatalogSourceResponse | null; + const hasItems = Array.isArray((payload as SkillsCatalogSourceResponse | null)?.items); + if (!response.ok || (!payload?.ok && !hasItems)) { + const fallback = await fetch(`/api/config/skills/catalog${queryParams}`, { + method: 'GET', + headers: { Accept: 'application/json' }, + }); + const fallbackPayload = (await fallback.json().catch(() => null)) as SkillsCatalogResponse | null; + const fallbackItems = fallbackPayload?.itemsBySource?.[sourceId]; + if (fallback.ok && fallbackPayload?.ok && Array.isArray(fallbackItems)) { + set((state) => ({ + itemsBySource: { ...state.itemsBySource, [sourceId]: fallbackItems }, + pageInfoBySource: { ...state.pageInfoBySource, [sourceId]: { nextCursor: null } }, + loadedSourceIds: { ...state.loadedSourceIds, [sourceId]: true }, + clawdhubHasMoreBySource: { ...state.clawdhubHasMoreBySource, [sourceId]: false }, + })); + return true; + } + + set({ + lastCatalogError: payload?.error || { kind: 'unknown', message: `Failed to load source (${response.status})` }, + }); + return false; + } + + const items = payload?.items || []; + const nextCursor = payload?.nextCursor ?? null; + + set((state) => ({ + itemsBySource: { ...state.itemsBySource, [sourceId]: items }, + pageInfoBySource: { ...state.pageInfoBySource, [sourceId]: { nextCursor } }, + loadedSourceIds: { ...state.loadedSourceIds, [sourceId]: true }, + clawdhubHasMoreBySource: { + ...state.clawdhubHasMoreBySource, + [sourceId]: items.length > 0, + }, + })); + + return true; + } catch (error) { + set({ + lastCatalogError: { kind: 'unknown', message: error instanceof Error ? error.message : String(error) }, + }); + return false; + } finally { + set({ isLoadingSource: false }); + } + }, + + loadMoreClawdHub: async () => { + const selectedSourceId = get().selectedSourceId; + if (!selectedSourceId) { + return false; + } + + const pageInfo = get().pageInfoBySource[selectedSourceId]; + const cursor = pageInfo?.nextCursor || null; + + set({ isLoadingMore: true }); + try { + const currentDirectory = getCurrentDirectory(); + const parts = [`sourceId=${encodeURIComponent(selectedSourceId)}`]; + if (currentDirectory) { + parts.push(`directory=${encodeURIComponent(currentDirectory)}`); + } + if (cursor) { + parts.push(`cursor=${encodeURIComponent(cursor)}`); + } + const queryParams = `?${parts.join('&')}`; + + const response = await fetch(`/api/config/skills/catalog/source${queryParams}`, { + method: 'GET', + headers: { Accept: 'application/json' }, + }); + + const payload = (await response.json().catch(() => null)) as SkillsCatalogSourceResponse | null; + if (!response.ok || !payload?.ok) { + return false; + } + + const nextCursor = payload.nextCursor ?? null; + const currentItems = get().itemsBySource[selectedSourceId] || []; + const items = payload.items || []; + const merged = new Map(currentItems.map((item) => [`${item.sourceId}:${item.skillDir}`, item])); + let newCount = 0; + + for (const item of items) { + const key = `${item.sourceId}:${item.skillDir}`; + if (!merged.has(key)) { + newCount += 1; + } + merged.set(key, item); + } + + const noMore = items.length === 0 || newCount === 0; + + set((state) => ({ + itemsBySource: { + ...state.itemsBySource, + [selectedSourceId]: Array.from(merged.values()), + }, + pageInfoBySource: { + ...state.pageInfoBySource, + [selectedSourceId]: { nextCursor }, + }, + clawdhubHasMoreBySource: { + ...state.clawdhubHasMoreBySource, + [selectedSourceId]: !noMore, + }, + })); + + return true; + } catch { + return false; + } finally { + set({ isLoadingMore: false }); + } + }, + scanRepo: async (request) => { set({ isScanning: true, lastScanError: null, scanResults: null }); try { diff --git a/packages/vscode/src/skillsCatalog.ts b/packages/vscode/src/skillsCatalog.ts index 71c7a9ce..712572d8 100644 --- a/packages/vscode/src/skillsCatalog.ts +++ b/packages/vscode/src/skillsCatalog.ts @@ -94,6 +94,7 @@ export const CURATED_SOURCES: CuratedSource[] = [ // ============== ClawdHub API ============== const CLAWDHUB_API_BASE = 'https://clawdhub.com/api/v1'; +const CLAWDHUB_PAGE_LIMIT = 25; const CLAWDHUB_RATE_LIMIT_MS = 100; let clawdhubLastRequest = 0; @@ -102,21 +103,40 @@ function isClawdHubSource(source: string): boolean { } async function clawdhubFetch(url: string, options?: RequestInit): Promise { - const now = Date.now(); - const elapsed = now - clawdhubLastRequest; - if (elapsed < CLAWDHUB_RATE_LIMIT_MS) { - await new Promise((resolve) => setTimeout(resolve, CLAWDHUB_RATE_LIMIT_MS - elapsed)); - } - clawdhubLastRequest = Date.now(); + const maxAttempts = 10; + let lastResponse: Response | null = null; - return fetch(url, { - ...options, - headers: { - Accept: 'application/json', - 'User-Agent': 'OpenChamber-VSCode/1.0', - ...options?.headers, - }, - }); + for (let attempt = 0; attempt < maxAttempts; attempt++) { + const now = Date.now(); + const elapsed = now - clawdhubLastRequest; + if (elapsed < CLAWDHUB_RATE_LIMIT_MS) { + await new Promise((resolve) => setTimeout(resolve, CLAWDHUB_RATE_LIMIT_MS - elapsed)); + } + clawdhubLastRequest = Date.now(); + + const response = await fetch(url, { + ...options, + headers: { + Accept: 'application/json', + 'User-Agent': 'OpenChamber-VSCode/1.0', + ...options?.headers, + }, + }); + + lastResponse = response; + + if (response.status === 429 || response.status >= 500) { + if (attempt < maxAttempts - 1) { + const waitMs = 50 * (attempt + 1); + await new Promise((resolve) => setTimeout(resolve, waitMs)); + continue; + } + } + + return response; + } + + return lastResponse as Response; } type ClawdHubSkillListItem = { @@ -142,16 +162,25 @@ async function scanClawdHub(): Promise { for (let page = 0; page < maxPages; page++) { const url = cursor - ? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}` - : `${CLAWDHUB_API_BASE}/skills`; + ? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}&limit=${CLAWDHUB_PAGE_LIMIT}` + : `${CLAWDHUB_API_BASE}/skills?limit=${CLAWDHUB_PAGE_LIMIT}`; - const response = await clawdhubFetch(url); - if (!response.ok) { - throw new Error(`ClawdHub API error: ${response.status}`); + let data: ClawdHubSkillsResponse; + + try { + const response = await clawdhubFetch(url); + if (!response.ok) { + throw new Error(`ClawdHub API error: ${response.status}`); + } + + data = (await response.json()) as ClawdHubSkillsResponse; + } catch (error) { + if (page > 0 && allItems.length > 0) { + break; + } + throw error; } - const data = (await response.json()) as ClawdHubSkillsResponse; - for (const item of data.items || []) { const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0'; @@ -275,15 +304,23 @@ export async function installSkillsFromClawdHub(options: { try { // Resolve 'latest' version - if (version === 'latest') { - try { - const info = await fetchClawdHubSkillInfo(slug); - version = info.skill?.tags?.latest || info.latestVersion?.version || version; - } catch { - // Fall back to 'latest' + if (version === 'latest') { + try { + const info = await fetchClawdHubSkillInfo(slug); + const latest = info.skill?.tags?.latest || info.latestVersion?.version || null; + if (latest) { + version = latest; } + } catch { + // ignore } + if (version === 'latest') { + skipped.push({ skillName: slug, reason: 'Unable to resolve latest version' }); + continue; + } + } + const targetDir = options.scope === 'user' ? path.join(userSkillDir, slug) : path.join(options.workingDirectory as string, '.opencode', 'skills', slug); diff --git a/packages/web/server/index.js b/packages/web/server/index.js index 17bd98c5..80456445 100644 --- a/packages/web/server/index.js +++ b/packages/web/server/index.js @@ -533,6 +533,25 @@ const resolveProjectDirectory = async (req) => { return { directory: validated.directory, error: null }; }; +const resolveOptionalProjectDirectory = async (req) => { + const headerDirectory = typeof req.get === 'function' ? req.get('x-opencode-directory') : null; + const queryDirectory = Array.isArray(req.query?.directory) + ? req.query.directory[0] + : req.query?.directory; + const requested = headerDirectory || queryDirectory || null; + + if (!requested) { + return { directory: null, error: null }; + } + + const validated = await validateDirectoryPath(requested); + if (!validated.ok) { + return { directory: null, error: validated.error }; + } + + return { directory: validated.directory, error: null }; +}; + const sanitizeTypographySizesPartial = (input) => { if (!input || typeof input !== 'object') { return undefined; @@ -2416,6 +2435,7 @@ function setupProxy(app) { req.path.startsWith('/push') || req.path.startsWith('/config/agents') || req.path.startsWith('/config/settings') || + req.path.startsWith('/config/skills') || req.path === '/config/reload' || req.path === '/health' ) { @@ -2448,6 +2468,7 @@ function setupProxy(app) { req.path.startsWith('/themes/custom') || req.path.startsWith('/config/agents') || req.path.startsWith('/config/settings') || + req.path.startsWith('/config/skills') || req.path === '/health' ) { return next(); @@ -3508,7 +3529,7 @@ async function main(options = {}) { const { parseSkillRepoSource } = await import('./lib/skills-catalog/source.js'); const { scanSkillsRepository } = await import('./lib/skills-catalog/scan.js'); const { installSkillsFromRepository } = await import('./lib/skills-catalog/install.js'); - const { scanClawdHub, installSkillsFromClawdHub, isClawdHubSource } = await import('./lib/skills-catalog/clawdhub/index.js'); + const { scanClawdHubPage, installSkillsFromClawdHub, isClawdHubSource } = await import('./lib/skills-catalog/clawdhub/index.js'); const { getProfiles, getProfile } = await import('./lib/git-identity-storage.js'); const listGitIdentitiesForResponse = () => { @@ -3538,11 +3559,10 @@ async function main(options = {}) { app.get('/api/config/skills/catalog', async (req, res) => { try { - const { directory, error } = await resolveProjectDirectory(req); - if (!directory) { + const { error } = await resolveOptionalProjectDirectory(req); + if (error) { return res.status(400).json({ error }); } - const refresh = String(req.query.refresh || '').toLowerCase() === 'true'; const curatedSources = getCuratedSkillsSources(); const settings = await readSettingsFromDisk(); @@ -3558,95 +3578,121 @@ async function main(options = {}) { })); const sources = [...curatedSources, ...customSources]; + const sourcesForUi = sources.map(({ gitIdentityId, ...rest }) => rest); - const discovered = discoverSkills(directory); + res.json({ ok: true, sources: sourcesForUi, itemsBySource: {}, pageInfoBySource: {} }); + } catch (error) { + console.error('Failed to load skills catalog:', error); + res.status(500).json({ ok: false, error: { kind: 'unknown', message: error.message || 'Failed to load catalog' } }); + } + }); + + app.get('/api/config/skills/catalog/source', async (req, res) => { + try { + const { directory, error } = await resolveOptionalProjectDirectory(req); + if (error) { + return res.status(400).json({ ok: false, error: { kind: 'invalidSource', message: error } }); + } + + const sourceId = typeof req.query.sourceId === 'string' ? req.query.sourceId : null; + if (!sourceId) { + return res.status(400).json({ ok: false, error: { kind: 'invalidSource', message: 'Missing sourceId' } }); + } + + const refresh = String(req.query.refresh || '').toLowerCase() === 'true'; + const cursor = typeof req.query.cursor === 'string' ? req.query.cursor : null; + + const curatedSources = getCuratedSkillsSources(); + const settings = await readSettingsFromDisk(); + const customSourcesRaw = sanitizeSkillCatalogs(settings.skillCatalogs) || []; + + const customSources = customSourcesRaw.map((entry) => ({ + id: entry.id, + label: entry.label, + description: entry.source, + source: entry.source, + defaultSubpath: entry.subpath, + gitIdentityId: entry.gitIdentityId, + })); + + const sources = [...curatedSources, ...customSources]; + const src = sources.find((entry) => entry.id === sourceId); + + if (!src) { + return res.status(404).json({ ok: false, error: { kind: 'invalidSource', message: 'Unknown source' } }); + } + + const discovered = directory ? discoverSkills(directory) : []; const installedByName = new Map(discovered.map((s) => [s.name, s])); - const itemsBySource = {}; - - for (const src of sources) { - // Handle ClawdHub sources separately (API-based, not git-based) - if (src.sourceType === 'clawdhub' || isClawdHubSource(src.source)) { - const cacheKey = 'clawdhub:registry'; - let scanResult = !refresh ? getCachedScan(cacheKey) : null; - - if (!scanResult) { - const scanned = await scanClawdHub(); - if (!scanned.ok) { - itemsBySource[src.id] = []; - continue; - } - scanResult = scanned; - setCachedScan(cacheKey, scanResult); - } - - const items = (scanResult.items || []).map((item) => { - const installed = installedByName.get(item.skillName); - return { - ...item, - sourceId: src.id, - installed: installed - ? { isInstalled: true, scope: installed.scope } - : { isInstalled: false }, - }; - }); - - itemsBySource[src.id] = items; - continue; + if (src.sourceType === 'clawdhub' || isClawdHubSource(src.source)) { + const scanned = await scanClawdHubPage({ cursor: cursor || null }); + if (!scanned.ok) { + return res.status(500).json({ ok: false, error: scanned.error }); } - // Handle GitHub sources (git clone based) - const parsed = parseSkillRepoSource(src.source); - if (!parsed.ok) { - itemsBySource[src.id] = []; - continue; - } - - const effectiveSubpath = src.defaultSubpath || parsed.effectiveSubpath || null; - const cacheKey = getCacheKey({ - normalizedRepo: parsed.normalizedRepo, - subpath: effectiveSubpath || '', - identityId: src.gitIdentityId || '', - }); - - let scanResult = !refresh ? getCachedScan(cacheKey) : null; - if (!scanResult) { - const scanned = await scanSkillsRepository({ - source: src.source, - subpath: src.defaultSubpath, - defaultSubpath: src.defaultSubpath, - identity: resolveGitIdentity(src.gitIdentityId), - }); - - if (!scanned.ok) { - itemsBySource[src.id] = []; - continue; - } - - scanResult = scanned; - setCachedScan(cacheKey, scanResult); - } - - const items = (scanResult.items || []).map((item) => { + const items = (scanned.items || []).map((item) => { const installed = installedByName.get(item.skillName); return { - sourceId: src.id, ...item, - gitIdentityId: src.gitIdentityId, + sourceId: src.id, installed: installed ? { isInstalled: true, scope: installed.scope } : { isInstalled: false }, }; }); - itemsBySource[src.id] = items; + return res.json({ ok: true, items, nextCursor: scanned.nextCursor || null }); } - const sourcesForUi = sources.map(({ gitIdentityId, ...rest }) => rest); - res.json({ ok: true, sources: sourcesForUi, itemsBySource }); + const parsed = parseSkillRepoSource(src.source); + if (!parsed.ok) { + return res.status(400).json({ ok: false, error: parsed.error }); + } + + const effectiveSubpath = src.defaultSubpath || parsed.effectiveSubpath || null; + const cacheKey = getCacheKey({ + normalizedRepo: parsed.normalizedRepo, + subpath: effectiveSubpath || '', + identityId: src.gitIdentityId || '', + }); + + let scanResult = !refresh ? getCachedScan(cacheKey) : null; + if (!scanResult) { + const scanned = await scanSkillsRepository({ + source: src.source, + subpath: src.defaultSubpath, + defaultSubpath: src.defaultSubpath, + identity: resolveGitIdentity(src.gitIdentityId), + }); + + if (!scanned.ok) { + return res.status(500).json({ ok: false, error: scanned.error }); + } + + scanResult = scanned; + setCachedScan(cacheKey, scanResult); + } + + const items = (scanResult.items || []).map((item) => { + const installed = installedByName.get(item.skillName); + return { + sourceId: src.id, + ...item, + gitIdentityId: src.gitIdentityId, + installed: installed + ? { isInstalled: true, scope: installed.scope } + : { isInstalled: false }, + }; + }); + + return res.json({ ok: true, items }); } catch (error) { - console.error('Failed to load skills catalog:', error); - res.status(500).json({ ok: false, error: { kind: 'unknown', message: error.message || 'Failed to load catalog' } }); + console.error('Failed to load catalog source:', error); + return res.status(500).json({ + ok: false, + error: { kind: 'unknown', message: error.message || 'Failed to load catalog source' }, + }); } }); diff --git a/packages/web/server/lib/skills-catalog/clawdhub/api.js b/packages/web/server/lib/skills-catalog/clawdhub/api.js index c82f996d..ac463b52 100644 --- a/packages/web/server/lib/skills-catalog/clawdhub/api.js +++ b/packages/web/server/lib/skills-catalog/clawdhub/api.js @@ -6,29 +6,48 @@ */ const CLAWDHUB_API_BASE = 'https://clawdhub.com/api/v1'; +const CLAWDHUB_PAGE_LIMIT = 25; // Rate limiting: ClawdHub allows 120 requests/minute const RATE_LIMIT_DELAY_MS = 100; let lastRequestTime = 0; async function rateLimitedFetch(url, options = {}) { - const now = Date.now(); - const elapsed = now - lastRequestTime; - if (elapsed < RATE_LIMIT_DELAY_MS) { - await new Promise((resolve) => setTimeout(resolve, RATE_LIMIT_DELAY_MS - elapsed)); + const maxAttempts = 10; + + let lastResponse = null; + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + const now = Date.now(); + const elapsed = now - lastRequestTime; + if (elapsed < RATE_LIMIT_DELAY_MS) { + await new Promise((resolve) => setTimeout(resolve, RATE_LIMIT_DELAY_MS - elapsed)); + } + lastRequestTime = Date.now(); + + const response = await fetch(url, { + ...options, + headers: { + Accept: 'application/json', + 'User-Agent': 'OpenChamber/1.0', + ...options.headers, + }, + }); + + lastResponse = response; + + if (response.status === 429 || response.status >= 500) { + if (attempt < maxAttempts - 1) { + const waitMs = 50 * (attempt + 1); + await new Promise((resolve) => setTimeout(resolve, waitMs)); + continue; + } + } + + return response; } - lastRequestTime = Date.now(); - const response = await fetch(url, { - ...options, - headers: { - Accept: 'application/json', - 'User-Agent': 'OpenChamber/1.0', - ...options.headers, - }, - }); - - return response; + return lastResponse; } /** @@ -39,8 +58,8 @@ async function rateLimitedFetch(url, options = {}) { */ export async function fetchClawdHubSkills({ cursor } = {}) { const url = cursor - ? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}` - : `${CLAWDHUB_API_BASE}/skills`; + ? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}&limit=${CLAWDHUB_PAGE_LIMIT}` + : `${CLAWDHUB_API_BASE}/skills?limit=${CLAWDHUB_PAGE_LIMIT}`; const response = await rateLimitedFetch(url); @@ -50,9 +69,16 @@ export async function fetchClawdHubSkills({ cursor } = {}) { } const data = await response.json(); + const nextCursor = + (typeof data.nextCursor === 'string' && data.nextCursor) || + (typeof data.next_cursor === 'string' && data.next_cursor) || + (typeof data.next === 'string' && data.next) || + (typeof data.cursor === 'string' && data.cursor) || + null; + return { items: data.items || [], - nextCursor: data.nextCursor || null, + nextCursor, }; } @@ -95,7 +121,10 @@ export async function fetchClawdHubSkillVersion(slug, version = 'latest') { * @returns {Promise} - ZIP file contents */ export async function downloadClawdHubSkill(slug, version) { - const url = `${CLAWDHUB_API_BASE}/download?slug=${encodeURIComponent(slug)}&version=${encodeURIComponent(version)}`; + const versionParam = typeof version === 'string' && version !== 'latest' + ? `&version=${encodeURIComponent(version)}` + : '&tag=latest'; + const url = `${CLAWDHUB_API_BASE}/download?slug=${encodeURIComponent(slug)}${versionParam}`; const response = await rateLimitedFetch(url, { headers: { diff --git a/packages/web/server/lib/skills-catalog/clawdhub/index.js b/packages/web/server/lib/skills-catalog/clawdhub/index.js index 29b022c1..f81ca2e4 100644 --- a/packages/web/server/lib/skills-catalog/clawdhub/index.js +++ b/packages/web/server/lib/skills-catalog/clawdhub/index.js @@ -5,7 +5,7 @@ * https://clawdhub.com */ -export { scanClawdHub } from './scan.js'; +export { scanClawdHub, scanClawdHubPage } from './scan.js'; export { installSkillsFromClawdHub } from './install.js'; export { fetchClawdHubSkills, diff --git a/packages/web/server/lib/skills-catalog/clawdhub/install.js b/packages/web/server/lib/skills-catalog/clawdhub/install.js index 34eed46e..39e69e69 100644 --- a/packages/web/server/lib/skills-catalog/clawdhub/install.js +++ b/packages/web/server/lib/skills-catalog/clawdhub/install.js @@ -150,10 +150,17 @@ export async function installSkillsFromClawdHub({ if (resolvedVersion === 'latest') { try { const info = await fetchClawdHubSkillInfo(plan.slug); - resolvedVersion = info.skill?.tags?.latest || info.latestVersion?.version || plan.version; + const latest = info.skill?.tags?.latest || info.latestVersion?.version || null; + if (latest) { + resolvedVersion = latest; + } } catch { - // Fall back to 'latest' tag if info fetch fails - resolvedVersion = 'latest'; + // ignore + } + + if (resolvedVersion === 'latest') { + skipped.push({ skillName: plan.slug, reason: 'Unable to resolve latest version' }); + continue; } } diff --git a/packages/web/server/lib/skills-catalog/clawdhub/scan.js b/packages/web/server/lib/skills-catalog/clawdhub/scan.js index 4448a343..2d70dad1 100644 --- a/packages/web/server/lib/skills-catalog/clawdhub/scan.js +++ b/packages/web/server/lib/skills-catalog/clawdhub/scan.js @@ -8,6 +8,36 @@ import { fetchClawdHubSkills } from './api.js'; const MAX_PAGES = 20; // Safety limit to prevent infinite loops +const CLAWDHUB_PAGE_LIMIT = 25; + +const mapClawdHubItem = (item) => { + const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0'; + + return { + sourceId: 'clawdhub', + repoSource: 'clawdhub:registry', + repoSubpath: null, + gitIdentityId: null, + skillDir: item.slug, + skillName: item.slug, + frontmatterName: item.displayName || item.slug, + description: item.summary || null, + installable: true, + warnings: [], + // ClawdHub-specific metadata + clawdhub: { + slug: item.slug, + version: latestVersion, + displayName: item.displayName, + owner: item.owner?.handle || null, + downloads: item.stats?.downloads || 0, + stars: item.stats?.stars || 0, + versionsCount: item.stats?.versions || 1, + createdAt: item.createdAt, + updatedAt: item.updatedAt, + }, + }; +}; /** * Scan ClawdHub registry for all available skills @@ -19,35 +49,23 @@ export async function scanClawdHub() { let cursor = null; for (let page = 0; page < MAX_PAGES; page++) { - const { items, nextCursor } = await fetchClawdHubSkills({ cursor }); + let items = []; + let nextCursor = null; + + try { + const pageResult = await fetchClawdHubSkills({ cursor }); + items = pageResult.items || []; + nextCursor = pageResult.nextCursor || null; + } catch (error) { + if (page > 0 && allItems.length > 0) { + console.warn('ClawdHub pagination failed; returning partial results.'); + break; + } + throw error; + } for (const item of items) { - const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0'; - - allItems.push({ - sourceId: 'clawdhub', - repoSource: 'clawdhub:registry', - repoSubpath: null, - gitIdentityId: null, - skillDir: item.slug, - skillName: item.slug, - frontmatterName: item.displayName || item.slug, - description: item.summary || null, - installable: true, - warnings: [], - // ClawdHub-specific metadata - clawdhub: { - slug: item.slug, - version: latestVersion, - displayName: item.displayName, - owner: item.owner?.handle || null, - downloads: item.stats?.downloads || 0, - stars: item.stats?.stars || 0, - versionsCount: item.stats?.versions || 1, - createdAt: item.createdAt, - updatedAt: item.updatedAt, - }, - }); + allItems.push(mapClawdHubItem(item)); } if (!nextCursor) { @@ -71,3 +89,25 @@ export async function scanClawdHub() { }; } } + +/** + * Scan a single ClawdHub page (cursor-based) + * @returns {Promise<{ ok: boolean, items?: Array, nextCursor?: string | null, error?: Object }>} + */ +export async function scanClawdHubPage({ cursor } = {}) { + try { + const { items, nextCursor } = await fetchClawdHubSkills({ cursor }); + const mapped = (items || []).map(mapClawdHubItem).slice(0, CLAWDHUB_PAGE_LIMIT); + mapped.sort((a, b) => (b.clawdhub?.downloads || 0) - (a.clawdhub?.downloads || 0)); + return { ok: true, items: mapped, nextCursor: nextCursor || null }; + } catch (error) { + console.error('ClawdHub page scan error:', error); + return { + ok: false, + error: { + kind: 'networkError', + message: error instanceof Error ? error.message : 'Failed to fetch skills from ClawdHub', + }, + }; + } +}