diff --git a/packages/desktop/src-tauri/src/skills_catalog.rs b/packages/desktop/src-tauri/src/skills_catalog.rs index eec8b531..54f1ded0 100644 --- a/packages/desktop/src-tauri/src/skills_catalog.rs +++ b/packages/desktop/src-tauri/src/skills_catalog.rs @@ -721,6 +721,7 @@ fn cache_key(normalized_repo: &str, subpath: Option<&str>, identity_id: Option<& // ============== ClawdHub API ============== const CLAWDHUB_API_BASE: &str = "https://clawdhub.com/api/v1"; +const CLAWDHUB_PAGE_LIMIT: usize = 25; fn is_clawdhub_source(source: &str) -> bool { source.starts_with("clawdhub:") @@ -789,23 +790,56 @@ async fn scan_clawdhub() -> Result> { let mut cursor: Option = None; let max_pages = 20; - for _ in 0..max_pages { + for page in 0..max_pages { let url = match &cursor { Some(c) => format!( - "{}{}?cursor={}", + "{}{}?cursor={}&limit={}", CLAWDHUB_API_BASE, "/skills", - urlencoding::encode(c) + urlencoding::encode(c), + CLAWDHUB_PAGE_LIMIT ), - None => format!("{}/skills", CLAWDHUB_API_BASE), + None => format!("{}/skills?limit={}", CLAWDHUB_API_BASE, CLAWDHUB_PAGE_LIMIT), }; - let response = client.get(&url).send().await?; - if !response.status().is_success() { - return Err(anyhow!("ClawdHub API error: {}", response.status())); + let mut response: Option = None; + let max_attempts = 10; + + for attempt in 0..max_attempts { + let resp = client.get(&url).send().await?; + if resp.status().is_success() { + response = Some(resp); + break; + } + + let should_retry = (resp.status() == reqwest::StatusCode::TOO_MANY_REQUESTS + || resp.status().is_server_error()) + && attempt + 1 < max_attempts; + + if should_retry { + tokio::time::sleep(Duration::from_millis(50 * (attempt + 1) as u64)).await; + continue; + } + + if page > 0 && !all_items.is_empty() { + break; + } + return Err(anyhow!("ClawdHub API error: {}", resp.status())); } - let data: ClawdHubSkillsResponse = response.json().await?; + let Some(response) = response else { + break; + }; + + let data: ClawdHubSkillsResponse = match response.json().await { + Ok(parsed) => parsed, + Err(err) => { + if page > 0 && !all_items.is_empty() { + break; + } + return Err(err.into()); + } + }; for item in data.items { let latest_version = item @@ -1510,12 +1544,22 @@ async fn install_skills_from_clawdhub( // Resolve 'latest' version if version == "latest" { if let Ok(info) = fetch_clawdhub_skill_info(slug).await { - version = info + if let Some(latest) = info .skill .and_then(|s| s.tags) .and_then(|t| t.latest) .or_else(|| info.latest_version.and_then(|v| v.version)) - .unwrap_or_else(|| "latest".to_string()); + { + version = latest; + } + } + + if version == "latest" { + skipped.push(SkippedSkill { + skill_name: slug.to_string(), + reason: "Unable to resolve latest version".to_string(), + }); + continue; } } diff --git a/packages/ui/src/components/sections/agents/AgentsPage.tsx b/packages/ui/src/components/sections/agents/AgentsPage.tsx index 9d72b022..5a7fa1f9 100644 --- a/packages/ui/src/components/sections/agents/AgentsPage.tsx +++ b/packages/ui/src/components/sections/agents/AgentsPage.tsx @@ -147,29 +147,6 @@ const permissionConfigToRuleset = (value: unknown): PermissionRule[] => { return rules; }; -const buildPermissionConfigFromRules = (ruleset: PermissionRule[]): AgentConfig['permission'] | undefined => { - const normalized = normalizeRuleset(ruleset); - if (normalized.length === 0) { - return undefined; - } - - const grouped: Record> = {}; - for (const rule of normalized) { - (grouped[rule.permission] ||= {})[rule.pattern] = rule.action; - } - - const result: Record = {}; - for (const [permissionName, patterns] of Object.entries(grouped)) { - if (Object.keys(patterns).length === 1 && patterns['*']) { - result[permissionName] = patterns['*']; - } else { - result[permissionName] = patterns; - } - } - - return result as AgentConfig['permission']; -}; - const buildPermissionConfigWithGlobal = ( globalAction: PermissionAction, ruleset: PermissionRule[], @@ -201,23 +178,6 @@ const buildPermissionConfigWithGlobal = ( return result as AgentConfig['permission']; }; -const buildPermissionDiffConfig = ( - baselineRules: PermissionRule[], - currentRules: PermissionRule[], -): AgentConfig['permission'] | undefined => { - const baselineMap = buildRuleMap(baselineRules); - const currentMap = buildRuleMap(currentRules); - const changedRules: PermissionRule[] = []; - - for (const [key, rule] of currentMap.entries()) { - const baselineRule = baselineMap.get(key); - if (!baselineRule || baselineRule.action !== rule.action) { - changedRules.push(rule); - } - } - - return buildPermissionConfigFromRules(changedRules); -}; export const AgentsPage: React.FC = () => { const { selectedAgentName, getAgentByName, createAgent, updateAgent, agents, agentDraft, setAgentDraft } = useAgentsStore(); @@ -324,12 +284,6 @@ export const AgentsPage: React.FC = () => { currentRuleMap.get(buildRuleKey(permissionName, '*'))?.action ), [currentRuleMap]); - const getEffectiveWildcardAction = React.useCallback((permissionName: string): PermissionAction => { - if (permissionName === '*') { - return globalPermission; - } - return getWildcardOverride(permissionName) ?? globalPermission; - }, [getWildcardOverride, globalPermission]); const getPatternRules = React.useCallback((permissionName: string): PermissionRule[] => ( permissionRules @@ -348,12 +302,6 @@ export const AgentsPage: React.FC = () => { return Array.from(names).sort((a, b) => a.localeCompare(b)); }, [knownPermissionNames]); - const getFallbackDefaultAction = React.useCallback((permissionName: string): PermissionAction => { - if (permissionName === 'doom_loop' || permissionName === 'external_directory') { - return 'ask'; - } - return 'allow'; - }, []); const getPermissionSummary = React.useCallback((permissionName: string) => { const defaultAction = permissionName === '*' diff --git a/packages/ui/src/components/sections/skills/catalog/SkillsCatalogPage.tsx b/packages/ui/src/components/sections/skills/catalog/SkillsCatalogPage.tsx index 09b63e57..a825b56e 100644 --- a/packages/ui/src/components/sections/skills/catalog/SkillsCatalogPage.tsx +++ b/packages/ui/src/components/sections/skills/catalog/SkillsCatalogPage.tsx @@ -65,7 +65,13 @@ export const SkillsCatalogPage: React.FC = ({ mode, onMo selectedSourceId, setSelectedSource, loadCatalog, + loadSource, + loadMoreClawdHub, isLoadingCatalog, + isLoadingSource, + isLoadingMore, + loadedSourceIds, + clawdhubHasMoreBySource, lastCatalogError, } = useSkillsCatalogStore(); @@ -79,6 +85,15 @@ export const SkillsCatalogPage: React.FC = ({ mode, onMo void loadCatalog(); }, [loadCatalog]); + React.useEffect(() => { + if (!selectedSourceId) { + return; + } + if (!loadedSourceIds[selectedSourceId]) { + void loadSource(selectedSourceId); + } + }, [selectedSourceId, loadedSourceIds, loadSource]); + const items = React.useMemo(() => { if (!selectedSourceId) return []; return itemsBySource[selectedSourceId] || []; @@ -98,6 +113,10 @@ export const SkillsCatalogPage: React.FC = ({ mode, onMo const selectedSource = React.useMemo(() => sources.find((s) => s.id === selectedSourceId) || null, [sources, selectedSourceId]); const isCustomSource = Boolean(selectedSourceId && selectedSourceId.startsWith('custom:')); + const isClawdHubSource = selectedSource?.source === 'clawdhub:registry' || selectedSource?.sourceType === 'clawdhub'; + const hasMoreClawdHub = Boolean( + selectedSourceId && (clawdhubHasMoreBySource[selectedSourceId] ?? true) + ); const removeSelectedCatalog = async () => { if (!selectedSourceId || !isCustomSource) { @@ -167,8 +186,14 @@ export const SkillsCatalogPage: React.FC = ({ mode, onMo + + + ); + })} + {isClawdHubSource && hasMoreClawdHub ? ( +
+
- ); - }) + ) : null} + )} diff --git a/packages/ui/src/lib/api/types.ts b/packages/ui/src/lib/api/types.ts index aadc49ff..21057395 100644 --- a/packages/ui/src/lib/api/types.ts +++ b/packages/ui/src/lib/api/types.ts @@ -828,6 +828,14 @@ export interface SkillsCatalogResponse { ok: boolean; sources?: SkillsCatalogSource[]; itemsBySource?: Record; + pageInfoBySource?: Record; + error?: { kind: string; message: string }; +} + +export interface SkillsCatalogSourceResponse { + ok: boolean; + items?: SkillsCatalogItem[]; + nextCursor?: string | null; error?: { kind: string; message: string }; } diff --git a/packages/ui/src/stores/useSkillsCatalogStore.ts b/packages/ui/src/stores/useSkillsCatalogStore.ts index d7ffc5a4..0c64007b 100644 --- a/packages/ui/src/stores/useSkillsCatalogStore.ts +++ b/packages/ui/src/stores/useSkillsCatalogStore.ts @@ -10,11 +10,30 @@ import type { SkillsInstallRequest, SkillsInstallResponse, SkillsInstallError, + SkillsCatalogSourceResponse, } from '@/lib/api/types'; import { useSkillsStore } from '@/stores/useSkillsStore'; import { opencodeClient } from '@/lib/opencode/client'; +const FALLBACK_SOURCES: SkillsCatalogSource[] = [ + { + id: 'anthropic', + label: 'Anthropic', + description: "Anthropic's public skills repository", + source: 'anthropics/skills', + defaultSubpath: 'skills', + sourceType: 'github', + }, + { + id: 'clawdhub', + label: 'ClawdHub', + description: 'Community skill registry with vector search', + source: 'clawdhub:registry', + sourceType: 'clawdhub', + }, +]; + const getCurrentDirectory = (): string | null => { const opencodeDirectory = opencodeClient.getDirectory(); if (typeof opencodeDirectory === 'string' && opencodeDirectory.trim().length > 0) { @@ -38,8 +57,13 @@ export interface SkillsCatalogState { sources: SkillsCatalogSource[]; itemsBySource: Record; selectedSourceId: string | null; + pageInfoBySource: Record; + loadedSourceIds: Record; + clawdhubHasMoreBySource: Record; isLoadingCatalog: boolean; + isLoadingSource: boolean; + isLoadingMore: boolean; isScanning: boolean; isInstalling: boolean; @@ -52,6 +76,8 @@ export interface SkillsCatalogState { setSelectedSource: (id: string | null) => void; loadCatalog: (options?: { refresh?: boolean }) => Promise; + loadSource: (sourceId: string, options?: { refresh?: boolean }) => Promise; + loadMoreClawdHub: () => Promise; scanRepo: (request: SkillsRepoScanRequest) => Promise; installSkills: (request: SkillsInstallRequest) => Promise; } @@ -59,11 +85,16 @@ export interface SkillsCatalogState { export const useSkillsCatalogStore = create()( devtools( (set, get) => ({ - sources: [], + sources: FALLBACK_SOURCES, itemsBySource: {}, - selectedSourceId: null, + selectedSourceId: FALLBACK_SOURCES[0]?.id ?? null, + pageInfoBySource: {}, + loadedSourceIds: {}, + clawdhubHasMoreBySource: {}, isLoadingCatalog: false, + isLoadingSource: false, + isLoadingMore: false, isScanning: false, isInstalling: false, @@ -81,55 +112,64 @@ export const useSkillsCatalogStore = create()( const previous = { sources: get().sources, itemsBySource: get().itemsBySource, + pageInfoBySource: get().pageInfoBySource, + loadedSourceIds: get().loadedSourceIds, + clawdhubHasMoreBySource: get().clawdhubHasMoreBySource, }; let lastError: SkillsCatalogResponse['error'] | null = null; try { - for (let attempt = 0; attempt < 3; attempt++) { - try { - const currentDirectory = getCurrentDirectory(); - const refresh = options?.refresh ? '&refresh=true' : ''; - const queryParams = currentDirectory - ? `?directory=${encodeURIComponent(currentDirectory)}${refresh}` - : refresh - ? `?refresh=true` - : ''; + const refresh = options?.refresh ? '?refresh=true' : ''; + const controller = new AbortController(); + const timeoutId = window.setTimeout(() => controller.abort(), 3000); - const response = await fetch(`/api/config/skills/catalog${queryParams}`, { - method: 'GET', - headers: { Accept: 'application/json' }, - }); + try { + const response = await fetch(`/api/config/skills/catalog${refresh}`, { + method: 'GET', + headers: { Accept: 'application/json' }, + signal: controller.signal, + }); - const payload = (await response.json().catch(() => null)) as SkillsCatalogResponse | null; - if (!response.ok || !payload?.ok) { - lastError = payload?.error || { kind: 'unknown', message: `Failed to load catalog (${response.status})` }; - const waitMs = 200 * (attempt + 1); - await new Promise((resolve) => setTimeout(resolve, waitMs)); - continue; - } - - const sources = payload.sources || []; - const itemsBySource = payload.itemsBySource || {}; - const currentSelected = get().selectedSourceId; - const selectedSourceId = - (currentSelected && sources.some((s) => s.id === currentSelected)) - ? currentSelected - : (sources[0]?.id ?? null); - - set({ sources, itemsBySource, selectedSourceId }); - return true; - - } catch (error) { - lastError = { kind: 'unknown', message: error instanceof Error ? error.message : String(error) }; - const waitMs = 200 * (attempt + 1); - await new Promise((resolve) => setTimeout(resolve, waitMs)); + const payload = (await response.json().catch(() => null)) as SkillsCatalogResponse | null; + if (!response.ok || !payload?.ok) { + lastError = payload?.error || { kind: 'unknown', message: `Failed to load catalog (${response.status})` }; + throw new Error(lastError.message); } - } + const sources = (payload.sources && payload.sources.length > 0) ? payload.sources : previous.sources; + const itemsBySource = options?.refresh ? {} : (get().itemsBySource || {}); + const pageInfoBySource = options?.refresh ? {} : (get().pageInfoBySource || {}); + const loadedSourceIds = options?.refresh ? {} : (get().loadedSourceIds || {}); + const clawdhubHasMoreBySource = options?.refresh ? {} : (get().clawdhubHasMoreBySource || {}); + const currentSelected = get().selectedSourceId; + const selectedSourceId = + (currentSelected && sources.some((s) => s.id === currentSelected)) + ? currentSelected + : (sources[0]?.id ?? null); + + set({ + sources, + itemsBySource, + pageInfoBySource, + loadedSourceIds, + clawdhubHasMoreBySource, + selectedSourceId, + }); + + return true; + } finally { + window.clearTimeout(timeoutId); + } + } catch (error) { + lastError = lastError || { kind: 'unknown', message: error instanceof Error ? error.message : String(error) }; + set({ sources: previous.sources, itemsBySource: previous.itemsBySource, + pageInfoBySource: previous.pageInfoBySource, + loadedSourceIds: previous.loadedSourceIds, + clawdhubHasMoreBySource: previous.clawdhubHasMoreBySource, lastCatalogError: lastError || { kind: 'unknown', message: 'Failed to load catalog' }, }); @@ -139,6 +179,144 @@ export const useSkillsCatalogStore = create()( } }, + loadSource: async (sourceId, options) => { + if (!sourceId) { + return false; + } + + set({ isLoadingSource: true, lastCatalogError: null }); + + try { + const currentDirectory = getCurrentDirectory(); + const refresh = options?.refresh ? '&refresh=true' : ''; + const queryParams = currentDirectory + ? `?directory=${encodeURIComponent(currentDirectory)}&sourceId=${encodeURIComponent(sourceId)}${refresh}` + : `?sourceId=${encodeURIComponent(sourceId)}${refresh}`; + + const response = await fetch(`/api/config/skills/catalog/source${queryParams}`, { + method: 'GET', + headers: { Accept: 'application/json' }, + }); + + const payload = (await response.json().catch(() => null)) as SkillsCatalogSourceResponse | null; + const hasItems = Array.isArray((payload as SkillsCatalogSourceResponse | null)?.items); + if (!response.ok || (!payload?.ok && !hasItems)) { + const fallback = await fetch(`/api/config/skills/catalog${queryParams}`, { + method: 'GET', + headers: { Accept: 'application/json' }, + }); + const fallbackPayload = (await fallback.json().catch(() => null)) as SkillsCatalogResponse | null; + const fallbackItems = fallbackPayload?.itemsBySource?.[sourceId]; + if (fallback.ok && fallbackPayload?.ok && Array.isArray(fallbackItems)) { + set((state) => ({ + itemsBySource: { ...state.itemsBySource, [sourceId]: fallbackItems }, + pageInfoBySource: { ...state.pageInfoBySource, [sourceId]: { nextCursor: null } }, + loadedSourceIds: { ...state.loadedSourceIds, [sourceId]: true }, + clawdhubHasMoreBySource: { ...state.clawdhubHasMoreBySource, [sourceId]: false }, + })); + return true; + } + + set({ + lastCatalogError: payload?.error || { kind: 'unknown', message: `Failed to load source (${response.status})` }, + }); + return false; + } + + const items = payload?.items || []; + const nextCursor = payload?.nextCursor ?? null; + + set((state) => ({ + itemsBySource: { ...state.itemsBySource, [sourceId]: items }, + pageInfoBySource: { ...state.pageInfoBySource, [sourceId]: { nextCursor } }, + loadedSourceIds: { ...state.loadedSourceIds, [sourceId]: true }, + clawdhubHasMoreBySource: { + ...state.clawdhubHasMoreBySource, + [sourceId]: items.length > 0, + }, + })); + + return true; + } catch (error) { + set({ + lastCatalogError: { kind: 'unknown', message: error instanceof Error ? error.message : String(error) }, + }); + return false; + } finally { + set({ isLoadingSource: false }); + } + }, + + loadMoreClawdHub: async () => { + const selectedSourceId = get().selectedSourceId; + if (!selectedSourceId) { + return false; + } + + const pageInfo = get().pageInfoBySource[selectedSourceId]; + const cursor = pageInfo?.nextCursor || null; + + set({ isLoadingMore: true }); + try { + const currentDirectory = getCurrentDirectory(); + const parts = [`sourceId=${encodeURIComponent(selectedSourceId)}`]; + if (currentDirectory) { + parts.push(`directory=${encodeURIComponent(currentDirectory)}`); + } + if (cursor) { + parts.push(`cursor=${encodeURIComponent(cursor)}`); + } + const queryParams = `?${parts.join('&')}`; + + const response = await fetch(`/api/config/skills/catalog/source${queryParams}`, { + method: 'GET', + headers: { Accept: 'application/json' }, + }); + + const payload = (await response.json().catch(() => null)) as SkillsCatalogSourceResponse | null; + if (!response.ok || !payload?.ok) { + return false; + } + + const nextCursor = payload.nextCursor ?? null; + const currentItems = get().itemsBySource[selectedSourceId] || []; + const items = payload.items || []; + const merged = new Map(currentItems.map((item) => [`${item.sourceId}:${item.skillDir}`, item])); + let newCount = 0; + + for (const item of items) { + const key = `${item.sourceId}:${item.skillDir}`; + if (!merged.has(key)) { + newCount += 1; + } + merged.set(key, item); + } + + const noMore = items.length === 0 || newCount === 0; + + set((state) => ({ + itemsBySource: { + ...state.itemsBySource, + [selectedSourceId]: Array.from(merged.values()), + }, + pageInfoBySource: { + ...state.pageInfoBySource, + [selectedSourceId]: { nextCursor }, + }, + clawdhubHasMoreBySource: { + ...state.clawdhubHasMoreBySource, + [selectedSourceId]: !noMore, + }, + })); + + return true; + } catch { + return false; + } finally { + set({ isLoadingMore: false }); + } + }, + scanRepo: async (request) => { set({ isScanning: true, lastScanError: null, scanResults: null }); try { diff --git a/packages/vscode/src/skillsCatalog.ts b/packages/vscode/src/skillsCatalog.ts index 71c7a9ce..712572d8 100644 --- a/packages/vscode/src/skillsCatalog.ts +++ b/packages/vscode/src/skillsCatalog.ts @@ -94,6 +94,7 @@ export const CURATED_SOURCES: CuratedSource[] = [ // ============== ClawdHub API ============== const CLAWDHUB_API_BASE = 'https://clawdhub.com/api/v1'; +const CLAWDHUB_PAGE_LIMIT = 25; const CLAWDHUB_RATE_LIMIT_MS = 100; let clawdhubLastRequest = 0; @@ -102,21 +103,40 @@ function isClawdHubSource(source: string): boolean { } async function clawdhubFetch(url: string, options?: RequestInit): Promise { - const now = Date.now(); - const elapsed = now - clawdhubLastRequest; - if (elapsed < CLAWDHUB_RATE_LIMIT_MS) { - await new Promise((resolve) => setTimeout(resolve, CLAWDHUB_RATE_LIMIT_MS - elapsed)); - } - clawdhubLastRequest = Date.now(); + const maxAttempts = 10; + let lastResponse: Response | null = null; - return fetch(url, { - ...options, - headers: { - Accept: 'application/json', - 'User-Agent': 'OpenChamber-VSCode/1.0', - ...options?.headers, - }, - }); + for (let attempt = 0; attempt < maxAttempts; attempt++) { + const now = Date.now(); + const elapsed = now - clawdhubLastRequest; + if (elapsed < CLAWDHUB_RATE_LIMIT_MS) { + await new Promise((resolve) => setTimeout(resolve, CLAWDHUB_RATE_LIMIT_MS - elapsed)); + } + clawdhubLastRequest = Date.now(); + + const response = await fetch(url, { + ...options, + headers: { + Accept: 'application/json', + 'User-Agent': 'OpenChamber-VSCode/1.0', + ...options?.headers, + }, + }); + + lastResponse = response; + + if (response.status === 429 || response.status >= 500) { + if (attempt < maxAttempts - 1) { + const waitMs = 50 * (attempt + 1); + await new Promise((resolve) => setTimeout(resolve, waitMs)); + continue; + } + } + + return response; + } + + return lastResponse as Response; } type ClawdHubSkillListItem = { @@ -142,16 +162,25 @@ async function scanClawdHub(): Promise { for (let page = 0; page < maxPages; page++) { const url = cursor - ? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}` - : `${CLAWDHUB_API_BASE}/skills`; + ? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}&limit=${CLAWDHUB_PAGE_LIMIT}` + : `${CLAWDHUB_API_BASE}/skills?limit=${CLAWDHUB_PAGE_LIMIT}`; - const response = await clawdhubFetch(url); - if (!response.ok) { - throw new Error(`ClawdHub API error: ${response.status}`); + let data: ClawdHubSkillsResponse; + + try { + const response = await clawdhubFetch(url); + if (!response.ok) { + throw new Error(`ClawdHub API error: ${response.status}`); + } + + data = (await response.json()) as ClawdHubSkillsResponse; + } catch (error) { + if (page > 0 && allItems.length > 0) { + break; + } + throw error; } - const data = (await response.json()) as ClawdHubSkillsResponse; - for (const item of data.items || []) { const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0'; @@ -275,15 +304,23 @@ export async function installSkillsFromClawdHub(options: { try { // Resolve 'latest' version - if (version === 'latest') { - try { - const info = await fetchClawdHubSkillInfo(slug); - version = info.skill?.tags?.latest || info.latestVersion?.version || version; - } catch { - // Fall back to 'latest' + if (version === 'latest') { + try { + const info = await fetchClawdHubSkillInfo(slug); + const latest = info.skill?.tags?.latest || info.latestVersion?.version || null; + if (latest) { + version = latest; } + } catch { + // ignore } + if (version === 'latest') { + skipped.push({ skillName: slug, reason: 'Unable to resolve latest version' }); + continue; + } + } + const targetDir = options.scope === 'user' ? path.join(userSkillDir, slug) : path.join(options.workingDirectory as string, '.opencode', 'skills', slug); diff --git a/packages/web/server/index.js b/packages/web/server/index.js index 17bd98c5..80456445 100644 --- a/packages/web/server/index.js +++ b/packages/web/server/index.js @@ -533,6 +533,25 @@ const resolveProjectDirectory = async (req) => { return { directory: validated.directory, error: null }; }; +const resolveOptionalProjectDirectory = async (req) => { + const headerDirectory = typeof req.get === 'function' ? req.get('x-opencode-directory') : null; + const queryDirectory = Array.isArray(req.query?.directory) + ? req.query.directory[0] + : req.query?.directory; + const requested = headerDirectory || queryDirectory || null; + + if (!requested) { + return { directory: null, error: null }; + } + + const validated = await validateDirectoryPath(requested); + if (!validated.ok) { + return { directory: null, error: validated.error }; + } + + return { directory: validated.directory, error: null }; +}; + const sanitizeTypographySizesPartial = (input) => { if (!input || typeof input !== 'object') { return undefined; @@ -2416,6 +2435,7 @@ function setupProxy(app) { req.path.startsWith('/push') || req.path.startsWith('/config/agents') || req.path.startsWith('/config/settings') || + req.path.startsWith('/config/skills') || req.path === '/config/reload' || req.path === '/health' ) { @@ -2448,6 +2468,7 @@ function setupProxy(app) { req.path.startsWith('/themes/custom') || req.path.startsWith('/config/agents') || req.path.startsWith('/config/settings') || + req.path.startsWith('/config/skills') || req.path === '/health' ) { return next(); @@ -3508,7 +3529,7 @@ async function main(options = {}) { const { parseSkillRepoSource } = await import('./lib/skills-catalog/source.js'); const { scanSkillsRepository } = await import('./lib/skills-catalog/scan.js'); const { installSkillsFromRepository } = await import('./lib/skills-catalog/install.js'); - const { scanClawdHub, installSkillsFromClawdHub, isClawdHubSource } = await import('./lib/skills-catalog/clawdhub/index.js'); + const { scanClawdHubPage, installSkillsFromClawdHub, isClawdHubSource } = await import('./lib/skills-catalog/clawdhub/index.js'); const { getProfiles, getProfile } = await import('./lib/git-identity-storage.js'); const listGitIdentitiesForResponse = () => { @@ -3538,11 +3559,10 @@ async function main(options = {}) { app.get('/api/config/skills/catalog', async (req, res) => { try { - const { directory, error } = await resolveProjectDirectory(req); - if (!directory) { + const { error } = await resolveOptionalProjectDirectory(req); + if (error) { return res.status(400).json({ error }); } - const refresh = String(req.query.refresh || '').toLowerCase() === 'true'; const curatedSources = getCuratedSkillsSources(); const settings = await readSettingsFromDisk(); @@ -3558,95 +3578,121 @@ async function main(options = {}) { })); const sources = [...curatedSources, ...customSources]; + const sourcesForUi = sources.map(({ gitIdentityId, ...rest }) => rest); - const discovered = discoverSkills(directory); + res.json({ ok: true, sources: sourcesForUi, itemsBySource: {}, pageInfoBySource: {} }); + } catch (error) { + console.error('Failed to load skills catalog:', error); + res.status(500).json({ ok: false, error: { kind: 'unknown', message: error.message || 'Failed to load catalog' } }); + } + }); + + app.get('/api/config/skills/catalog/source', async (req, res) => { + try { + const { directory, error } = await resolveOptionalProjectDirectory(req); + if (error) { + return res.status(400).json({ ok: false, error: { kind: 'invalidSource', message: error } }); + } + + const sourceId = typeof req.query.sourceId === 'string' ? req.query.sourceId : null; + if (!sourceId) { + return res.status(400).json({ ok: false, error: { kind: 'invalidSource', message: 'Missing sourceId' } }); + } + + const refresh = String(req.query.refresh || '').toLowerCase() === 'true'; + const cursor = typeof req.query.cursor === 'string' ? req.query.cursor : null; + + const curatedSources = getCuratedSkillsSources(); + const settings = await readSettingsFromDisk(); + const customSourcesRaw = sanitizeSkillCatalogs(settings.skillCatalogs) || []; + + const customSources = customSourcesRaw.map((entry) => ({ + id: entry.id, + label: entry.label, + description: entry.source, + source: entry.source, + defaultSubpath: entry.subpath, + gitIdentityId: entry.gitIdentityId, + })); + + const sources = [...curatedSources, ...customSources]; + const src = sources.find((entry) => entry.id === sourceId); + + if (!src) { + return res.status(404).json({ ok: false, error: { kind: 'invalidSource', message: 'Unknown source' } }); + } + + const discovered = directory ? discoverSkills(directory) : []; const installedByName = new Map(discovered.map((s) => [s.name, s])); - const itemsBySource = {}; - - for (const src of sources) { - // Handle ClawdHub sources separately (API-based, not git-based) - if (src.sourceType === 'clawdhub' || isClawdHubSource(src.source)) { - const cacheKey = 'clawdhub:registry'; - let scanResult = !refresh ? getCachedScan(cacheKey) : null; - - if (!scanResult) { - const scanned = await scanClawdHub(); - if (!scanned.ok) { - itemsBySource[src.id] = []; - continue; - } - scanResult = scanned; - setCachedScan(cacheKey, scanResult); - } - - const items = (scanResult.items || []).map((item) => { - const installed = installedByName.get(item.skillName); - return { - ...item, - sourceId: src.id, - installed: installed - ? { isInstalled: true, scope: installed.scope } - : { isInstalled: false }, - }; - }); - - itemsBySource[src.id] = items; - continue; + if (src.sourceType === 'clawdhub' || isClawdHubSource(src.source)) { + const scanned = await scanClawdHubPage({ cursor: cursor || null }); + if (!scanned.ok) { + return res.status(500).json({ ok: false, error: scanned.error }); } - // Handle GitHub sources (git clone based) - const parsed = parseSkillRepoSource(src.source); - if (!parsed.ok) { - itemsBySource[src.id] = []; - continue; - } - - const effectiveSubpath = src.defaultSubpath || parsed.effectiveSubpath || null; - const cacheKey = getCacheKey({ - normalizedRepo: parsed.normalizedRepo, - subpath: effectiveSubpath || '', - identityId: src.gitIdentityId || '', - }); - - let scanResult = !refresh ? getCachedScan(cacheKey) : null; - if (!scanResult) { - const scanned = await scanSkillsRepository({ - source: src.source, - subpath: src.defaultSubpath, - defaultSubpath: src.defaultSubpath, - identity: resolveGitIdentity(src.gitIdentityId), - }); - - if (!scanned.ok) { - itemsBySource[src.id] = []; - continue; - } - - scanResult = scanned; - setCachedScan(cacheKey, scanResult); - } - - const items = (scanResult.items || []).map((item) => { + const items = (scanned.items || []).map((item) => { const installed = installedByName.get(item.skillName); return { - sourceId: src.id, ...item, - gitIdentityId: src.gitIdentityId, + sourceId: src.id, installed: installed ? { isInstalled: true, scope: installed.scope } : { isInstalled: false }, }; }); - itemsBySource[src.id] = items; + return res.json({ ok: true, items, nextCursor: scanned.nextCursor || null }); } - const sourcesForUi = sources.map(({ gitIdentityId, ...rest }) => rest); - res.json({ ok: true, sources: sourcesForUi, itemsBySource }); + const parsed = parseSkillRepoSource(src.source); + if (!parsed.ok) { + return res.status(400).json({ ok: false, error: parsed.error }); + } + + const effectiveSubpath = src.defaultSubpath || parsed.effectiveSubpath || null; + const cacheKey = getCacheKey({ + normalizedRepo: parsed.normalizedRepo, + subpath: effectiveSubpath || '', + identityId: src.gitIdentityId || '', + }); + + let scanResult = !refresh ? getCachedScan(cacheKey) : null; + if (!scanResult) { + const scanned = await scanSkillsRepository({ + source: src.source, + subpath: src.defaultSubpath, + defaultSubpath: src.defaultSubpath, + identity: resolveGitIdentity(src.gitIdentityId), + }); + + if (!scanned.ok) { + return res.status(500).json({ ok: false, error: scanned.error }); + } + + scanResult = scanned; + setCachedScan(cacheKey, scanResult); + } + + const items = (scanResult.items || []).map((item) => { + const installed = installedByName.get(item.skillName); + return { + sourceId: src.id, + ...item, + gitIdentityId: src.gitIdentityId, + installed: installed + ? { isInstalled: true, scope: installed.scope } + : { isInstalled: false }, + }; + }); + + return res.json({ ok: true, items }); } catch (error) { - console.error('Failed to load skills catalog:', error); - res.status(500).json({ ok: false, error: { kind: 'unknown', message: error.message || 'Failed to load catalog' } }); + console.error('Failed to load catalog source:', error); + return res.status(500).json({ + ok: false, + error: { kind: 'unknown', message: error.message || 'Failed to load catalog source' }, + }); } }); diff --git a/packages/web/server/lib/skills-catalog/clawdhub/api.js b/packages/web/server/lib/skills-catalog/clawdhub/api.js index c82f996d..ac463b52 100644 --- a/packages/web/server/lib/skills-catalog/clawdhub/api.js +++ b/packages/web/server/lib/skills-catalog/clawdhub/api.js @@ -6,29 +6,48 @@ */ const CLAWDHUB_API_BASE = 'https://clawdhub.com/api/v1'; +const CLAWDHUB_PAGE_LIMIT = 25; // Rate limiting: ClawdHub allows 120 requests/minute const RATE_LIMIT_DELAY_MS = 100; let lastRequestTime = 0; async function rateLimitedFetch(url, options = {}) { - const now = Date.now(); - const elapsed = now - lastRequestTime; - if (elapsed < RATE_LIMIT_DELAY_MS) { - await new Promise((resolve) => setTimeout(resolve, RATE_LIMIT_DELAY_MS - elapsed)); + const maxAttempts = 10; + + let lastResponse = null; + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + const now = Date.now(); + const elapsed = now - lastRequestTime; + if (elapsed < RATE_LIMIT_DELAY_MS) { + await new Promise((resolve) => setTimeout(resolve, RATE_LIMIT_DELAY_MS - elapsed)); + } + lastRequestTime = Date.now(); + + const response = await fetch(url, { + ...options, + headers: { + Accept: 'application/json', + 'User-Agent': 'OpenChamber/1.0', + ...options.headers, + }, + }); + + lastResponse = response; + + if (response.status === 429 || response.status >= 500) { + if (attempt < maxAttempts - 1) { + const waitMs = 50 * (attempt + 1); + await new Promise((resolve) => setTimeout(resolve, waitMs)); + continue; + } + } + + return response; } - lastRequestTime = Date.now(); - const response = await fetch(url, { - ...options, - headers: { - Accept: 'application/json', - 'User-Agent': 'OpenChamber/1.0', - ...options.headers, - }, - }); - - return response; + return lastResponse; } /** @@ -39,8 +58,8 @@ async function rateLimitedFetch(url, options = {}) { */ export async function fetchClawdHubSkills({ cursor } = {}) { const url = cursor - ? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}` - : `${CLAWDHUB_API_BASE}/skills`; + ? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}&limit=${CLAWDHUB_PAGE_LIMIT}` + : `${CLAWDHUB_API_BASE}/skills?limit=${CLAWDHUB_PAGE_LIMIT}`; const response = await rateLimitedFetch(url); @@ -50,9 +69,16 @@ export async function fetchClawdHubSkills({ cursor } = {}) { } const data = await response.json(); + const nextCursor = + (typeof data.nextCursor === 'string' && data.nextCursor) || + (typeof data.next_cursor === 'string' && data.next_cursor) || + (typeof data.next === 'string' && data.next) || + (typeof data.cursor === 'string' && data.cursor) || + null; + return { items: data.items || [], - nextCursor: data.nextCursor || null, + nextCursor, }; } @@ -95,7 +121,10 @@ export async function fetchClawdHubSkillVersion(slug, version = 'latest') { * @returns {Promise} - ZIP file contents */ export async function downloadClawdHubSkill(slug, version) { - const url = `${CLAWDHUB_API_BASE}/download?slug=${encodeURIComponent(slug)}&version=${encodeURIComponent(version)}`; + const versionParam = typeof version === 'string' && version !== 'latest' + ? `&version=${encodeURIComponent(version)}` + : '&tag=latest'; + const url = `${CLAWDHUB_API_BASE}/download?slug=${encodeURIComponent(slug)}${versionParam}`; const response = await rateLimitedFetch(url, { headers: { diff --git a/packages/web/server/lib/skills-catalog/clawdhub/index.js b/packages/web/server/lib/skills-catalog/clawdhub/index.js index 29b022c1..f81ca2e4 100644 --- a/packages/web/server/lib/skills-catalog/clawdhub/index.js +++ b/packages/web/server/lib/skills-catalog/clawdhub/index.js @@ -5,7 +5,7 @@ * https://clawdhub.com */ -export { scanClawdHub } from './scan.js'; +export { scanClawdHub, scanClawdHubPage } from './scan.js'; export { installSkillsFromClawdHub } from './install.js'; export { fetchClawdHubSkills, diff --git a/packages/web/server/lib/skills-catalog/clawdhub/install.js b/packages/web/server/lib/skills-catalog/clawdhub/install.js index 34eed46e..39e69e69 100644 --- a/packages/web/server/lib/skills-catalog/clawdhub/install.js +++ b/packages/web/server/lib/skills-catalog/clawdhub/install.js @@ -150,10 +150,17 @@ export async function installSkillsFromClawdHub({ if (resolvedVersion === 'latest') { try { const info = await fetchClawdHubSkillInfo(plan.slug); - resolvedVersion = info.skill?.tags?.latest || info.latestVersion?.version || plan.version; + const latest = info.skill?.tags?.latest || info.latestVersion?.version || null; + if (latest) { + resolvedVersion = latest; + } } catch { - // Fall back to 'latest' tag if info fetch fails - resolvedVersion = 'latest'; + // ignore + } + + if (resolvedVersion === 'latest') { + skipped.push({ skillName: plan.slug, reason: 'Unable to resolve latest version' }); + continue; } } diff --git a/packages/web/server/lib/skills-catalog/clawdhub/scan.js b/packages/web/server/lib/skills-catalog/clawdhub/scan.js index 4448a343..2d70dad1 100644 --- a/packages/web/server/lib/skills-catalog/clawdhub/scan.js +++ b/packages/web/server/lib/skills-catalog/clawdhub/scan.js @@ -8,6 +8,36 @@ import { fetchClawdHubSkills } from './api.js'; const MAX_PAGES = 20; // Safety limit to prevent infinite loops +const CLAWDHUB_PAGE_LIMIT = 25; + +const mapClawdHubItem = (item) => { + const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0'; + + return { + sourceId: 'clawdhub', + repoSource: 'clawdhub:registry', + repoSubpath: null, + gitIdentityId: null, + skillDir: item.slug, + skillName: item.slug, + frontmatterName: item.displayName || item.slug, + description: item.summary || null, + installable: true, + warnings: [], + // ClawdHub-specific metadata + clawdhub: { + slug: item.slug, + version: latestVersion, + displayName: item.displayName, + owner: item.owner?.handle || null, + downloads: item.stats?.downloads || 0, + stars: item.stats?.stars || 0, + versionsCount: item.stats?.versions || 1, + createdAt: item.createdAt, + updatedAt: item.updatedAt, + }, + }; +}; /** * Scan ClawdHub registry for all available skills @@ -19,35 +49,23 @@ export async function scanClawdHub() { let cursor = null; for (let page = 0; page < MAX_PAGES; page++) { - const { items, nextCursor } = await fetchClawdHubSkills({ cursor }); + let items = []; + let nextCursor = null; + + try { + const pageResult = await fetchClawdHubSkills({ cursor }); + items = pageResult.items || []; + nextCursor = pageResult.nextCursor || null; + } catch (error) { + if (page > 0 && allItems.length > 0) { + console.warn('ClawdHub pagination failed; returning partial results.'); + break; + } + throw error; + } for (const item of items) { - const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0'; - - allItems.push({ - sourceId: 'clawdhub', - repoSource: 'clawdhub:registry', - repoSubpath: null, - gitIdentityId: null, - skillDir: item.slug, - skillName: item.slug, - frontmatterName: item.displayName || item.slug, - description: item.summary || null, - installable: true, - warnings: [], - // ClawdHub-specific metadata - clawdhub: { - slug: item.slug, - version: latestVersion, - displayName: item.displayName, - owner: item.owner?.handle || null, - downloads: item.stats?.downloads || 0, - stars: item.stats?.stars || 0, - versionsCount: item.stats?.versions || 1, - createdAt: item.createdAt, - updatedAt: item.updatedAt, - }, - }); + allItems.push(mapClawdHubItem(item)); } if (!nextCursor) { @@ -71,3 +89,25 @@ export async function scanClawdHub() { }; } } + +/** + * Scan a single ClawdHub page (cursor-based) + * @returns {Promise<{ ok: boolean, items?: Array, nextCursor?: string | null, error?: Object }>} + */ +export async function scanClawdHubPage({ cursor } = {}) { + try { + const { items, nextCursor } = await fetchClawdHubSkills({ cursor }); + const mapped = (items || []).map(mapClawdHubItem).slice(0, CLAWDHUB_PAGE_LIMIT); + mapped.sort((a, b) => (b.clawdhub?.downloads || 0) - (a.clawdhub?.downloads || 0)); + return { ok: true, items: mapped, nextCursor: nextCursor || null }; + } catch (error) { + console.error('ClawdHub page scan error:', error); + return { + ok: false, + error: { + kind: 'networkError', + message: error instanceof Error ? error.message : 'Failed to fetch skills from ClawdHub', + }, + }; + } +}