feat: enhance ClawdHub integration with paging and retry

Add paging support when loading ClawdHub skills
Retry on API errors and throttle for ClawdHub requests
Load source content on source change and show loading indicators
This commit is contained in:
Bohdan Triapitsyn
2026-01-26 19:50:11 +02:00
parent eec170771c
commit 25525d5b1b
11 changed files with 699 additions and 319 deletions
@@ -721,6 +721,7 @@ fn cache_key(normalized_repo: &str, subpath: Option<&str>, identity_id: Option<&
// ============== ClawdHub API ============== // ============== ClawdHub API ==============
const CLAWDHUB_API_BASE: &str = "https://clawdhub.com/api/v1"; const CLAWDHUB_API_BASE: &str = "https://clawdhub.com/api/v1";
const CLAWDHUB_PAGE_LIMIT: usize = 25;
fn is_clawdhub_source(source: &str) -> bool { fn is_clawdhub_source(source: &str) -> bool {
source.starts_with("clawdhub:") source.starts_with("clawdhub:")
@@ -789,23 +790,56 @@ async fn scan_clawdhub() -> Result<Vec<SkillsCatalogItem>> {
let mut cursor: Option<String> = None; let mut cursor: Option<String> = None;
let max_pages = 20; let max_pages = 20;
for _ in 0..max_pages { for page in 0..max_pages {
let url = match &cursor { let url = match &cursor {
Some(c) => format!( Some(c) => format!(
"{}{}?cursor={}", "{}{}?cursor={}&limit={}",
CLAWDHUB_API_BASE, CLAWDHUB_API_BASE,
"/skills", "/skills",
urlencoding::encode(c) urlencoding::encode(c),
CLAWDHUB_PAGE_LIMIT
), ),
None => format!("{}/skills", CLAWDHUB_API_BASE), None => format!("{}/skills?limit={}", CLAWDHUB_API_BASE, CLAWDHUB_PAGE_LIMIT),
}; };
let response = client.get(&url).send().await?; let mut response: Option<reqwest::Response> = None;
if !response.status().is_success() { let max_attempts = 10;
return Err(anyhow!("ClawdHub API error: {}", response.status()));
for attempt in 0..max_attempts {
let resp = client.get(&url).send().await?;
if resp.status().is_success() {
response = Some(resp);
break;
} }
let data: ClawdHubSkillsResponse = response.json().await?; let should_retry = (resp.status() == reqwest::StatusCode::TOO_MANY_REQUESTS
|| resp.status().is_server_error())
&& attempt + 1 < max_attempts;
if should_retry {
tokio::time::sleep(Duration::from_millis(50 * (attempt + 1) as u64)).await;
continue;
}
if page > 0 && !all_items.is_empty() {
break;
}
return Err(anyhow!("ClawdHub API error: {}", resp.status()));
}
let Some(response) = response else {
break;
};
let data: ClawdHubSkillsResponse = match response.json().await {
Ok(parsed) => parsed,
Err(err) => {
if page > 0 && !all_items.is_empty() {
break;
}
return Err(err.into());
}
};
for item in data.items { for item in data.items {
let latest_version = item let latest_version = item
@@ -1510,12 +1544,22 @@ async fn install_skills_from_clawdhub(
// Resolve 'latest' version // Resolve 'latest' version
if version == "latest" { if version == "latest" {
if let Ok(info) = fetch_clawdhub_skill_info(slug).await { if let Ok(info) = fetch_clawdhub_skill_info(slug).await {
version = info if let Some(latest) = info
.skill .skill
.and_then(|s| s.tags) .and_then(|s| s.tags)
.and_then(|t| t.latest) .and_then(|t| t.latest)
.or_else(|| info.latest_version.and_then(|v| v.version)) .or_else(|| info.latest_version.and_then(|v| v.version))
.unwrap_or_else(|| "latest".to_string()); {
version = latest;
}
}
if version == "latest" {
skipped.push(SkippedSkill {
skill_name: slug.to_string(),
reason: "Unable to resolve latest version".to_string(),
});
continue;
} }
} }
@@ -147,29 +147,6 @@ const permissionConfigToRuleset = (value: unknown): PermissionRule[] => {
return rules; return rules;
}; };
const buildPermissionConfigFromRules = (ruleset: PermissionRule[]): AgentConfig['permission'] | undefined => {
const normalized = normalizeRuleset(ruleset);
if (normalized.length === 0) {
return undefined;
}
const grouped: Record<string, Record<string, PermissionAction>> = {};
for (const rule of normalized) {
(grouped[rule.permission] ||= {})[rule.pattern] = rule.action;
}
const result: Record<string, PermissionConfigValue> = {};
for (const [permissionName, patterns] of Object.entries(grouped)) {
if (Object.keys(patterns).length === 1 && patterns['*']) {
result[permissionName] = patterns['*'];
} else {
result[permissionName] = patterns;
}
}
return result as AgentConfig['permission'];
};
const buildPermissionConfigWithGlobal = ( const buildPermissionConfigWithGlobal = (
globalAction: PermissionAction, globalAction: PermissionAction,
ruleset: PermissionRule[], ruleset: PermissionRule[],
@@ -201,23 +178,6 @@ const buildPermissionConfigWithGlobal = (
return result as AgentConfig['permission']; return result as AgentConfig['permission'];
}; };
const buildPermissionDiffConfig = (
baselineRules: PermissionRule[],
currentRules: PermissionRule[],
): AgentConfig['permission'] | undefined => {
const baselineMap = buildRuleMap(baselineRules);
const currentMap = buildRuleMap(currentRules);
const changedRules: PermissionRule[] = [];
for (const [key, rule] of currentMap.entries()) {
const baselineRule = baselineMap.get(key);
if (!baselineRule || baselineRule.action !== rule.action) {
changedRules.push(rule);
}
}
return buildPermissionConfigFromRules(changedRules);
};
export const AgentsPage: React.FC = () => { export const AgentsPage: React.FC = () => {
const { selectedAgentName, getAgentByName, createAgent, updateAgent, agents, agentDraft, setAgentDraft } = useAgentsStore(); const { selectedAgentName, getAgentByName, createAgent, updateAgent, agents, agentDraft, setAgentDraft } = useAgentsStore();
@@ -324,12 +284,6 @@ export const AgentsPage: React.FC = () => {
currentRuleMap.get(buildRuleKey(permissionName, '*'))?.action currentRuleMap.get(buildRuleKey(permissionName, '*'))?.action
), [currentRuleMap]); ), [currentRuleMap]);
const getEffectiveWildcardAction = React.useCallback((permissionName: string): PermissionAction => {
if (permissionName === '*') {
return globalPermission;
}
return getWildcardOverride(permissionName) ?? globalPermission;
}, [getWildcardOverride, globalPermission]);
const getPatternRules = React.useCallback((permissionName: string): PermissionRule[] => ( const getPatternRules = React.useCallback((permissionName: string): PermissionRule[] => (
permissionRules permissionRules
@@ -348,12 +302,6 @@ export const AgentsPage: React.FC = () => {
return Array.from(names).sort((a, b) => a.localeCompare(b)); return Array.from(names).sort((a, b) => a.localeCompare(b));
}, [knownPermissionNames]); }, [knownPermissionNames]);
const getFallbackDefaultAction = React.useCallback((permissionName: string): PermissionAction => {
if (permissionName === 'doom_loop' || permissionName === 'external_directory') {
return 'ask';
}
return 'allow';
}, []);
const getPermissionSummary = React.useCallback((permissionName: string) => { const getPermissionSummary = React.useCallback((permissionName: string) => {
const defaultAction = permissionName === '*' const defaultAction = permissionName === '*'
@@ -65,7 +65,13 @@ export const SkillsCatalogPage: React.FC<SkillsCatalogPageProps> = ({ mode, onMo
selectedSourceId, selectedSourceId,
setSelectedSource, setSelectedSource,
loadCatalog, loadCatalog,
loadSource,
loadMoreClawdHub,
isLoadingCatalog, isLoadingCatalog,
isLoadingSource,
isLoadingMore,
loadedSourceIds,
clawdhubHasMoreBySource,
lastCatalogError, lastCatalogError,
} = useSkillsCatalogStore(); } = useSkillsCatalogStore();
@@ -79,6 +85,15 @@ export const SkillsCatalogPage: React.FC<SkillsCatalogPageProps> = ({ mode, onMo
void loadCatalog(); void loadCatalog();
}, [loadCatalog]); }, [loadCatalog]);
React.useEffect(() => {
if (!selectedSourceId) {
return;
}
if (!loadedSourceIds[selectedSourceId]) {
void loadSource(selectedSourceId);
}
}, [selectedSourceId, loadedSourceIds, loadSource]);
const items = React.useMemo(() => { const items = React.useMemo(() => {
if (!selectedSourceId) return []; if (!selectedSourceId) return [];
return itemsBySource[selectedSourceId] || []; return itemsBySource[selectedSourceId] || [];
@@ -98,6 +113,10 @@ export const SkillsCatalogPage: React.FC<SkillsCatalogPageProps> = ({ mode, onMo
const selectedSource = React.useMemo(() => sources.find((s) => s.id === selectedSourceId) || null, [sources, selectedSourceId]); const selectedSource = React.useMemo(() => sources.find((s) => s.id === selectedSourceId) || null, [sources, selectedSourceId]);
const isCustomSource = Boolean(selectedSourceId && selectedSourceId.startsWith('custom:')); const isCustomSource = Boolean(selectedSourceId && selectedSourceId.startsWith('custom:'));
const isClawdHubSource = selectedSource?.source === 'clawdhub:registry' || selectedSource?.sourceType === 'clawdhub';
const hasMoreClawdHub = Boolean(
selectedSourceId && (clawdhubHasMoreBySource[selectedSourceId] ?? true)
);
const removeSelectedCatalog = async () => { const removeSelectedCatalog = async () => {
if (!selectedSourceId || !isCustomSource) { if (!selectedSourceId || !isCustomSource) {
@@ -167,8 +186,14 @@ export const SkillsCatalogPage: React.FC<SkillsCatalogPageProps> = ({ mode, onMo
<Button <Button
type="button" type="button"
variant="outline" variant="outline"
onClick={() => void loadCatalog({ refresh: true })} onClick={() => {
disabled={isLoadingCatalog} if (selectedSourceId) {
void loadSource(selectedSourceId, { refresh: true });
} else {
void loadCatalog({ refresh: true });
}
}}
disabled={isLoadingCatalog || isLoadingSource}
className="gap-2" className="gap-2"
> >
<RiRefreshLine className="h-4 w-4" /> <RiRefreshLine className="h-4 w-4" />
@@ -219,13 +244,18 @@ export const SkillsCatalogPage: React.FC<SkillsCatalogPageProps> = ({ mode, onMo
</div> </div>
<div className="space-y-2"> <div className="space-y-2">
{filtered.length === 0 ? ( {filtered.length === 0 && !isLoadingSource ? (
<div className="py-10 text-center text-muted-foreground"> <div className="py-10 text-center text-muted-foreground">
<p className="typography-body">No skills found</p> <p className="typography-body">No skills found</p>
<p className="typography-meta mt-1 opacity-75">Try a different search or refresh the catalog</p> <p className="typography-meta mt-1 opacity-75">Try a different search or refresh the catalog</p>
</div> </div>
) : isLoadingSource ? (
<div className="py-10 text-center text-muted-foreground">
<p className="typography-body">Loading skills…</p>
</div>
) : ( ) : (
filtered.map((item) => { <>
{filtered.map((item) => {
const installed = item.installed?.isInstalled; const installed = item.installed?.isInstalled;
const installedScope = item.installed?.scope; const installedScope = item.installed?.scope;
@@ -293,7 +323,20 @@ export const SkillsCatalogPage: React.FC<SkillsCatalogPageProps> = ({ mode, onMo
</div> </div>
</div> </div>
); );
}) })}
{isClawdHubSource && hasMoreClawdHub ? (
<div className="flex justify-center pt-2">
<Button
type="button"
variant="outline"
onClick={() => void loadMoreClawdHub()}
disabled={isLoadingMore || isLoadingSource}
>
{isLoadingMore ? 'Loading…' : 'Load more'}
</Button>
</div>
) : null}
</>
)} )}
</div> </div>
+8
View File
@@ -828,6 +828,14 @@ export interface SkillsCatalogResponse {
ok: boolean; ok: boolean;
sources?: SkillsCatalogSource[]; sources?: SkillsCatalogSource[];
itemsBySource?: Record<SkillsCatalogSourceId, SkillsCatalogItem[]>; itemsBySource?: Record<SkillsCatalogSourceId, SkillsCatalogItem[]>;
pageInfoBySource?: Record<SkillsCatalogSourceId, { nextCursor?: string | null }>;
error?: { kind: string; message: string };
}
export interface SkillsCatalogSourceResponse {
ok: boolean;
items?: SkillsCatalogItem[];
nextCursor?: string | null;
error?: { kind: string; message: string }; error?: { kind: string; message: string };
} }
+202 -24
View File
@@ -10,11 +10,30 @@ import type {
SkillsInstallRequest, SkillsInstallRequest,
SkillsInstallResponse, SkillsInstallResponse,
SkillsInstallError, SkillsInstallError,
SkillsCatalogSourceResponse,
} from '@/lib/api/types'; } from '@/lib/api/types';
import { useSkillsStore } from '@/stores/useSkillsStore'; import { useSkillsStore } from '@/stores/useSkillsStore';
import { opencodeClient } from '@/lib/opencode/client'; import { opencodeClient } from '@/lib/opencode/client';
const FALLBACK_SOURCES: SkillsCatalogSource[] = [
{
id: 'anthropic',
label: 'Anthropic',
description: "Anthropic's public skills repository",
source: 'anthropics/skills',
defaultSubpath: 'skills',
sourceType: 'github',
},
{
id: 'clawdhub',
label: 'ClawdHub',
description: 'Community skill registry with vector search',
source: 'clawdhub:registry',
sourceType: 'clawdhub',
},
];
const getCurrentDirectory = (): string | null => { const getCurrentDirectory = (): string | null => {
const opencodeDirectory = opencodeClient.getDirectory(); const opencodeDirectory = opencodeClient.getDirectory();
if (typeof opencodeDirectory === 'string' && opencodeDirectory.trim().length > 0) { if (typeof opencodeDirectory === 'string' && opencodeDirectory.trim().length > 0) {
@@ -38,8 +57,13 @@ export interface SkillsCatalogState {
sources: SkillsCatalogSource[]; sources: SkillsCatalogSource[];
itemsBySource: Record<string, SkillsCatalogItem[]>; itemsBySource: Record<string, SkillsCatalogItem[]>;
selectedSourceId: string | null; selectedSourceId: string | null;
pageInfoBySource: Record<string, { nextCursor?: string | null }>;
loadedSourceIds: Record<string, boolean>;
clawdhubHasMoreBySource: Record<string, boolean>;
isLoadingCatalog: boolean; isLoadingCatalog: boolean;
isLoadingSource: boolean;
isLoadingMore: boolean;
isScanning: boolean; isScanning: boolean;
isInstalling: boolean; isInstalling: boolean;
@@ -52,6 +76,8 @@ export interface SkillsCatalogState {
setSelectedSource: (id: string | null) => void; setSelectedSource: (id: string | null) => void;
loadCatalog: (options?: { refresh?: boolean }) => Promise<boolean>; loadCatalog: (options?: { refresh?: boolean }) => Promise<boolean>;
loadSource: (sourceId: string, options?: { refresh?: boolean }) => Promise<boolean>;
loadMoreClawdHub: () => Promise<boolean>;
scanRepo: (request: SkillsRepoScanRequest) => Promise<SkillsRepoScanResponse>; scanRepo: (request: SkillsRepoScanRequest) => Promise<SkillsRepoScanResponse>;
installSkills: (request: SkillsInstallRequest) => Promise<SkillsInstallResponse>; installSkills: (request: SkillsInstallRequest) => Promise<SkillsInstallResponse>;
} }
@@ -59,11 +85,16 @@ export interface SkillsCatalogState {
export const useSkillsCatalogStore = create<SkillsCatalogState>()( export const useSkillsCatalogStore = create<SkillsCatalogState>()(
devtools( devtools(
(set, get) => ({ (set, get) => ({
sources: [], sources: FALLBACK_SOURCES,
itemsBySource: {}, itemsBySource: {},
selectedSourceId: null, selectedSourceId: FALLBACK_SOURCES[0]?.id ?? null,
pageInfoBySource: {},
loadedSourceIds: {},
clawdhubHasMoreBySource: {},
isLoadingCatalog: false, isLoadingCatalog: false,
isLoadingSource: false,
isLoadingMore: false,
isScanning: false, isScanning: false,
isInstalling: false, isInstalling: false,
@@ -81,55 +112,64 @@ export const useSkillsCatalogStore = create<SkillsCatalogState>()(
const previous = { const previous = {
sources: get().sources, sources: get().sources,
itemsBySource: get().itemsBySource, itemsBySource: get().itemsBySource,
pageInfoBySource: get().pageInfoBySource,
loadedSourceIds: get().loadedSourceIds,
clawdhubHasMoreBySource: get().clawdhubHasMoreBySource,
}; };
let lastError: SkillsCatalogResponse['error'] | null = null; let lastError: SkillsCatalogResponse['error'] | null = null;
try { try {
for (let attempt = 0; attempt < 3; attempt++) { const refresh = options?.refresh ? '?refresh=true' : '';
try { const controller = new AbortController();
const currentDirectory = getCurrentDirectory(); const timeoutId = window.setTimeout(() => controller.abort(), 3000);
const refresh = options?.refresh ? '&refresh=true' : '';
const queryParams = currentDirectory
? `?directory=${encodeURIComponent(currentDirectory)}${refresh}`
: refresh
? `?refresh=true`
: '';
const response = await fetch(`/api/config/skills/catalog${queryParams}`, { try {
const response = await fetch(`/api/config/skills/catalog${refresh}`, {
method: 'GET', method: 'GET',
headers: { Accept: 'application/json' }, headers: { Accept: 'application/json' },
signal: controller.signal,
}); });
const payload = (await response.json().catch(() => null)) as SkillsCatalogResponse | null; const payload = (await response.json().catch(() => null)) as SkillsCatalogResponse | null;
if (!response.ok || !payload?.ok) { if (!response.ok || !payload?.ok) {
lastError = payload?.error || { kind: 'unknown', message: `Failed to load catalog (${response.status})` }; lastError = payload?.error || { kind: 'unknown', message: `Failed to load catalog (${response.status})` };
const waitMs = 200 * (attempt + 1); throw new Error(lastError.message);
await new Promise((resolve) => setTimeout(resolve, waitMs));
continue;
} }
const sources = payload.sources || []; const sources = (payload.sources && payload.sources.length > 0) ? payload.sources : previous.sources;
const itemsBySource = payload.itemsBySource || {}; const itemsBySource = options?.refresh ? {} : (get().itemsBySource || {});
const pageInfoBySource = options?.refresh ? {} : (get().pageInfoBySource || {});
const loadedSourceIds = options?.refresh ? {} : (get().loadedSourceIds || {});
const clawdhubHasMoreBySource = options?.refresh ? {} : (get().clawdhubHasMoreBySource || {});
const currentSelected = get().selectedSourceId; const currentSelected = get().selectedSourceId;
const selectedSourceId = const selectedSourceId =
(currentSelected && sources.some((s) => s.id === currentSelected)) (currentSelected && sources.some((s) => s.id === currentSelected))
? currentSelected ? currentSelected
: (sources[0]?.id ?? null); : (sources[0]?.id ?? null);
set({ sources, itemsBySource, selectedSourceId }); set({
return true; sources,
itemsBySource,
pageInfoBySource,
loadedSourceIds,
clawdhubHasMoreBySource,
selectedSourceId,
});
return true;
} finally {
window.clearTimeout(timeoutId);
}
} catch (error) { } catch (error) {
lastError = { kind: 'unknown', message: error instanceof Error ? error.message : String(error) }; lastError = lastError || { kind: 'unknown', message: error instanceof Error ? error.message : String(error) };
const waitMs = 200 * (attempt + 1);
await new Promise((resolve) => setTimeout(resolve, waitMs));
}
}
set({ set({
sources: previous.sources, sources: previous.sources,
itemsBySource: previous.itemsBySource, itemsBySource: previous.itemsBySource,
pageInfoBySource: previous.pageInfoBySource,
loadedSourceIds: previous.loadedSourceIds,
clawdhubHasMoreBySource: previous.clawdhubHasMoreBySource,
lastCatalogError: lastError || { kind: 'unknown', message: 'Failed to load catalog' }, lastCatalogError: lastError || { kind: 'unknown', message: 'Failed to load catalog' },
}); });
@@ -139,6 +179,144 @@ export const useSkillsCatalogStore = create<SkillsCatalogState>()(
} }
}, },
loadSource: async (sourceId, options) => {
if (!sourceId) {
return false;
}
set({ isLoadingSource: true, lastCatalogError: null });
try {
const currentDirectory = getCurrentDirectory();
const refresh = options?.refresh ? '&refresh=true' : '';
const queryParams = currentDirectory
? `?directory=${encodeURIComponent(currentDirectory)}&sourceId=${encodeURIComponent(sourceId)}${refresh}`
: `?sourceId=${encodeURIComponent(sourceId)}${refresh}`;
const response = await fetch(`/api/config/skills/catalog/source${queryParams}`, {
method: 'GET',
headers: { Accept: 'application/json' },
});
const payload = (await response.json().catch(() => null)) as SkillsCatalogSourceResponse | null;
const hasItems = Array.isArray((payload as SkillsCatalogSourceResponse | null)?.items);
if (!response.ok || (!payload?.ok && !hasItems)) {
const fallback = await fetch(`/api/config/skills/catalog${queryParams}`, {
method: 'GET',
headers: { Accept: 'application/json' },
});
const fallbackPayload = (await fallback.json().catch(() => null)) as SkillsCatalogResponse | null;
const fallbackItems = fallbackPayload?.itemsBySource?.[sourceId];
if (fallback.ok && fallbackPayload?.ok && Array.isArray(fallbackItems)) {
set((state) => ({
itemsBySource: { ...state.itemsBySource, [sourceId]: fallbackItems },
pageInfoBySource: { ...state.pageInfoBySource, [sourceId]: { nextCursor: null } },
loadedSourceIds: { ...state.loadedSourceIds, [sourceId]: true },
clawdhubHasMoreBySource: { ...state.clawdhubHasMoreBySource, [sourceId]: false },
}));
return true;
}
set({
lastCatalogError: payload?.error || { kind: 'unknown', message: `Failed to load source (${response.status})` },
});
return false;
}
const items = payload?.items || [];
const nextCursor = payload?.nextCursor ?? null;
set((state) => ({
itemsBySource: { ...state.itemsBySource, [sourceId]: items },
pageInfoBySource: { ...state.pageInfoBySource, [sourceId]: { nextCursor } },
loadedSourceIds: { ...state.loadedSourceIds, [sourceId]: true },
clawdhubHasMoreBySource: {
...state.clawdhubHasMoreBySource,
[sourceId]: items.length > 0,
},
}));
return true;
} catch (error) {
set({
lastCatalogError: { kind: 'unknown', message: error instanceof Error ? error.message : String(error) },
});
return false;
} finally {
set({ isLoadingSource: false });
}
},
loadMoreClawdHub: async () => {
const selectedSourceId = get().selectedSourceId;
if (!selectedSourceId) {
return false;
}
const pageInfo = get().pageInfoBySource[selectedSourceId];
const cursor = pageInfo?.nextCursor || null;
set({ isLoadingMore: true });
try {
const currentDirectory = getCurrentDirectory();
const parts = [`sourceId=${encodeURIComponent(selectedSourceId)}`];
if (currentDirectory) {
parts.push(`directory=${encodeURIComponent(currentDirectory)}`);
}
if (cursor) {
parts.push(`cursor=${encodeURIComponent(cursor)}`);
}
const queryParams = `?${parts.join('&')}`;
const response = await fetch(`/api/config/skills/catalog/source${queryParams}`, {
method: 'GET',
headers: { Accept: 'application/json' },
});
const payload = (await response.json().catch(() => null)) as SkillsCatalogSourceResponse | null;
if (!response.ok || !payload?.ok) {
return false;
}
const nextCursor = payload.nextCursor ?? null;
const currentItems = get().itemsBySource[selectedSourceId] || [];
const items = payload.items || [];
const merged = new Map(currentItems.map((item) => [`${item.sourceId}:${item.skillDir}`, item]));
let newCount = 0;
for (const item of items) {
const key = `${item.sourceId}:${item.skillDir}`;
if (!merged.has(key)) {
newCount += 1;
}
merged.set(key, item);
}
const noMore = items.length === 0 || newCount === 0;
set((state) => ({
itemsBySource: {
...state.itemsBySource,
[selectedSourceId]: Array.from(merged.values()),
},
pageInfoBySource: {
...state.pageInfoBySource,
[selectedSourceId]: { nextCursor },
},
clawdhubHasMoreBySource: {
...state.clawdhubHasMoreBySource,
[selectedSourceId]: !noMore,
},
}));
return true;
} catch {
return false;
} finally {
set({ isLoadingMore: false });
}
},
scanRepo: async (request) => { scanRepo: async (request) => {
set({ isScanning: true, lastScanError: null, scanResults: null }); set({ isScanning: true, lastScanError: null, scanResults: null });
try { try {
+43 -6
View File
@@ -94,6 +94,7 @@ export const CURATED_SOURCES: CuratedSource[] = [
// ============== ClawdHub API ============== // ============== ClawdHub API ==============
const CLAWDHUB_API_BASE = 'https://clawdhub.com/api/v1'; const CLAWDHUB_API_BASE = 'https://clawdhub.com/api/v1';
const CLAWDHUB_PAGE_LIMIT = 25;
const CLAWDHUB_RATE_LIMIT_MS = 100; const CLAWDHUB_RATE_LIMIT_MS = 100;
let clawdhubLastRequest = 0; let clawdhubLastRequest = 0;
@@ -102,6 +103,10 @@ function isClawdHubSource(source: string): boolean {
} }
async function clawdhubFetch(url: string, options?: RequestInit): Promise<Response> { async function clawdhubFetch(url: string, options?: RequestInit): Promise<Response> {
const maxAttempts = 10;
let lastResponse: Response | null = null;
for (let attempt = 0; attempt < maxAttempts; attempt++) {
const now = Date.now(); const now = Date.now();
const elapsed = now - clawdhubLastRequest; const elapsed = now - clawdhubLastRequest;
if (elapsed < CLAWDHUB_RATE_LIMIT_MS) { if (elapsed < CLAWDHUB_RATE_LIMIT_MS) {
@@ -109,7 +114,7 @@ async function clawdhubFetch(url: string, options?: RequestInit): Promise<Respon
} }
clawdhubLastRequest = Date.now(); clawdhubLastRequest = Date.now();
return fetch(url, { const response = await fetch(url, {
...options, ...options,
headers: { headers: {
Accept: 'application/json', Accept: 'application/json',
@@ -117,6 +122,21 @@ async function clawdhubFetch(url: string, options?: RequestInit): Promise<Respon
...options?.headers, ...options?.headers,
}, },
}); });
lastResponse = response;
if (response.status === 429 || response.status >= 500) {
if (attempt < maxAttempts - 1) {
const waitMs = 50 * (attempt + 1);
await new Promise((resolve) => setTimeout(resolve, waitMs));
continue;
}
}
return response;
}
return lastResponse as Response;
} }
type ClawdHubSkillListItem = { type ClawdHubSkillListItem = {
@@ -142,15 +162,24 @@ async function scanClawdHub(): Promise<SkillsRepoScanResult> {
for (let page = 0; page < maxPages; page++) { for (let page = 0; page < maxPages; page++) {
const url = cursor const url = cursor
? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}` ? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}&limit=${CLAWDHUB_PAGE_LIMIT}`
: `${CLAWDHUB_API_BASE}/skills`; : `${CLAWDHUB_API_BASE}/skills?limit=${CLAWDHUB_PAGE_LIMIT}`;
let data: ClawdHubSkillsResponse;
try {
const response = await clawdhubFetch(url); const response = await clawdhubFetch(url);
if (!response.ok) { if (!response.ok) {
throw new Error(`ClawdHub API error: ${response.status}`); throw new Error(`ClawdHub API error: ${response.status}`);
} }
const data = (await response.json()) as ClawdHubSkillsResponse; data = (await response.json()) as ClawdHubSkillsResponse;
} catch (error) {
if (page > 0 && allItems.length > 0) {
break;
}
throw error;
}
for (const item of data.items || []) { for (const item of data.items || []) {
const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0'; const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0';
@@ -278,9 +307,17 @@ export async function installSkillsFromClawdHub(options: {
if (version === 'latest') { if (version === 'latest') {
try { try {
const info = await fetchClawdHubSkillInfo(slug); const info = await fetchClawdHubSkillInfo(slug);
version = info.skill?.tags?.latest || info.latestVersion?.version || version; const latest = info.skill?.tags?.latest || info.latestVersion?.version || null;
if (latest) {
version = latest;
}
} catch { } catch {
// Fall back to 'latest' // ignore
}
if (version === 'latest') {
skipped.push({ skillName: slug, reason: 'Unable to resolve latest version' });
continue;
} }
} }
+80 -34
View File
@@ -533,6 +533,25 @@ const resolveProjectDirectory = async (req) => {
return { directory: validated.directory, error: null }; return { directory: validated.directory, error: null };
}; };
const resolveOptionalProjectDirectory = async (req) => {
const headerDirectory = typeof req.get === 'function' ? req.get('x-opencode-directory') : null;
const queryDirectory = Array.isArray(req.query?.directory)
? req.query.directory[0]
: req.query?.directory;
const requested = headerDirectory || queryDirectory || null;
if (!requested) {
return { directory: null, error: null };
}
const validated = await validateDirectoryPath(requested);
if (!validated.ok) {
return { directory: null, error: validated.error };
}
return { directory: validated.directory, error: null };
};
const sanitizeTypographySizesPartial = (input) => { const sanitizeTypographySizesPartial = (input) => {
if (!input || typeof input !== 'object') { if (!input || typeof input !== 'object') {
return undefined; return undefined;
@@ -2416,6 +2435,7 @@ function setupProxy(app) {
req.path.startsWith('/push') || req.path.startsWith('/push') ||
req.path.startsWith('/config/agents') || req.path.startsWith('/config/agents') ||
req.path.startsWith('/config/settings') || req.path.startsWith('/config/settings') ||
req.path.startsWith('/config/skills') ||
req.path === '/config/reload' || req.path === '/config/reload' ||
req.path === '/health' req.path === '/health'
) { ) {
@@ -2448,6 +2468,7 @@ function setupProxy(app) {
req.path.startsWith('/themes/custom') || req.path.startsWith('/themes/custom') ||
req.path.startsWith('/config/agents') || req.path.startsWith('/config/agents') ||
req.path.startsWith('/config/settings') || req.path.startsWith('/config/settings') ||
req.path.startsWith('/config/skills') ||
req.path === '/health' req.path === '/health'
) { ) {
return next(); return next();
@@ -3508,7 +3529,7 @@ async function main(options = {}) {
const { parseSkillRepoSource } = await import('./lib/skills-catalog/source.js'); const { parseSkillRepoSource } = await import('./lib/skills-catalog/source.js');
const { scanSkillsRepository } = await import('./lib/skills-catalog/scan.js'); const { scanSkillsRepository } = await import('./lib/skills-catalog/scan.js');
const { installSkillsFromRepository } = await import('./lib/skills-catalog/install.js'); const { installSkillsFromRepository } = await import('./lib/skills-catalog/install.js');
const { scanClawdHub, installSkillsFromClawdHub, isClawdHubSource } = await import('./lib/skills-catalog/clawdhub/index.js'); const { scanClawdHubPage, installSkillsFromClawdHub, isClawdHubSource } = await import('./lib/skills-catalog/clawdhub/index.js');
const { getProfiles, getProfile } = await import('./lib/git-identity-storage.js'); const { getProfiles, getProfile } = await import('./lib/git-identity-storage.js');
const listGitIdentitiesForResponse = () => { const listGitIdentitiesForResponse = () => {
@@ -3538,11 +3559,10 @@ async function main(options = {}) {
app.get('/api/config/skills/catalog', async (req, res) => { app.get('/api/config/skills/catalog', async (req, res) => {
try { try {
const { directory, error } = await resolveProjectDirectory(req); const { error } = await resolveOptionalProjectDirectory(req);
if (!directory) { if (error) {
return res.status(400).json({ error }); return res.status(400).json({ error });
} }
const refresh = String(req.query.refresh || '').toLowerCase() === 'true';
const curatedSources = getCuratedSkillsSources(); const curatedSources = getCuratedSkillsSources();
const settings = await readSettingsFromDisk(); const settings = await readSettingsFromDisk();
@@ -3558,29 +3578,60 @@ async function main(options = {}) {
})); }));
const sources = [...curatedSources, ...customSources]; const sources = [...curatedSources, ...customSources];
const sourcesForUi = sources.map(({ gitIdentityId, ...rest }) => rest);
const discovered = discoverSkills(directory); res.json({ ok: true, sources: sourcesForUi, itemsBySource: {}, pageInfoBySource: {} });
} catch (error) {
console.error('Failed to load skills catalog:', error);
res.status(500).json({ ok: false, error: { kind: 'unknown', message: error.message || 'Failed to load catalog' } });
}
});
app.get('/api/config/skills/catalog/source', async (req, res) => {
try {
const { directory, error } = await resolveOptionalProjectDirectory(req);
if (error) {
return res.status(400).json({ ok: false, error: { kind: 'invalidSource', message: error } });
}
const sourceId = typeof req.query.sourceId === 'string' ? req.query.sourceId : null;
if (!sourceId) {
return res.status(400).json({ ok: false, error: { kind: 'invalidSource', message: 'Missing sourceId' } });
}
const refresh = String(req.query.refresh || '').toLowerCase() === 'true';
const cursor = typeof req.query.cursor === 'string' ? req.query.cursor : null;
const curatedSources = getCuratedSkillsSources();
const settings = await readSettingsFromDisk();
const customSourcesRaw = sanitizeSkillCatalogs(settings.skillCatalogs) || [];
const customSources = customSourcesRaw.map((entry) => ({
id: entry.id,
label: entry.label,
description: entry.source,
source: entry.source,
defaultSubpath: entry.subpath,
gitIdentityId: entry.gitIdentityId,
}));
const sources = [...curatedSources, ...customSources];
const src = sources.find((entry) => entry.id === sourceId);
if (!src) {
return res.status(404).json({ ok: false, error: { kind: 'invalidSource', message: 'Unknown source' } });
}
const discovered = directory ? discoverSkills(directory) : [];
const installedByName = new Map(discovered.map((s) => [s.name, s])); const installedByName = new Map(discovered.map((s) => [s.name, s]));
const itemsBySource = {};
for (const src of sources) {
// Handle ClawdHub sources separately (API-based, not git-based)
if (src.sourceType === 'clawdhub' || isClawdHubSource(src.source)) { if (src.sourceType === 'clawdhub' || isClawdHubSource(src.source)) {
const cacheKey = 'clawdhub:registry'; const scanned = await scanClawdHubPage({ cursor: cursor || null });
let scanResult = !refresh ? getCachedScan(cacheKey) : null;
if (!scanResult) {
const scanned = await scanClawdHub();
if (!scanned.ok) { if (!scanned.ok) {
itemsBySource[src.id] = []; return res.status(500).json({ ok: false, error: scanned.error });
continue;
}
scanResult = scanned;
setCachedScan(cacheKey, scanResult);
} }
const items = (scanResult.items || []).map((item) => { const items = (scanned.items || []).map((item) => {
const installed = installedByName.get(item.skillName); const installed = installedByName.get(item.skillName);
return { return {
...item, ...item,
@@ -3591,15 +3642,12 @@ async function main(options = {}) {
}; };
}); });
itemsBySource[src.id] = items; return res.json({ ok: true, items, nextCursor: scanned.nextCursor || null });
continue;
} }
// Handle GitHub sources (git clone based)
const parsed = parseSkillRepoSource(src.source); const parsed = parseSkillRepoSource(src.source);
if (!parsed.ok) { if (!parsed.ok) {
itemsBySource[src.id] = []; return res.status(400).json({ ok: false, error: parsed.error });
continue;
} }
const effectiveSubpath = src.defaultSubpath || parsed.effectiveSubpath || null; const effectiveSubpath = src.defaultSubpath || parsed.effectiveSubpath || null;
@@ -3619,8 +3667,7 @@ async function main(options = {}) {
}); });
if (!scanned.ok) { if (!scanned.ok) {
itemsBySource[src.id] = []; return res.status(500).json({ ok: false, error: scanned.error });
continue;
} }
scanResult = scanned; scanResult = scanned;
@@ -3639,14 +3686,13 @@ async function main(options = {}) {
}; };
}); });
itemsBySource[src.id] = items; return res.json({ ok: true, items });
}
const sourcesForUi = sources.map(({ gitIdentityId, ...rest }) => rest);
res.json({ ok: true, sources: sourcesForUi, itemsBySource });
} catch (error) { } catch (error) {
console.error('Failed to load skills catalog:', error); console.error('Failed to load catalog source:', error);
res.status(500).json({ ok: false, error: { kind: 'unknown', message: error.message || 'Failed to load catalog' } }); return res.status(500).json({
ok: false,
error: { kind: 'unknown', message: error.message || 'Failed to load catalog source' },
});
} }
}); });
@@ -6,12 +6,18 @@
*/ */
const CLAWDHUB_API_BASE = 'https://clawdhub.com/api/v1'; const CLAWDHUB_API_BASE = 'https://clawdhub.com/api/v1';
const CLAWDHUB_PAGE_LIMIT = 25;
// Rate limiting: ClawdHub allows 120 requests/minute // Rate limiting: ClawdHub allows 120 requests/minute
const RATE_LIMIT_DELAY_MS = 100; const RATE_LIMIT_DELAY_MS = 100;
let lastRequestTime = 0; let lastRequestTime = 0;
async function rateLimitedFetch(url, options = {}) { async function rateLimitedFetch(url, options = {}) {
const maxAttempts = 10;
let lastResponse = null;
for (let attempt = 0; attempt < maxAttempts; attempt++) {
const now = Date.now(); const now = Date.now();
const elapsed = now - lastRequestTime; const elapsed = now - lastRequestTime;
if (elapsed < RATE_LIMIT_DELAY_MS) { if (elapsed < RATE_LIMIT_DELAY_MS) {
@@ -28,7 +34,20 @@ async function rateLimitedFetch(url, options = {}) {
}, },
}); });
lastResponse = response;
if (response.status === 429 || response.status >= 500) {
if (attempt < maxAttempts - 1) {
const waitMs = 50 * (attempt + 1);
await new Promise((resolve) => setTimeout(resolve, waitMs));
continue;
}
}
return response; return response;
}
return lastResponse;
} }
/** /**
@@ -39,8 +58,8 @@ async function rateLimitedFetch(url, options = {}) {
*/ */
export async function fetchClawdHubSkills({ cursor } = {}) { export async function fetchClawdHubSkills({ cursor } = {}) {
const url = cursor const url = cursor
? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}` ? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}&limit=${CLAWDHUB_PAGE_LIMIT}`
: `${CLAWDHUB_API_BASE}/skills`; : `${CLAWDHUB_API_BASE}/skills?limit=${CLAWDHUB_PAGE_LIMIT}`;
const response = await rateLimitedFetch(url); const response = await rateLimitedFetch(url);
@@ -50,9 +69,16 @@ export async function fetchClawdHubSkills({ cursor } = {}) {
} }
const data = await response.json(); const data = await response.json();
const nextCursor =
(typeof data.nextCursor === 'string' && data.nextCursor) ||
(typeof data.next_cursor === 'string' && data.next_cursor) ||
(typeof data.next === 'string' && data.next) ||
(typeof data.cursor === 'string' && data.cursor) ||
null;
return { return {
items: data.items || [], items: data.items || [],
nextCursor: data.nextCursor || null, nextCursor,
}; };
} }
@@ -95,7 +121,10 @@ export async function fetchClawdHubSkillVersion(slug, version = 'latest') {
* @returns {Promise<ArrayBuffer>} - ZIP file contents * @returns {Promise<ArrayBuffer>} - ZIP file contents
*/ */
export async function downloadClawdHubSkill(slug, version) { export async function downloadClawdHubSkill(slug, version) {
const url = `${CLAWDHUB_API_BASE}/download?slug=${encodeURIComponent(slug)}&version=${encodeURIComponent(version)}`; const versionParam = typeof version === 'string' && version !== 'latest'
? `&version=${encodeURIComponent(version)}`
: '&tag=latest';
const url = `${CLAWDHUB_API_BASE}/download?slug=${encodeURIComponent(slug)}${versionParam}`;
const response = await rateLimitedFetch(url, { const response = await rateLimitedFetch(url, {
headers: { headers: {
@@ -5,7 +5,7 @@
* https://clawdhub.com * https://clawdhub.com
*/ */
export { scanClawdHub } from './scan.js'; export { scanClawdHub, scanClawdHubPage } from './scan.js';
export { installSkillsFromClawdHub } from './install.js'; export { installSkillsFromClawdHub } from './install.js';
export { export {
fetchClawdHubSkills, fetchClawdHubSkills,
@@ -150,10 +150,17 @@ export async function installSkillsFromClawdHub({
if (resolvedVersion === 'latest') { if (resolvedVersion === 'latest') {
try { try {
const info = await fetchClawdHubSkillInfo(plan.slug); const info = await fetchClawdHubSkillInfo(plan.slug);
resolvedVersion = info.skill?.tags?.latest || info.latestVersion?.version || plan.version; const latest = info.skill?.tags?.latest || info.latestVersion?.version || null;
if (latest) {
resolvedVersion = latest;
}
} catch { } catch {
// Fall back to 'latest' tag if info fetch fails // ignore
resolvedVersion = 'latest'; }
if (resolvedVersion === 'latest') {
skipped.push({ skillName: plan.slug, reason: 'Unable to resolve latest version' });
continue;
} }
} }
@@ -8,23 +8,12 @@
import { fetchClawdHubSkills } from './api.js'; import { fetchClawdHubSkills } from './api.js';
const MAX_PAGES = 20; // Safety limit to prevent infinite loops const MAX_PAGES = 20; // Safety limit to prevent infinite loops
const CLAWDHUB_PAGE_LIMIT = 25;
/** const mapClawdHubItem = (item) => {
* Scan ClawdHub registry for all available skills
* @returns {Promise<{ ok: boolean, items?: Array, error?: Object }>}
*/
export async function scanClawdHub() {
try {
const allItems = [];
let cursor = null;
for (let page = 0; page < MAX_PAGES; page++) {
const { items, nextCursor } = await fetchClawdHubSkills({ cursor });
for (const item of items) {
const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0'; const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0';
allItems.push({ return {
sourceId: 'clawdhub', sourceId: 'clawdhub',
repoSource: 'clawdhub:registry', repoSource: 'clawdhub:registry',
repoSubpath: null, repoSubpath: null,
@@ -47,7 +36,36 @@ export async function scanClawdHub() {
createdAt: item.createdAt, createdAt: item.createdAt,
updatedAt: item.updatedAt, updatedAt: item.updatedAt,
}, },
}); };
};
/**
* Scan ClawdHub registry for all available skills
* @returns {Promise<{ ok: boolean, items?: Array, error?: Object }>}
*/
export async function scanClawdHub() {
try {
const allItems = [];
let cursor = null;
for (let page = 0; page < MAX_PAGES; page++) {
let items = [];
let nextCursor = null;
try {
const pageResult = await fetchClawdHubSkills({ cursor });
items = pageResult.items || [];
nextCursor = pageResult.nextCursor || null;
} catch (error) {
if (page > 0 && allItems.length > 0) {
console.warn('ClawdHub pagination failed; returning partial results.');
break;
}
throw error;
}
for (const item of items) {
allItems.push(mapClawdHubItem(item));
} }
if (!nextCursor) { if (!nextCursor) {
@@ -71,3 +89,25 @@ export async function scanClawdHub() {
}; };
} }
} }
/**
* Scan a single ClawdHub page (cursor-based)
* @returns {Promise<{ ok: boolean, items?: Array, nextCursor?: string | null, error?: Object }>}
*/
export async function scanClawdHubPage({ cursor } = {}) {
try {
const { items, nextCursor } = await fetchClawdHubSkills({ cursor });
const mapped = (items || []).map(mapClawdHubItem).slice(0, CLAWDHUB_PAGE_LIMIT);
mapped.sort((a, b) => (b.clawdhub?.downloads || 0) - (a.clawdhub?.downloads || 0));
return { ok: true, items: mapped, nextCursor: nextCursor || null };
} catch (error) {
console.error('ClawdHub page scan error:', error);
return {
ok: false,
error: {
kind: 'networkError',
message: error instanceof Error ? error.message : 'Failed to fetch skills from ClawdHub',
},
};
}
}