feat: enhance ClawdHub integration with paging and retry

Add paging support when loading ClawdHub skills
Retry on API errors and throttle for ClawdHub requests
Load source content on source change and show loading indicators
This commit is contained in:
Bohdan Triapitsyn
2026-01-26 19:50:11 +02:00
parent eec170771c
commit 25525d5b1b
11 changed files with 699 additions and 319 deletions
@@ -721,6 +721,7 @@ fn cache_key(normalized_repo: &str, subpath: Option<&str>, identity_id: Option<&
// ============== ClawdHub API ==============
const CLAWDHUB_API_BASE: &str = "https://clawdhub.com/api/v1";
const CLAWDHUB_PAGE_LIMIT: usize = 25;
fn is_clawdhub_source(source: &str) -> bool {
source.starts_with("clawdhub:")
@@ -789,23 +790,56 @@ async fn scan_clawdhub() -> Result<Vec<SkillsCatalogItem>> {
let mut cursor: Option<String> = None;
let max_pages = 20;
for _ in 0..max_pages {
for page in 0..max_pages {
let url = match &cursor {
Some(c) => format!(
"{}{}?cursor={}",
"{}{}?cursor={}&limit={}",
CLAWDHUB_API_BASE,
"/skills",
urlencoding::encode(c)
urlencoding::encode(c),
CLAWDHUB_PAGE_LIMIT
),
None => format!("{}/skills", CLAWDHUB_API_BASE),
None => format!("{}/skills?limit={}", CLAWDHUB_API_BASE, CLAWDHUB_PAGE_LIMIT),
};
let response = client.get(&url).send().await?;
if !response.status().is_success() {
return Err(anyhow!("ClawdHub API error: {}", response.status()));
let mut response: Option<reqwest::Response> = None;
let max_attempts = 10;
for attempt in 0..max_attempts {
let resp = client.get(&url).send().await?;
if resp.status().is_success() {
response = Some(resp);
break;
}
let should_retry = (resp.status() == reqwest::StatusCode::TOO_MANY_REQUESTS
|| resp.status().is_server_error())
&& attempt + 1 < max_attempts;
if should_retry {
tokio::time::sleep(Duration::from_millis(50 * (attempt + 1) as u64)).await;
continue;
}
if page > 0 && !all_items.is_empty() {
break;
}
return Err(anyhow!("ClawdHub API error: {}", resp.status()));
}
let data: ClawdHubSkillsResponse = response.json().await?;
let Some(response) = response else {
break;
};
let data: ClawdHubSkillsResponse = match response.json().await {
Ok(parsed) => parsed,
Err(err) => {
if page > 0 && !all_items.is_empty() {
break;
}
return Err(err.into());
}
};
for item in data.items {
let latest_version = item
@@ -1510,12 +1544,22 @@ async fn install_skills_from_clawdhub(
// Resolve 'latest' version
if version == "latest" {
if let Ok(info) = fetch_clawdhub_skill_info(slug).await {
version = info
if let Some(latest) = info
.skill
.and_then(|s| s.tags)
.and_then(|t| t.latest)
.or_else(|| info.latest_version.and_then(|v| v.version))
.unwrap_or_else(|| "latest".to_string());
{
version = latest;
}
}
if version == "latest" {
skipped.push(SkippedSkill {
skill_name: slug.to_string(),
reason: "Unable to resolve latest version".to_string(),
});
continue;
}
}
@@ -147,29 +147,6 @@ const permissionConfigToRuleset = (value: unknown): PermissionRule[] => {
return rules;
};
const buildPermissionConfigFromRules = (ruleset: PermissionRule[]): AgentConfig['permission'] | undefined => {
const normalized = normalizeRuleset(ruleset);
if (normalized.length === 0) {
return undefined;
}
const grouped: Record<string, Record<string, PermissionAction>> = {};
for (const rule of normalized) {
(grouped[rule.permission] ||= {})[rule.pattern] = rule.action;
}
const result: Record<string, PermissionConfigValue> = {};
for (const [permissionName, patterns] of Object.entries(grouped)) {
if (Object.keys(patterns).length === 1 && patterns['*']) {
result[permissionName] = patterns['*'];
} else {
result[permissionName] = patterns;
}
}
return result as AgentConfig['permission'];
};
const buildPermissionConfigWithGlobal = (
globalAction: PermissionAction,
ruleset: PermissionRule[],
@@ -201,23 +178,6 @@ const buildPermissionConfigWithGlobal = (
return result as AgentConfig['permission'];
};
const buildPermissionDiffConfig = (
baselineRules: PermissionRule[],
currentRules: PermissionRule[],
): AgentConfig['permission'] | undefined => {
const baselineMap = buildRuleMap(baselineRules);
const currentMap = buildRuleMap(currentRules);
const changedRules: PermissionRule[] = [];
for (const [key, rule] of currentMap.entries()) {
const baselineRule = baselineMap.get(key);
if (!baselineRule || baselineRule.action !== rule.action) {
changedRules.push(rule);
}
}
return buildPermissionConfigFromRules(changedRules);
};
export const AgentsPage: React.FC = () => {
const { selectedAgentName, getAgentByName, createAgent, updateAgent, agents, agentDraft, setAgentDraft } = useAgentsStore();
@@ -324,12 +284,6 @@ export const AgentsPage: React.FC = () => {
currentRuleMap.get(buildRuleKey(permissionName, '*'))?.action
), [currentRuleMap]);
const getEffectiveWildcardAction = React.useCallback((permissionName: string): PermissionAction => {
if (permissionName === '*') {
return globalPermission;
}
return getWildcardOverride(permissionName) ?? globalPermission;
}, [getWildcardOverride, globalPermission]);
const getPatternRules = React.useCallback((permissionName: string): PermissionRule[] => (
permissionRules
@@ -348,12 +302,6 @@ export const AgentsPage: React.FC = () => {
return Array.from(names).sort((a, b) => a.localeCompare(b));
}, [knownPermissionNames]);
const getFallbackDefaultAction = React.useCallback((permissionName: string): PermissionAction => {
if (permissionName === 'doom_loop' || permissionName === 'external_directory') {
return 'ask';
}
return 'allow';
}, []);
const getPermissionSummary = React.useCallback((permissionName: string) => {
const defaultAction = permissionName === '*'
@@ -65,7 +65,13 @@ export const SkillsCatalogPage: React.FC<SkillsCatalogPageProps> = ({ mode, onMo
selectedSourceId,
setSelectedSource,
loadCatalog,
loadSource,
loadMoreClawdHub,
isLoadingCatalog,
isLoadingSource,
isLoadingMore,
loadedSourceIds,
clawdhubHasMoreBySource,
lastCatalogError,
} = useSkillsCatalogStore();
@@ -79,6 +85,15 @@ export const SkillsCatalogPage: React.FC<SkillsCatalogPageProps> = ({ mode, onMo
void loadCatalog();
}, [loadCatalog]);
React.useEffect(() => {
if (!selectedSourceId) {
return;
}
if (!loadedSourceIds[selectedSourceId]) {
void loadSource(selectedSourceId);
}
}, [selectedSourceId, loadedSourceIds, loadSource]);
const items = React.useMemo(() => {
if (!selectedSourceId) return [];
return itemsBySource[selectedSourceId] || [];
@@ -98,6 +113,10 @@ export const SkillsCatalogPage: React.FC<SkillsCatalogPageProps> = ({ mode, onMo
const selectedSource = React.useMemo(() => sources.find((s) => s.id === selectedSourceId) || null, [sources, selectedSourceId]);
const isCustomSource = Boolean(selectedSourceId && selectedSourceId.startsWith('custom:'));
const isClawdHubSource = selectedSource?.source === 'clawdhub:registry' || selectedSource?.sourceType === 'clawdhub';
const hasMoreClawdHub = Boolean(
selectedSourceId && (clawdhubHasMoreBySource[selectedSourceId] ?? true)
);
const removeSelectedCatalog = async () => {
if (!selectedSourceId || !isCustomSource) {
@@ -167,8 +186,14 @@ export const SkillsCatalogPage: React.FC<SkillsCatalogPageProps> = ({ mode, onMo
<Button
type="button"
variant="outline"
onClick={() => void loadCatalog({ refresh: true })}
disabled={isLoadingCatalog}
onClick={() => {
if (selectedSourceId) {
void loadSource(selectedSourceId, { refresh: true });
} else {
void loadCatalog({ refresh: true });
}
}}
disabled={isLoadingCatalog || isLoadingSource}
className="gap-2"
>
<RiRefreshLine className="h-4 w-4" />
@@ -219,81 +244,99 @@ export const SkillsCatalogPage: React.FC<SkillsCatalogPageProps> = ({ mode, onMo
</div>
<div className="space-y-2">
{filtered.length === 0 ? (
{filtered.length === 0 && !isLoadingSource ? (
<div className="py-10 text-center text-muted-foreground">
<p className="typography-body">No skills found</p>
<p className="typography-meta mt-1 opacity-75">Try a different search or refresh the catalog</p>
</div>
) : isLoadingSource ? (
<div className="py-10 text-center text-muted-foreground">
<p className="typography-body">Loading skills</p>
</div>
) : (
filtered.map((item) => {
const installed = item.installed?.isInstalled;
const installedScope = item.installed?.scope;
<>
{filtered.map((item) => {
const installed = item.installed?.isInstalled;
const installedScope = item.installed?.scope;
return (
<div
key={`${item.sourceId}:${item.skillDir}`}
className="rounded-lg border bg-muted/10 px-3 py-2"
>
<div className="flex items-start justify-between gap-3">
<div className="min-w-0">
<div className="flex items-center gap-2">
<div className="typography-ui-label truncate">{item.skillName}</div>
{installed ? (
<span className="typography-micro text-muted-foreground bg-muted px-1 rounded flex-shrink-0 leading-none pb-px border border-border/50">
installed ({installedScope || 'unknown'})
</span>
) : null}
{!item.installable ? (
<span className="typography-micro text-muted-foreground bg-muted px-1 rounded flex-shrink-0 leading-none pb-px border border-border/50">
not installable
</span>
) : null}
</div>
{item.description ? (
<div className="typography-meta text-muted-foreground mt-0.5 line-clamp-2">{item.description}</div>
) : (
<div className="typography-micro text-muted-foreground mt-0.5">No description provided</div>
)}
{item.clawdhub ? (
<div className="typography-micro text-muted-foreground mt-1 flex items-center gap-3">
{item.clawdhub.owner ? (
<span>by {item.clawdhub.owner}</span>
) : null}
<span className="flex items-center gap-1">
<RiDownloadLine className="h-3 w-3" />
{item.clawdhub.downloads?.toLocaleString() ?? 0}
</span>
{(item.clawdhub.stars ?? 0) > 0 ? (
<span className="flex items-center gap-1">
<RiStarLine className="h-3 w-3" />
{item.clawdhub.stars}
return (
<div
key={`${item.sourceId}:${item.skillDir}`}
className="rounded-lg border bg-muted/10 px-3 py-2"
>
<div className="flex items-start justify-between gap-3">
<div className="min-w-0">
<div className="flex items-center gap-2">
<div className="typography-ui-label truncate">{item.skillName}</div>
{installed ? (
<span className="typography-micro text-muted-foreground bg-muted px-1 rounded flex-shrink-0 leading-none pb-px border border-border/50">
installed ({installedScope || 'unknown'})
</span>
) : null}
{!item.installable ? (
<span className="typography-micro text-muted-foreground bg-muted px-1 rounded flex-shrink-0 leading-none pb-px border border-border/50">
not installable
</span>
) : null}
<span>v{item.clawdhub.version}</span>
</div>
) : null}
{item.warnings?.length ? (
<div className="typography-micro text-muted-foreground mt-1">{item.warnings.join(' · ')}</div>
) : null}
</div>
{item.description ? (
<div className="typography-meta text-muted-foreground mt-0.5 line-clamp-2">{item.description}</div>
) : (
<div className="typography-micro text-muted-foreground mt-0.5">No description provided</div>
)}
{item.clawdhub ? (
<div className="typography-micro text-muted-foreground mt-1 flex items-center gap-3">
{item.clawdhub.owner ? (
<span>by {item.clawdhub.owner}</span>
) : null}
<span className="flex items-center gap-1">
<RiDownloadLine className="h-3 w-3" />
{item.clawdhub.downloads?.toLocaleString() ?? 0}
</span>
{(item.clawdhub.stars ?? 0) > 0 ? (
<span className="flex items-center gap-1">
<RiStarLine className="h-3 w-3" />
{item.clawdhub.stars}
</span>
) : null}
<span>v{item.clawdhub.version}</span>
</div>
) : null}
{item.warnings?.length ? (
<div className="typography-micro text-muted-foreground mt-1">{item.warnings.join(' · ')}</div>
) : null}
</div>
<Button
type="button"
variant="outline"
size="sm"
className="gap-2"
disabled={!item.installable}
onClick={() => {
setInstallItem(item);
setInstallDialogOpen(true);
}}
>
Install
</Button>
<Button
type="button"
variant="outline"
size="sm"
className="gap-2"
disabled={!item.installable}
onClick={() => {
setInstallItem(item);
setInstallDialogOpen(true);
}}
>
Install
</Button>
</div>
</div>
);
})}
{isClawdHubSource && hasMoreClawdHub ? (
<div className="flex justify-center pt-2">
<Button
type="button"
variant="outline"
onClick={() => void loadMoreClawdHub()}
disabled={isLoadingMore || isLoadingSource}
>
{isLoadingMore ? 'Loading…' : 'Load more'}
</Button>
</div>
);
})
) : null}
</>
)}
</div>
+8
View File
@@ -828,6 +828,14 @@ export interface SkillsCatalogResponse {
ok: boolean;
sources?: SkillsCatalogSource[];
itemsBySource?: Record<SkillsCatalogSourceId, SkillsCatalogItem[]>;
pageInfoBySource?: Record<SkillsCatalogSourceId, { nextCursor?: string | null }>;
error?: { kind: string; message: string };
}
export interface SkillsCatalogSourceResponse {
ok: boolean;
items?: SkillsCatalogItem[];
nextCursor?: string | null;
error?: { kind: string; message: string };
}
+217 -39
View File
@@ -10,11 +10,30 @@ import type {
SkillsInstallRequest,
SkillsInstallResponse,
SkillsInstallError,
SkillsCatalogSourceResponse,
} from '@/lib/api/types';
import { useSkillsStore } from '@/stores/useSkillsStore';
import { opencodeClient } from '@/lib/opencode/client';
const FALLBACK_SOURCES: SkillsCatalogSource[] = [
{
id: 'anthropic',
label: 'Anthropic',
description: "Anthropic's public skills repository",
source: 'anthropics/skills',
defaultSubpath: 'skills',
sourceType: 'github',
},
{
id: 'clawdhub',
label: 'ClawdHub',
description: 'Community skill registry with vector search',
source: 'clawdhub:registry',
sourceType: 'clawdhub',
},
];
const getCurrentDirectory = (): string | null => {
const opencodeDirectory = opencodeClient.getDirectory();
if (typeof opencodeDirectory === 'string' && opencodeDirectory.trim().length > 0) {
@@ -38,8 +57,13 @@ export interface SkillsCatalogState {
sources: SkillsCatalogSource[];
itemsBySource: Record<string, SkillsCatalogItem[]>;
selectedSourceId: string | null;
pageInfoBySource: Record<string, { nextCursor?: string | null }>;
loadedSourceIds: Record<string, boolean>;
clawdhubHasMoreBySource: Record<string, boolean>;
isLoadingCatalog: boolean;
isLoadingSource: boolean;
isLoadingMore: boolean;
isScanning: boolean;
isInstalling: boolean;
@@ -52,6 +76,8 @@ export interface SkillsCatalogState {
setSelectedSource: (id: string | null) => void;
loadCatalog: (options?: { refresh?: boolean }) => Promise<boolean>;
loadSource: (sourceId: string, options?: { refresh?: boolean }) => Promise<boolean>;
loadMoreClawdHub: () => Promise<boolean>;
scanRepo: (request: SkillsRepoScanRequest) => Promise<SkillsRepoScanResponse>;
installSkills: (request: SkillsInstallRequest) => Promise<SkillsInstallResponse>;
}
@@ -59,11 +85,16 @@ export interface SkillsCatalogState {
export const useSkillsCatalogStore = create<SkillsCatalogState>()(
devtools(
(set, get) => ({
sources: [],
sources: FALLBACK_SOURCES,
itemsBySource: {},
selectedSourceId: null,
selectedSourceId: FALLBACK_SOURCES[0]?.id ?? null,
pageInfoBySource: {},
loadedSourceIds: {},
clawdhubHasMoreBySource: {},
isLoadingCatalog: false,
isLoadingSource: false,
isLoadingMore: false,
isScanning: false,
isInstalling: false,
@@ -81,55 +112,64 @@ export const useSkillsCatalogStore = create<SkillsCatalogState>()(
const previous = {
sources: get().sources,
itemsBySource: get().itemsBySource,
pageInfoBySource: get().pageInfoBySource,
loadedSourceIds: get().loadedSourceIds,
clawdhubHasMoreBySource: get().clawdhubHasMoreBySource,
};
let lastError: SkillsCatalogResponse['error'] | null = null;
try {
for (let attempt = 0; attempt < 3; attempt++) {
try {
const currentDirectory = getCurrentDirectory();
const refresh = options?.refresh ? '&refresh=true' : '';
const queryParams = currentDirectory
? `?directory=${encodeURIComponent(currentDirectory)}${refresh}`
: refresh
? `?refresh=true`
: '';
const refresh = options?.refresh ? '?refresh=true' : '';
const controller = new AbortController();
const timeoutId = window.setTimeout(() => controller.abort(), 3000);
const response = await fetch(`/api/config/skills/catalog${queryParams}`, {
method: 'GET',
headers: { Accept: 'application/json' },
});
try {
const response = await fetch(`/api/config/skills/catalog${refresh}`, {
method: 'GET',
headers: { Accept: 'application/json' },
signal: controller.signal,
});
const payload = (await response.json().catch(() => null)) as SkillsCatalogResponse | null;
if (!response.ok || !payload?.ok) {
lastError = payload?.error || { kind: 'unknown', message: `Failed to load catalog (${response.status})` };
const waitMs = 200 * (attempt + 1);
await new Promise((resolve) => setTimeout(resolve, waitMs));
continue;
}
const sources = payload.sources || [];
const itemsBySource = payload.itemsBySource || {};
const currentSelected = get().selectedSourceId;
const selectedSourceId =
(currentSelected && sources.some((s) => s.id === currentSelected))
? currentSelected
: (sources[0]?.id ?? null);
set({ sources, itemsBySource, selectedSourceId });
return true;
} catch (error) {
lastError = { kind: 'unknown', message: error instanceof Error ? error.message : String(error) };
const waitMs = 200 * (attempt + 1);
await new Promise((resolve) => setTimeout(resolve, waitMs));
const payload = (await response.json().catch(() => null)) as SkillsCatalogResponse | null;
if (!response.ok || !payload?.ok) {
lastError = payload?.error || { kind: 'unknown', message: `Failed to load catalog (${response.status})` };
throw new Error(lastError.message);
}
}
const sources = (payload.sources && payload.sources.length > 0) ? payload.sources : previous.sources;
const itemsBySource = options?.refresh ? {} : (get().itemsBySource || {});
const pageInfoBySource = options?.refresh ? {} : (get().pageInfoBySource || {});
const loadedSourceIds = options?.refresh ? {} : (get().loadedSourceIds || {});
const clawdhubHasMoreBySource = options?.refresh ? {} : (get().clawdhubHasMoreBySource || {});
const currentSelected = get().selectedSourceId;
const selectedSourceId =
(currentSelected && sources.some((s) => s.id === currentSelected))
? currentSelected
: (sources[0]?.id ?? null);
set({
sources,
itemsBySource,
pageInfoBySource,
loadedSourceIds,
clawdhubHasMoreBySource,
selectedSourceId,
});
return true;
} finally {
window.clearTimeout(timeoutId);
}
} catch (error) {
lastError = lastError || { kind: 'unknown', message: error instanceof Error ? error.message : String(error) };
set({
sources: previous.sources,
itemsBySource: previous.itemsBySource,
pageInfoBySource: previous.pageInfoBySource,
loadedSourceIds: previous.loadedSourceIds,
clawdhubHasMoreBySource: previous.clawdhubHasMoreBySource,
lastCatalogError: lastError || { kind: 'unknown', message: 'Failed to load catalog' },
});
@@ -139,6 +179,144 @@ export const useSkillsCatalogStore = create<SkillsCatalogState>()(
}
},
loadSource: async (sourceId, options) => {
if (!sourceId) {
return false;
}
set({ isLoadingSource: true, lastCatalogError: null });
try {
const currentDirectory = getCurrentDirectory();
const refresh = options?.refresh ? '&refresh=true' : '';
const queryParams = currentDirectory
? `?directory=${encodeURIComponent(currentDirectory)}&sourceId=${encodeURIComponent(sourceId)}${refresh}`
: `?sourceId=${encodeURIComponent(sourceId)}${refresh}`;
const response = await fetch(`/api/config/skills/catalog/source${queryParams}`, {
method: 'GET',
headers: { Accept: 'application/json' },
});
const payload = (await response.json().catch(() => null)) as SkillsCatalogSourceResponse | null;
const hasItems = Array.isArray((payload as SkillsCatalogSourceResponse | null)?.items);
if (!response.ok || (!payload?.ok && !hasItems)) {
const fallback = await fetch(`/api/config/skills/catalog${queryParams}`, {
method: 'GET',
headers: { Accept: 'application/json' },
});
const fallbackPayload = (await fallback.json().catch(() => null)) as SkillsCatalogResponse | null;
const fallbackItems = fallbackPayload?.itemsBySource?.[sourceId];
if (fallback.ok && fallbackPayload?.ok && Array.isArray(fallbackItems)) {
set((state) => ({
itemsBySource: { ...state.itemsBySource, [sourceId]: fallbackItems },
pageInfoBySource: { ...state.pageInfoBySource, [sourceId]: { nextCursor: null } },
loadedSourceIds: { ...state.loadedSourceIds, [sourceId]: true },
clawdhubHasMoreBySource: { ...state.clawdhubHasMoreBySource, [sourceId]: false },
}));
return true;
}
set({
lastCatalogError: payload?.error || { kind: 'unknown', message: `Failed to load source (${response.status})` },
});
return false;
}
const items = payload?.items || [];
const nextCursor = payload?.nextCursor ?? null;
set((state) => ({
itemsBySource: { ...state.itemsBySource, [sourceId]: items },
pageInfoBySource: { ...state.pageInfoBySource, [sourceId]: { nextCursor } },
loadedSourceIds: { ...state.loadedSourceIds, [sourceId]: true },
clawdhubHasMoreBySource: {
...state.clawdhubHasMoreBySource,
[sourceId]: items.length > 0,
},
}));
return true;
} catch (error) {
set({
lastCatalogError: { kind: 'unknown', message: error instanceof Error ? error.message : String(error) },
});
return false;
} finally {
set({ isLoadingSource: false });
}
},
loadMoreClawdHub: async () => {
const selectedSourceId = get().selectedSourceId;
if (!selectedSourceId) {
return false;
}
const pageInfo = get().pageInfoBySource[selectedSourceId];
const cursor = pageInfo?.nextCursor || null;
set({ isLoadingMore: true });
try {
const currentDirectory = getCurrentDirectory();
const parts = [`sourceId=${encodeURIComponent(selectedSourceId)}`];
if (currentDirectory) {
parts.push(`directory=${encodeURIComponent(currentDirectory)}`);
}
if (cursor) {
parts.push(`cursor=${encodeURIComponent(cursor)}`);
}
const queryParams = `?${parts.join('&')}`;
const response = await fetch(`/api/config/skills/catalog/source${queryParams}`, {
method: 'GET',
headers: { Accept: 'application/json' },
});
const payload = (await response.json().catch(() => null)) as SkillsCatalogSourceResponse | null;
if (!response.ok || !payload?.ok) {
return false;
}
const nextCursor = payload.nextCursor ?? null;
const currentItems = get().itemsBySource[selectedSourceId] || [];
const items = payload.items || [];
const merged = new Map(currentItems.map((item) => [`${item.sourceId}:${item.skillDir}`, item]));
let newCount = 0;
for (const item of items) {
const key = `${item.sourceId}:${item.skillDir}`;
if (!merged.has(key)) {
newCount += 1;
}
merged.set(key, item);
}
const noMore = items.length === 0 || newCount === 0;
set((state) => ({
itemsBySource: {
...state.itemsBySource,
[selectedSourceId]: Array.from(merged.values()),
},
pageInfoBySource: {
...state.pageInfoBySource,
[selectedSourceId]: { nextCursor },
},
clawdhubHasMoreBySource: {
...state.clawdhubHasMoreBySource,
[selectedSourceId]: !noMore,
},
}));
return true;
} catch {
return false;
} finally {
set({ isLoadingMore: false });
}
},
scanRepo: async (request) => {
set({ isScanning: true, lastScanError: null, scanResults: null });
try {
+64 -27
View File
@@ -94,6 +94,7 @@ export const CURATED_SOURCES: CuratedSource[] = [
// ============== ClawdHub API ==============
const CLAWDHUB_API_BASE = 'https://clawdhub.com/api/v1';
const CLAWDHUB_PAGE_LIMIT = 25;
const CLAWDHUB_RATE_LIMIT_MS = 100;
let clawdhubLastRequest = 0;
@@ -102,21 +103,40 @@ function isClawdHubSource(source: string): boolean {
}
async function clawdhubFetch(url: string, options?: RequestInit): Promise<Response> {
const now = Date.now();
const elapsed = now - clawdhubLastRequest;
if (elapsed < CLAWDHUB_RATE_LIMIT_MS) {
await new Promise((resolve) => setTimeout(resolve, CLAWDHUB_RATE_LIMIT_MS - elapsed));
}
clawdhubLastRequest = Date.now();
const maxAttempts = 10;
let lastResponse: Response | null = null;
return fetch(url, {
...options,
headers: {
Accept: 'application/json',
'User-Agent': 'OpenChamber-VSCode/1.0',
...options?.headers,
},
});
for (let attempt = 0; attempt < maxAttempts; attempt++) {
const now = Date.now();
const elapsed = now - clawdhubLastRequest;
if (elapsed < CLAWDHUB_RATE_LIMIT_MS) {
await new Promise((resolve) => setTimeout(resolve, CLAWDHUB_RATE_LIMIT_MS - elapsed));
}
clawdhubLastRequest = Date.now();
const response = await fetch(url, {
...options,
headers: {
Accept: 'application/json',
'User-Agent': 'OpenChamber-VSCode/1.0',
...options?.headers,
},
});
lastResponse = response;
if (response.status === 429 || response.status >= 500) {
if (attempt < maxAttempts - 1) {
const waitMs = 50 * (attempt + 1);
await new Promise((resolve) => setTimeout(resolve, waitMs));
continue;
}
}
return response;
}
return lastResponse as Response;
}
type ClawdHubSkillListItem = {
@@ -142,16 +162,25 @@ async function scanClawdHub(): Promise<SkillsRepoScanResult> {
for (let page = 0; page < maxPages; page++) {
const url = cursor
? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}`
: `${CLAWDHUB_API_BASE}/skills`;
? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}&limit=${CLAWDHUB_PAGE_LIMIT}`
: `${CLAWDHUB_API_BASE}/skills?limit=${CLAWDHUB_PAGE_LIMIT}`;
const response = await clawdhubFetch(url);
if (!response.ok) {
throw new Error(`ClawdHub API error: ${response.status}`);
let data: ClawdHubSkillsResponse;
try {
const response = await clawdhubFetch(url);
if (!response.ok) {
throw new Error(`ClawdHub API error: ${response.status}`);
}
data = (await response.json()) as ClawdHubSkillsResponse;
} catch (error) {
if (page > 0 && allItems.length > 0) {
break;
}
throw error;
}
const data = (await response.json()) as ClawdHubSkillsResponse;
for (const item of data.items || []) {
const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0';
@@ -275,15 +304,23 @@ export async function installSkillsFromClawdHub(options: {
try {
// Resolve 'latest' version
if (version === 'latest') {
try {
const info = await fetchClawdHubSkillInfo(slug);
version = info.skill?.tags?.latest || info.latestVersion?.version || version;
} catch {
// Fall back to 'latest'
if (version === 'latest') {
try {
const info = await fetchClawdHubSkillInfo(slug);
const latest = info.skill?.tags?.latest || info.latestVersion?.version || null;
if (latest) {
version = latest;
}
} catch {
// ignore
}
if (version === 'latest') {
skipped.push({ skillName: slug, reason: 'Unable to resolve latest version' });
continue;
}
}
const targetDir = options.scope === 'user'
? path.join(userSkillDir, slug)
: path.join(options.workingDirectory as string, '.opencode', 'skills', slug);
+122 -76
View File
@@ -533,6 +533,25 @@ const resolveProjectDirectory = async (req) => {
return { directory: validated.directory, error: null };
};
const resolveOptionalProjectDirectory = async (req) => {
const headerDirectory = typeof req.get === 'function' ? req.get('x-opencode-directory') : null;
const queryDirectory = Array.isArray(req.query?.directory)
? req.query.directory[0]
: req.query?.directory;
const requested = headerDirectory || queryDirectory || null;
if (!requested) {
return { directory: null, error: null };
}
const validated = await validateDirectoryPath(requested);
if (!validated.ok) {
return { directory: null, error: validated.error };
}
return { directory: validated.directory, error: null };
};
const sanitizeTypographySizesPartial = (input) => {
if (!input || typeof input !== 'object') {
return undefined;
@@ -2416,6 +2435,7 @@ function setupProxy(app) {
req.path.startsWith('/push') ||
req.path.startsWith('/config/agents') ||
req.path.startsWith('/config/settings') ||
req.path.startsWith('/config/skills') ||
req.path === '/config/reload' ||
req.path === '/health'
) {
@@ -2448,6 +2468,7 @@ function setupProxy(app) {
req.path.startsWith('/themes/custom') ||
req.path.startsWith('/config/agents') ||
req.path.startsWith('/config/settings') ||
req.path.startsWith('/config/skills') ||
req.path === '/health'
) {
return next();
@@ -3508,7 +3529,7 @@ async function main(options = {}) {
const { parseSkillRepoSource } = await import('./lib/skills-catalog/source.js');
const { scanSkillsRepository } = await import('./lib/skills-catalog/scan.js');
const { installSkillsFromRepository } = await import('./lib/skills-catalog/install.js');
const { scanClawdHub, installSkillsFromClawdHub, isClawdHubSource } = await import('./lib/skills-catalog/clawdhub/index.js');
const { scanClawdHubPage, installSkillsFromClawdHub, isClawdHubSource } = await import('./lib/skills-catalog/clawdhub/index.js');
const { getProfiles, getProfile } = await import('./lib/git-identity-storage.js');
const listGitIdentitiesForResponse = () => {
@@ -3538,11 +3559,10 @@ async function main(options = {}) {
app.get('/api/config/skills/catalog', async (req, res) => {
try {
const { directory, error } = await resolveProjectDirectory(req);
if (!directory) {
const { error } = await resolveOptionalProjectDirectory(req);
if (error) {
return res.status(400).json({ error });
}
const refresh = String(req.query.refresh || '').toLowerCase() === 'true';
const curatedSources = getCuratedSkillsSources();
const settings = await readSettingsFromDisk();
@@ -3558,95 +3578,121 @@ async function main(options = {}) {
}));
const sources = [...curatedSources, ...customSources];
const sourcesForUi = sources.map(({ gitIdentityId, ...rest }) => rest);
const discovered = discoverSkills(directory);
res.json({ ok: true, sources: sourcesForUi, itemsBySource: {}, pageInfoBySource: {} });
} catch (error) {
console.error('Failed to load skills catalog:', error);
res.status(500).json({ ok: false, error: { kind: 'unknown', message: error.message || 'Failed to load catalog' } });
}
});
app.get('/api/config/skills/catalog/source', async (req, res) => {
try {
const { directory, error } = await resolveOptionalProjectDirectory(req);
if (error) {
return res.status(400).json({ ok: false, error: { kind: 'invalidSource', message: error } });
}
const sourceId = typeof req.query.sourceId === 'string' ? req.query.sourceId : null;
if (!sourceId) {
return res.status(400).json({ ok: false, error: { kind: 'invalidSource', message: 'Missing sourceId' } });
}
const refresh = String(req.query.refresh || '').toLowerCase() === 'true';
const cursor = typeof req.query.cursor === 'string' ? req.query.cursor : null;
const curatedSources = getCuratedSkillsSources();
const settings = await readSettingsFromDisk();
const customSourcesRaw = sanitizeSkillCatalogs(settings.skillCatalogs) || [];
const customSources = customSourcesRaw.map((entry) => ({
id: entry.id,
label: entry.label,
description: entry.source,
source: entry.source,
defaultSubpath: entry.subpath,
gitIdentityId: entry.gitIdentityId,
}));
const sources = [...curatedSources, ...customSources];
const src = sources.find((entry) => entry.id === sourceId);
if (!src) {
return res.status(404).json({ ok: false, error: { kind: 'invalidSource', message: 'Unknown source' } });
}
const discovered = directory ? discoverSkills(directory) : [];
const installedByName = new Map(discovered.map((s) => [s.name, s]));
const itemsBySource = {};
for (const src of sources) {
// Handle ClawdHub sources separately (API-based, not git-based)
if (src.sourceType === 'clawdhub' || isClawdHubSource(src.source)) {
const cacheKey = 'clawdhub:registry';
let scanResult = !refresh ? getCachedScan(cacheKey) : null;
if (!scanResult) {
const scanned = await scanClawdHub();
if (!scanned.ok) {
itemsBySource[src.id] = [];
continue;
}
scanResult = scanned;
setCachedScan(cacheKey, scanResult);
}
const items = (scanResult.items || []).map((item) => {
const installed = installedByName.get(item.skillName);
return {
...item,
sourceId: src.id,
installed: installed
? { isInstalled: true, scope: installed.scope }
: { isInstalled: false },
};
});
itemsBySource[src.id] = items;
continue;
if (src.sourceType === 'clawdhub' || isClawdHubSource(src.source)) {
const scanned = await scanClawdHubPage({ cursor: cursor || null });
if (!scanned.ok) {
return res.status(500).json({ ok: false, error: scanned.error });
}
// Handle GitHub sources (git clone based)
const parsed = parseSkillRepoSource(src.source);
if (!parsed.ok) {
itemsBySource[src.id] = [];
continue;
}
const effectiveSubpath = src.defaultSubpath || parsed.effectiveSubpath || null;
const cacheKey = getCacheKey({
normalizedRepo: parsed.normalizedRepo,
subpath: effectiveSubpath || '',
identityId: src.gitIdentityId || '',
});
let scanResult = !refresh ? getCachedScan(cacheKey) : null;
if (!scanResult) {
const scanned = await scanSkillsRepository({
source: src.source,
subpath: src.defaultSubpath,
defaultSubpath: src.defaultSubpath,
identity: resolveGitIdentity(src.gitIdentityId),
});
if (!scanned.ok) {
itemsBySource[src.id] = [];
continue;
}
scanResult = scanned;
setCachedScan(cacheKey, scanResult);
}
const items = (scanResult.items || []).map((item) => {
const items = (scanned.items || []).map((item) => {
const installed = installedByName.get(item.skillName);
return {
sourceId: src.id,
...item,
gitIdentityId: src.gitIdentityId,
sourceId: src.id,
installed: installed
? { isInstalled: true, scope: installed.scope }
: { isInstalled: false },
};
});
itemsBySource[src.id] = items;
return res.json({ ok: true, items, nextCursor: scanned.nextCursor || null });
}
const sourcesForUi = sources.map(({ gitIdentityId, ...rest }) => rest);
res.json({ ok: true, sources: sourcesForUi, itemsBySource });
const parsed = parseSkillRepoSource(src.source);
if (!parsed.ok) {
return res.status(400).json({ ok: false, error: parsed.error });
}
const effectiveSubpath = src.defaultSubpath || parsed.effectiveSubpath || null;
const cacheKey = getCacheKey({
normalizedRepo: parsed.normalizedRepo,
subpath: effectiveSubpath || '',
identityId: src.gitIdentityId || '',
});
let scanResult = !refresh ? getCachedScan(cacheKey) : null;
if (!scanResult) {
const scanned = await scanSkillsRepository({
source: src.source,
subpath: src.defaultSubpath,
defaultSubpath: src.defaultSubpath,
identity: resolveGitIdentity(src.gitIdentityId),
});
if (!scanned.ok) {
return res.status(500).json({ ok: false, error: scanned.error });
}
scanResult = scanned;
setCachedScan(cacheKey, scanResult);
}
const items = (scanResult.items || []).map((item) => {
const installed = installedByName.get(item.skillName);
return {
sourceId: src.id,
...item,
gitIdentityId: src.gitIdentityId,
installed: installed
? { isInstalled: true, scope: installed.scope }
: { isInstalled: false },
};
});
return res.json({ ok: true, items });
} catch (error) {
console.error('Failed to load skills catalog:', error);
res.status(500).json({ ok: false, error: { kind: 'unknown', message: error.message || 'Failed to load catalog' } });
console.error('Failed to load catalog source:', error);
return res.status(500).json({
ok: false,
error: { kind: 'unknown', message: error.message || 'Failed to load catalog source' },
});
}
});
@@ -6,29 +6,48 @@
*/
const CLAWDHUB_API_BASE = 'https://clawdhub.com/api/v1';
const CLAWDHUB_PAGE_LIMIT = 25;
// Rate limiting: ClawdHub allows 120 requests/minute
const RATE_LIMIT_DELAY_MS = 100;
let lastRequestTime = 0;
async function rateLimitedFetch(url, options = {}) {
const now = Date.now();
const elapsed = now - lastRequestTime;
if (elapsed < RATE_LIMIT_DELAY_MS) {
await new Promise((resolve) => setTimeout(resolve, RATE_LIMIT_DELAY_MS - elapsed));
const maxAttempts = 10;
let lastResponse = null;
for (let attempt = 0; attempt < maxAttempts; attempt++) {
const now = Date.now();
const elapsed = now - lastRequestTime;
if (elapsed < RATE_LIMIT_DELAY_MS) {
await new Promise((resolve) => setTimeout(resolve, RATE_LIMIT_DELAY_MS - elapsed));
}
lastRequestTime = Date.now();
const response = await fetch(url, {
...options,
headers: {
Accept: 'application/json',
'User-Agent': 'OpenChamber/1.0',
...options.headers,
},
});
lastResponse = response;
if (response.status === 429 || response.status >= 500) {
if (attempt < maxAttempts - 1) {
const waitMs = 50 * (attempt + 1);
await new Promise((resolve) => setTimeout(resolve, waitMs));
continue;
}
}
return response;
}
lastRequestTime = Date.now();
const response = await fetch(url, {
...options,
headers: {
Accept: 'application/json',
'User-Agent': 'OpenChamber/1.0',
...options.headers,
},
});
return response;
return lastResponse;
}
/**
@@ -39,8 +58,8 @@ async function rateLimitedFetch(url, options = {}) {
*/
export async function fetchClawdHubSkills({ cursor } = {}) {
const url = cursor
? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}`
: `${CLAWDHUB_API_BASE}/skills`;
? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}&limit=${CLAWDHUB_PAGE_LIMIT}`
: `${CLAWDHUB_API_BASE}/skills?limit=${CLAWDHUB_PAGE_LIMIT}`;
const response = await rateLimitedFetch(url);
@@ -50,9 +69,16 @@ export async function fetchClawdHubSkills({ cursor } = {}) {
}
const data = await response.json();
const nextCursor =
(typeof data.nextCursor === 'string' && data.nextCursor) ||
(typeof data.next_cursor === 'string' && data.next_cursor) ||
(typeof data.next === 'string' && data.next) ||
(typeof data.cursor === 'string' && data.cursor) ||
null;
return {
items: data.items || [],
nextCursor: data.nextCursor || null,
nextCursor,
};
}
@@ -95,7 +121,10 @@ export async function fetchClawdHubSkillVersion(slug, version = 'latest') {
* @returns {Promise<ArrayBuffer>} - ZIP file contents
*/
export async function downloadClawdHubSkill(slug, version) {
const url = `${CLAWDHUB_API_BASE}/download?slug=${encodeURIComponent(slug)}&version=${encodeURIComponent(version)}`;
const versionParam = typeof version === 'string' && version !== 'latest'
? `&version=${encodeURIComponent(version)}`
: '&tag=latest';
const url = `${CLAWDHUB_API_BASE}/download?slug=${encodeURIComponent(slug)}${versionParam}`;
const response = await rateLimitedFetch(url, {
headers: {
@@ -5,7 +5,7 @@
* https://clawdhub.com
*/
export { scanClawdHub } from './scan.js';
export { scanClawdHub, scanClawdHubPage } from './scan.js';
export { installSkillsFromClawdHub } from './install.js';
export {
fetchClawdHubSkills,
@@ -150,10 +150,17 @@ export async function installSkillsFromClawdHub({
if (resolvedVersion === 'latest') {
try {
const info = await fetchClawdHubSkillInfo(plan.slug);
resolvedVersion = info.skill?.tags?.latest || info.latestVersion?.version || plan.version;
const latest = info.skill?.tags?.latest || info.latestVersion?.version || null;
if (latest) {
resolvedVersion = latest;
}
} catch {
// Fall back to 'latest' tag if info fetch fails
resolvedVersion = 'latest';
// ignore
}
if (resolvedVersion === 'latest') {
skipped.push({ skillName: plan.slug, reason: 'Unable to resolve latest version' });
continue;
}
}
@@ -8,6 +8,36 @@
import { fetchClawdHubSkills } from './api.js';
const MAX_PAGES = 20; // Safety limit to prevent infinite loops
const CLAWDHUB_PAGE_LIMIT = 25;
const mapClawdHubItem = (item) => {
const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0';
return {
sourceId: 'clawdhub',
repoSource: 'clawdhub:registry',
repoSubpath: null,
gitIdentityId: null,
skillDir: item.slug,
skillName: item.slug,
frontmatterName: item.displayName || item.slug,
description: item.summary || null,
installable: true,
warnings: [],
// ClawdHub-specific metadata
clawdhub: {
slug: item.slug,
version: latestVersion,
displayName: item.displayName,
owner: item.owner?.handle || null,
downloads: item.stats?.downloads || 0,
stars: item.stats?.stars || 0,
versionsCount: item.stats?.versions || 1,
createdAt: item.createdAt,
updatedAt: item.updatedAt,
},
};
};
/**
* Scan ClawdHub registry for all available skills
@@ -19,35 +49,23 @@ export async function scanClawdHub() {
let cursor = null;
for (let page = 0; page < MAX_PAGES; page++) {
const { items, nextCursor } = await fetchClawdHubSkills({ cursor });
let items = [];
let nextCursor = null;
try {
const pageResult = await fetchClawdHubSkills({ cursor });
items = pageResult.items || [];
nextCursor = pageResult.nextCursor || null;
} catch (error) {
if (page > 0 && allItems.length > 0) {
console.warn('ClawdHub pagination failed; returning partial results.');
break;
}
throw error;
}
for (const item of items) {
const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0';
allItems.push({
sourceId: 'clawdhub',
repoSource: 'clawdhub:registry',
repoSubpath: null,
gitIdentityId: null,
skillDir: item.slug,
skillName: item.slug,
frontmatterName: item.displayName || item.slug,
description: item.summary || null,
installable: true,
warnings: [],
// ClawdHub-specific metadata
clawdhub: {
slug: item.slug,
version: latestVersion,
displayName: item.displayName,
owner: item.owner?.handle || null,
downloads: item.stats?.downloads || 0,
stars: item.stats?.stars || 0,
versionsCount: item.stats?.versions || 1,
createdAt: item.createdAt,
updatedAt: item.updatedAt,
},
});
allItems.push(mapClawdHubItem(item));
}
if (!nextCursor) {
@@ -71,3 +89,25 @@ export async function scanClawdHub() {
};
}
}
/**
* Scan a single ClawdHub page (cursor-based)
* @returns {Promise<{ ok: boolean, items?: Array, nextCursor?: string | null, error?: Object }>}
*/
export async function scanClawdHubPage({ cursor } = {}) {
try {
const { items, nextCursor } = await fetchClawdHubSkills({ cursor });
const mapped = (items || []).map(mapClawdHubItem).slice(0, CLAWDHUB_PAGE_LIMIT);
mapped.sort((a, b) => (b.clawdhub?.downloads || 0) - (a.clawdhub?.downloads || 0));
return { ok: true, items: mapped, nextCursor: nextCursor || null };
} catch (error) {
console.error('ClawdHub page scan error:', error);
return {
ok: false,
error: {
kind: 'networkError',
message: error instanceof Error ? error.message : 'Failed to fetch skills from ClawdHub',
},
};
}
}