BuzzbeeSCD/openchamber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enhance ClawdHub integration with paging and retry

Browse Source 
Add paging support when loading ClawdHub skills
Retry on API errors and throttle for ClawdHub requests
Load source content on source change and show loading indicators
This commit is contained in:
Bohdan Triapitsyn
2026-01-26 19:50:11 +02:00
parent eec170771c
commit 25525d5b1b
11 changed files with 699 additions and 319 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/web/server/index.js
+122 -76
View File
@@ -533,6 +533,25 @@ const resolveProjectDirectory = async (req) => {
return { directory: validated.directory, error: null };
};
const resolveOptionalProjectDirectory = async (req) => {
const headerDirectory = typeof req.get === 'function' ? req.get('x-opencode-directory') : null;
const queryDirectory = Array.isArray(req.query?.directory)
? req.query.directory[0]
: req.query?.directory;
const requested = headerDirectory || queryDirectory || null;
if (!requested) {
return { directory: null, error: null };
}
const validated = await validateDirectoryPath(requested);
if (!validated.ok) {
return { directory: null, error: validated.error };
}
return { directory: validated.directory, error: null };
};
const sanitizeTypographySizesPartial = (input) => {
if (!input || typeof input !== 'object') {
return undefined;
@@ -2416,6 +2435,7 @@ function setupProxy(app) {
req.path.startsWith('/push') ||
req.path.startsWith('/config/agents') ||
req.path.startsWith('/config/settings') ||
req.path.startsWith('/config/skills') ||
req.path === '/config/reload' ||
req.path === '/health'
) {
@@ -2448,6 +2468,7 @@ function setupProxy(app) {
req.path.startsWith('/themes/custom') ||
req.path.startsWith('/config/agents') ||
req.path.startsWith('/config/settings') ||
req.path.startsWith('/config/skills') ||
req.path === '/health'
) {
return next();
@@ -3508,7 +3529,7 @@ async function main(options = {}) {
const { parseSkillRepoSource } = await import('./lib/skills-catalog/source.js');
const { scanSkillsRepository } = await import('./lib/skills-catalog/scan.js');
const { installSkillsFromRepository } = await import('./lib/skills-catalog/install.js');
const { scanClawdHub, installSkillsFromClawdHub, isClawdHubSource } = await import('./lib/skills-catalog/clawdhub/index.js');
const { scanClawdHubPage, installSkillsFromClawdHub, isClawdHubSource } = await import('./lib/skills-catalog/clawdhub/index.js');
const { getProfiles, getProfile } = await import('./lib/git-identity-storage.js');
const listGitIdentitiesForResponse = () => {
@@ -3538,11 +3559,10 @@ async function main(options = {}) {
app.get('/api/config/skills/catalog', async (req, res) => {
try {
const { directory, error } = await resolveProjectDirectory(req);
if (!directory) {
const { error } = await resolveOptionalProjectDirectory(req);
if (error) {
return res.status(400).json({ error });
}
const refresh = String(req.query.refresh || '').toLowerCase() === 'true';
const curatedSources = getCuratedSkillsSources();
const settings = await readSettingsFromDisk();
@@ -3558,95 +3578,121 @@ async function main(options = {}) {
}));
const sources = [...curatedSources, ...customSources];
const sourcesForUi = sources.map(({ gitIdentityId, ...rest }) => rest);
const discovered = discoverSkills(directory);
res.json({ ok: true, sources: sourcesForUi, itemsBySource: {}, pageInfoBySource: {} });
} catch (error) {
console.error('Failed to load skills catalog:', error);
res.status(500).json({ ok: false, error: { kind: 'unknown', message: error.message || 'Failed to load catalog' } });
}
});
app.get('/api/config/skills/catalog/source', async (req, res) => {
try {
const { directory, error } = await resolveOptionalProjectDirectory(req);
if (error) {
return res.status(400).json({ ok: false, error: { kind: 'invalidSource', message: error } });
}
const sourceId = typeof req.query.sourceId === 'string' ? req.query.sourceId : null;
if (!sourceId) {
return res.status(400).json({ ok: false, error: { kind: 'invalidSource', message: 'Missing sourceId' } });
}
const refresh = String(req.query.refresh || '').toLowerCase() === 'true';
const cursor = typeof req.query.cursor === 'string' ? req.query.cursor : null;
const curatedSources = getCuratedSkillsSources();
const settings = await readSettingsFromDisk();
const customSourcesRaw = sanitizeSkillCatalogs(settings.skillCatalogs) || [];
const customSources = customSourcesRaw.map((entry) => ({
id: entry.id,
label: entry.label,
description: entry.source,
source: entry.source,
defaultSubpath: entry.subpath,
gitIdentityId: entry.gitIdentityId,
}));
const sources = [...curatedSources, ...customSources];
const src = sources.find((entry) => entry.id === sourceId);
if (!src) {
return res.status(404).json({ ok: false, error: { kind: 'invalidSource', message: 'Unknown source' } });
}
const discovered = directory ? discoverSkills(directory) : [];
const installedByName = new Map(discovered.map((s) => [s.name, s]));
const itemsBySource = {};
for (const src of sources) {
// Handle ClawdHub sources separately (API-based, not git-based)
if (src.sourceType === 'clawdhub' || isClawdHubSource(src.source)) {
const cacheKey = 'clawdhub:registry';
let scanResult = !refresh ? getCachedScan(cacheKey) : null;
if (!scanResult) {
const scanned = await scanClawdHub();
if (!scanned.ok) {
itemsBySource[src.id] = [];
continue;
}
scanResult = scanned;
setCachedScan(cacheKey, scanResult);
}
const items = (scanResult.items || []).map((item) => {
const installed = installedByName.get(item.skillName);
return {
...item,
sourceId: src.id,
installed: installed
? { isInstalled: true, scope: installed.scope }
: { isInstalled: false },
};
});
itemsBySource[src.id] = items;
continue;
if (src.sourceType === 'clawdhub' || isClawdHubSource(src.source)) {
const scanned = await scanClawdHubPage({ cursor: cursor || null });
if (!scanned.ok) {
return res.status(500).json({ ok: false, error: scanned.error });
}
// Handle GitHub sources (git clone based)
const parsed = parseSkillRepoSource(src.source);
if (!parsed.ok) {
itemsBySource[src.id] = [];
continue;
}
const effectiveSubpath = src.defaultSubpath || parsed.effectiveSubpath || null;
const cacheKey = getCacheKey({
normalizedRepo: parsed.normalizedRepo,
subpath: effectiveSubpath || '',
identityId: src.gitIdentityId || '',
});
let scanResult = !refresh ? getCachedScan(cacheKey) : null;
if (!scanResult) {
const scanned = await scanSkillsRepository({
source: src.source,
subpath: src.defaultSubpath,
defaultSubpath: src.defaultSubpath,
identity: resolveGitIdentity(src.gitIdentityId),
});
if (!scanned.ok) {
itemsBySource[src.id] = [];
continue;
}
scanResult = scanned;
setCachedScan(cacheKey, scanResult);
}
const items = (scanResult.items || []).map((item) => {
const items = (scanned.items || []).map((item) => {
const installed = installedByName.get(item.skillName);
return {
sourceId: src.id,
...item,
gitIdentityId: src.gitIdentityId,
sourceId: src.id,
installed: installed
? { isInstalled: true, scope: installed.scope }
: { isInstalled: false },
};
});
itemsBySource[src.id] = items;
return res.json({ ok: true, items, nextCursor: scanned.nextCursor || null });
}
const sourcesForUi = sources.map(({ gitIdentityId, ...rest }) => rest);
res.json({ ok: true, sources: sourcesForUi, itemsBySource });
const parsed = parseSkillRepoSource(src.source);
if (!parsed.ok) {
return res.status(400).json({ ok: false, error: parsed.error });
}
const effectiveSubpath = src.defaultSubpath || parsed.effectiveSubpath || null;
const cacheKey = getCacheKey({
normalizedRepo: parsed.normalizedRepo,
subpath: effectiveSubpath || '',
identityId: src.gitIdentityId || '',
});
let scanResult = !refresh ? getCachedScan(cacheKey) : null;
if (!scanResult) {
const scanned = await scanSkillsRepository({
source: src.source,
subpath: src.defaultSubpath,
defaultSubpath: src.defaultSubpath,
identity: resolveGitIdentity(src.gitIdentityId),
});
if (!scanned.ok) {
return res.status(500).json({ ok: false, error: scanned.error });
}
scanResult = scanned;
setCachedScan(cacheKey, scanResult);
}
const items = (scanResult.items || []).map((item) => {
const installed = installedByName.get(item.skillName);
return {
sourceId: src.id,
...item,
gitIdentityId: src.gitIdentityId,
installed: installed
? { isInstalled: true, scope: installed.scope }
: { isInstalled: false },
};
});
return res.json({ ok: true, items });
} catch (error) {
console.error('Failed to load skills catalog:', error);
res.status(500).json({ ok: false, error: { kind: 'unknown', message: error.message || 'Failed to load catalog' } });
console.error('Failed to load catalog source:', error);
return res.status(500).json({
ok: false,
error: { kind: 'unknown', message: error.message || 'Failed to load catalog source' },
});
}
});
packages/web/server/lib/skills-catalog/clawdhub/api.js
+48 -19
View File
@@ -6,29 +6,48 @@
*/
const CLAWDHUB_API_BASE = 'https://clawdhub.com/api/v1';
const CLAWDHUB_PAGE_LIMIT = 25;
// Rate limiting: ClawdHub allows 120 requests/minute
const RATE_LIMIT_DELAY_MS = 100;
let lastRequestTime = 0;
async function rateLimitedFetch(url, options = {}) {
const now = Date.now();
const elapsed = now - lastRequestTime;
if (elapsed < RATE_LIMIT_DELAY_MS) {
await new Promise((resolve) => setTimeout(resolve, RATE_LIMIT_DELAY_MS - elapsed));
const maxAttempts = 10;
let lastResponse = null;
for (let attempt = 0; attempt < maxAttempts; attempt++) {
const now = Date.now();
const elapsed = now - lastRequestTime;
if (elapsed < RATE_LIMIT_DELAY_MS) {
await new Promise((resolve) => setTimeout(resolve, RATE_LIMIT_DELAY_MS - elapsed));
}
lastRequestTime = Date.now();
const response = await fetch(url, {
...options,
headers: {
Accept: 'application/json',
'User-Agent': 'OpenChamber/1.0',
...options.headers,
},
});
lastResponse = response;
if (response.status === 429 || response.status >= 500) {
if (attempt < maxAttempts - 1) {
const waitMs = 50 * (attempt + 1);
await new Promise((resolve) => setTimeout(resolve, waitMs));
continue;
}
}
return response;
}
lastRequestTime = Date.now();
const response = await fetch(url, {
...options,
headers: {
Accept: 'application/json',
'User-Agent': 'OpenChamber/1.0',
...options.headers,
},
});
return response;
return lastResponse;
}
/**
@@ -39,8 +58,8 @@ async function rateLimitedFetch(url, options = {}) {
*/
export async function fetchClawdHubSkills({ cursor } = {}) {
const url = cursor
? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}`
: `${CLAWDHUB_API_BASE}/skills`;
? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}&limit=${CLAWDHUB_PAGE_LIMIT}`
: `${CLAWDHUB_API_BASE}/skills?limit=${CLAWDHUB_PAGE_LIMIT}`;
const response = await rateLimitedFetch(url);
@@ -50,9 +69,16 @@ export async function fetchClawdHubSkills({ cursor } = {}) {
}
const data = await response.json();
const nextCursor =
(typeof data.nextCursor === 'string' && data.nextCursor) ||
(typeof data.next_cursor === 'string' && data.next_cursor) ||
(typeof data.next === 'string' && data.next) ||
(typeof data.cursor === 'string' && data.cursor) ||
null;
return {
items: data.items || [],
nextCursor: data.nextCursor || null,
nextCursor,
};
}
@@ -95,7 +121,10 @@ export async function fetchClawdHubSkillVersion(slug, version = 'latest') {
* @returns {Promise<ArrayBuffer>} - ZIP file contents
*/
export async function downloadClawdHubSkill(slug, version) {
const url = `${CLAWDHUB_API_BASE}/download?slug=${encodeURIComponent(slug)}&version=${encodeURIComponent(version)}`;
const versionParam = typeof version === 'string' && version !== 'latest'
? `&version=${encodeURIComponent(version)}`
: '&tag=latest';
const url = `${CLAWDHUB_API_BASE}/download?slug=${encodeURIComponent(slug)}${versionParam}`;
const response = await rateLimitedFetch(url, {
headers: {
packages/web/server/lib/skills-catalog/clawdhub/index.js
+1 -1
View File
@@ -5,7 +5,7 @@
* https://clawdhub.com
*/
export { scanClawdHub } from './scan.js';
export { scanClawdHub, scanClawdHubPage } from './scan.js';
export { installSkillsFromClawdHub } from './install.js';
export {
fetchClawdHubSkills,
packages/web/server/lib/skills-catalog/clawdhub/install.js
+10 -3
View File
@@ -150,10 +150,17 @@ export async function installSkillsFromClawdHub({
if (resolvedVersion === 'latest') {
try {
const info = await fetchClawdHubSkillInfo(plan.slug);
resolvedVersion = info.skill?.tags?.latest || info.latestVersion?.version || plan.version;
const latest = info.skill?.tags?.latest || info.latestVersion?.version || null;
if (latest) {
resolvedVersion = latest;
}
} catch {
// Fall back to 'latest' tag if info fetch fails
resolvedVersion = 'latest';
// ignore
}
if (resolvedVersion === 'latest') {
skipped.push({ skillName: plan.slug, reason: 'Unable to resolve latest version' });
continue;
}
}
packages/web/server/lib/skills-catalog/clawdhub/scan.js
+67 -27
View File
@@ -8,6 +8,36 @@
import { fetchClawdHubSkills } from './api.js';
const MAX_PAGES = 20; // Safety limit to prevent infinite loops
const CLAWDHUB_PAGE_LIMIT = 25;
const mapClawdHubItem = (item) => {
const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0';
return {
sourceId: 'clawdhub',
repoSource: 'clawdhub:registry',
repoSubpath: null,
gitIdentityId: null,
skillDir: item.slug,
skillName: item.slug,
frontmatterName: item.displayName || item.slug,
description: item.summary || null,
installable: true,
warnings: [],
// ClawdHub-specific metadata
clawdhub: {
slug: item.slug,
version: latestVersion,
displayName: item.displayName,
owner: item.owner?.handle || null,
downloads: item.stats?.downloads || 0,
stars: item.stats?.stars || 0,
versionsCount: item.stats?.versions || 1,
createdAt: item.createdAt,
updatedAt: item.updatedAt,
},
};
};
/**
* Scan ClawdHub registry for all available skills
@@ -19,35 +49,23 @@ export async function scanClawdHub() {
let cursor = null;
for (let page = 0; page < MAX_PAGES; page++) {
const { items, nextCursor } = await fetchClawdHubSkills({ cursor });
let items = [];
let nextCursor = null;
try {
const pageResult = await fetchClawdHubSkills({ cursor });
items = pageResult.items || [];
nextCursor = pageResult.nextCursor || null;
} catch (error) {
if (page > 0 && allItems.length > 0) {
console.warn('ClawdHub pagination failed; returning partial results.');
break;
}
throw error;
}
for (const item of items) {
const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0';
allItems.push({
sourceId: 'clawdhub',
repoSource: 'clawdhub:registry',
repoSubpath: null,
gitIdentityId: null,
skillDir: item.slug,
skillName: item.slug,
frontmatterName: item.displayName || item.slug,
description: item.summary || null,
installable: true,
warnings: [],
// ClawdHub-specific metadata
clawdhub: {
slug: item.slug,
version: latestVersion,
displayName: item.displayName,
owner: item.owner?.handle || null,
downloads: item.stats?.downloads || 0,
stars: item.stats?.stars || 0,
versionsCount: item.stats?.versions || 1,
createdAt: item.createdAt,
updatedAt: item.updatedAt,
},
});
allItems.push(mapClawdHubItem(item));
}
if (!nextCursor) {
@@ -71,3 +89,25 @@ export async function scanClawdHub() {
};
}
}
/**
* Scan a single ClawdHub page (cursor-based)
* @returns {Promise<{ ok: boolean, items?: Array, nextCursor?: string | null, error?: Object }>}
*/
export async function scanClawdHubPage({ cursor } = {}) {
try {
const { items, nextCursor } = await fetchClawdHubSkills({ cursor });
const mapped = (items || []).map(mapClawdHubItem).slice(0, CLAWDHUB_PAGE_LIMIT);
mapped.sort((a, b) => (b.clawdhub?.downloads || 0) - (a.clawdhub?.downloads || 0));
return { ok: true, items: mapped, nextCursor: nextCursor || null };
} catch (error) {
console.error('ClawdHub page scan error:', error);
return {
ok: false,
error: {
kind: 'networkError',
message: error instanceof Error ? error.message : 'Failed to fetch skills from ClawdHub',
},
};
}
}