feat: enhance ClawdHub integration with paging and retry

Add paging support when loading ClawdHub skills Retry on API errors and throttle for ClawdHub requests Load source content on source change and show loading indicators
2026-01-26 19:50:11 +02:00
parent eec170771c
commit 25525d5b1b
11 changed files with 699 additions and 319 deletions
@@ -6,29 +6,48 @@
 */

 const CLAWDHUB_API_BASE = 'https://clawdhub.com/api/v1';
+const CLAWDHUB_PAGE_LIMIT = 25;

 // Rate limiting: ClawdHub allows 120 requests/minute
 const RATE_LIMIT_DELAY_MS = 100;
 let lastRequestTime = 0;

 async function rateLimitedFetch(url, options = {}) {
-  const now = Date.now();
-  const elapsed = now - lastRequestTime;
-  if (elapsed < RATE_LIMIT_DELAY_MS) {
-    await new Promise((resolve) => setTimeout(resolve, RATE_LIMIT_DELAY_MS - elapsed));
+  const maxAttempts = 10;
+
+  let lastResponse = null;
+
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    const now = Date.now();
+    const elapsed = now - lastRequestTime;
+    if (elapsed < RATE_LIMIT_DELAY_MS) {
+      await new Promise((resolve) => setTimeout(resolve, RATE_LIMIT_DELAY_MS - elapsed));
+    }
+    lastRequestTime = Date.now();
+
+    const response = await fetch(url, {
+      ...options,
+      headers: {
+        Accept: 'application/json',
+        'User-Agent': 'OpenChamber/1.0',
+        ...options.headers,
+      },
+    });
+
+    lastResponse = response;
+
+    if (response.status === 429 || response.status >= 500) {
+      if (attempt < maxAttempts - 1) {
+        const waitMs = 50 * (attempt + 1);
+        await new Promise((resolve) => setTimeout(resolve, waitMs));
+        continue;
+      }
+    }
+
+    return response;
  }
-  lastRequestTime = Date.now();

-  const response = await fetch(url, {
-    ...options,
-    headers: {
-      Accept: 'application/json',
-      'User-Agent': 'OpenChamber/1.0',
-      ...options.headers,
-    },
-  });
-
-  return response;
+  return lastResponse;
 }

 /**
@@ -39,8 +58,8 @@ async function rateLimitedFetch(url, options = {}) {
 */
 export async function fetchClawdHubSkills({ cursor } = {}) {
  const url = cursor
-    ? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}`
-    : `${CLAWDHUB_API_BASE}/skills`;
+    ? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}&limit=${CLAWDHUB_PAGE_LIMIT}`
+    : `${CLAWDHUB_API_BASE}/skills?limit=${CLAWDHUB_PAGE_LIMIT}`;

  const response = await rateLimitedFetch(url);

@@ -50,9 +69,16 @@ export async function fetchClawdHubSkills({ cursor } = {}) {
  }

  const data = await response.json();
+  const nextCursor =
+    (typeof data.nextCursor === 'string' && data.nextCursor) ||
+    (typeof data.next_cursor === 'string' && data.next_cursor) ||
+    (typeof data.next === 'string' && data.next) ||
+    (typeof data.cursor === 'string' && data.cursor) ||
+    null;
+
  return {
    items: data.items || [],
-    nextCursor: data.nextCursor || null,
+    nextCursor,
  };
 }

@@ -95,7 +121,10 @@ export async function fetchClawdHubSkillVersion(slug, version = 'latest') {
 * @returns {Promise<ArrayBuffer>} - ZIP file contents
 */
 export async function downloadClawdHubSkill(slug, version) {
-  const url = `${CLAWDHUB_API_BASE}/download?slug=${encodeURIComponent(slug)}&version=${encodeURIComponent(version)}`;
+  const versionParam = typeof version === 'string' && version !== 'latest'
+    ? `&version=${encodeURIComponent(version)}`
+    : '&tag=latest';
+  const url = `${CLAWDHUB_API_BASE}/download?slug=${encodeURIComponent(slug)}${versionParam}`;

  const response = await rateLimitedFetch(url, {
    headers: {
@@ -5,7 +5,7 @@
 * https://clawdhub.com
 */

-export { scanClawdHub } from './scan.js';
+export { scanClawdHub, scanClawdHubPage } from './scan.js';
 export { installSkillsFromClawdHub } from './install.js';
 export {
  fetchClawdHubSkills,
@@ -150,10 +150,17 @@ export async function installSkillsFromClawdHub({
      if (resolvedVersion === 'latest') {
        try {
          const info = await fetchClawdHubSkillInfo(plan.slug);
-          resolvedVersion = info.skill?.tags?.latest || info.latestVersion?.version || plan.version;
+          const latest = info.skill?.tags?.latest || info.latestVersion?.version || null;
+          if (latest) {
+            resolvedVersion = latest;
+          }
        } catch {
-          // Fall back to 'latest' tag if info fetch fails
-          resolvedVersion = 'latest';
+          // ignore
+        }
+
+        if (resolvedVersion === 'latest') {
+          skipped.push({ skillName: plan.slug, reason: 'Unable to resolve latest version' });
+          continue;
        }
      }

@@ -8,6 +8,36 @@
 import { fetchClawdHubSkills } from './api.js';

 const MAX_PAGES = 20; // Safety limit to prevent infinite loops
+const CLAWDHUB_PAGE_LIMIT = 25;
+
+const mapClawdHubItem = (item) => {
+  const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0';
+
+  return {
+    sourceId: 'clawdhub',
+    repoSource: 'clawdhub:registry',
+    repoSubpath: null,
+    gitIdentityId: null,
+    skillDir: item.slug,
+    skillName: item.slug,
+    frontmatterName: item.displayName || item.slug,
+    description: item.summary || null,
+    installable: true,
+    warnings: [],
+    // ClawdHub-specific metadata
+    clawdhub: {
+      slug: item.slug,
+      version: latestVersion,
+      displayName: item.displayName,
+      owner: item.owner?.handle || null,
+      downloads: item.stats?.downloads || 0,
+      stars: item.stats?.stars || 0,
+      versionsCount: item.stats?.versions || 1,
+      createdAt: item.createdAt,
+      updatedAt: item.updatedAt,
+    },
+  };
+};

 /**
 * Scan ClawdHub registry for all available skills
@@ -19,35 +49,23 @@ export async function scanClawdHub() {
    let cursor = null;

    for (let page = 0; page < MAX_PAGES; page++) {
-      const { items, nextCursor } = await fetchClawdHubSkills({ cursor });
+      let items = [];
+      let nextCursor = null;
+
+      try {
+        const pageResult = await fetchClawdHubSkills({ cursor });
+        items = pageResult.items || [];
+        nextCursor = pageResult.nextCursor || null;
+      } catch (error) {
+        if (page > 0 && allItems.length > 0) {
+          console.warn('ClawdHub pagination failed; returning partial results.');
+          break;
+        }
+        throw error;
+      }

      for (const item of items) {
-        const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0';
-
-        allItems.push({
-          sourceId: 'clawdhub',
-          repoSource: 'clawdhub:registry',
-          repoSubpath: null,
-          gitIdentityId: null,
-          skillDir: item.slug,
-          skillName: item.slug,
-          frontmatterName: item.displayName || item.slug,
-          description: item.summary || null,
-          installable: true,
-          warnings: [],
-          // ClawdHub-specific metadata
-          clawdhub: {
-            slug: item.slug,
-            version: latestVersion,
-            displayName: item.displayName,
-            owner: item.owner?.handle || null,
-            downloads: item.stats?.downloads || 0,
-            stars: item.stats?.stars || 0,
-            versionsCount: item.stats?.versions || 1,
-            createdAt: item.createdAt,
-            updatedAt: item.updatedAt,
-          },
-        });
+        allItems.push(mapClawdHubItem(item));
      }

      if (!nextCursor) {
@@ -71,3 +89,25 @@ export async function scanClawdHub() {
    };
  }
 }
+
+/**
+ * Scan a single ClawdHub page (cursor-based)
+ * @returns {Promise<{ ok: boolean, items?: Array, nextCursor?: string | null, error?: Object }>}
+ */
+export async function scanClawdHubPage({ cursor } = {}) {
+  try {
+    const { items, nextCursor } = await fetchClawdHubSkills({ cursor });
+    const mapped = (items || []).map(mapClawdHubItem).slice(0, CLAWDHUB_PAGE_LIMIT);
+    mapped.sort((a, b) => (b.clawdhub?.downloads || 0) - (a.clawdhub?.downloads || 0));
+    return { ok: true, items: mapped, nextCursor: nextCursor || null };
+  } catch (error) {
+    console.error('ClawdHub page scan error:', error);
+    return {
+      ok: false,
+      error: {
+        kind: 'networkError',
+        message: error instanceof Error ? error.message : 'Failed to fetch skills from ClawdHub',
+      },
+    };
+  }
+}