feat(skills): curated GitHub catalog redesign (#3016)
* feat(skills): remove ClawHub catalog integration Drop the ClawHub registry as a skills catalog source across web server, shared UI, VS Code, docs, and locales. The catalog now serves git-based sources only: the curated Anthropic repo and user-defined repositories. Also removes the now-unused adm-zip dependency. * feat(skills): redesign catalog around curated GitHub repositories Replace the single-source dropdown with a card grid of curated GitHub repositories (Anthropic, OpenAI, Cursor pstack/skills, Matt Pocock) plus user-defined sources. Source cards show skill counts, GitHub stars, and last-updated time; a global search covers all loaded sources. Server: curated sources gain GitHub repo metadata (stars, pushed_at) fetched best-effort with a 3-hour in-memory and on-disk cache; scans run through a concurrency-limited, deduplicated cache with 3-hour TTL persisted across restarts. Refresh still bypasses the cache. Shared UI: source cards, global search with clear button, per-skill GitHub links, install/installed states. VS Code curated list updated to match. All new copy translated across 12 locales. * fix(skills): address catalog review findings - GitHub metadata fetch timeout drops to 1.5s (under the catalog client's 3s deadline) and failed lookups cache briefly (5 min) so repeated catalog loads do not re-hit a failing API. - Disk cache files are written with owner-only permissions (0o600); rename preserves the mode. - loadSource deduplicates concurrent in-flight requests per source and the shared isLoadingSource flag now clears only when the last active source load finishes.
This commit is contained in:
Bohdan Triapitsyn
committed by
GitHub
GitHub
parent
90d8868bfc
commit
1ed3f1f575
@@ -1,49 +0,0 @@
|
||||
import { beforeEach, describe, expect, mock, test } from 'bun:test';
|
||||
|
||||
mock.module('@/lib/opencode/client', () => ({
|
||||
opencodeClient: {
|
||||
getDirectory: () => undefined,
|
||||
},
|
||||
}));
|
||||
|
||||
mock.module('@/stores/useProjectsStore', () => ({
|
||||
useProjectsStore: {
|
||||
getState: () => ({
|
||||
getActiveProject: () => null,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
mock.module('@/lib/runtime-fetch', () => ({
|
||||
runtimeFetch: async () => new Response('{}', { status: 500 }),
|
||||
}));
|
||||
|
||||
mock.module('@/stores/useSkillsStore', () => ({
|
||||
invalidateSkillsLoadCache: () => undefined,
|
||||
refreshSkillsAfterOpenCodeRestart: async () => undefined,
|
||||
useSkillsStore: {
|
||||
getState: () => ({}),
|
||||
},
|
||||
}));
|
||||
|
||||
mock.module('@/lib/configUpdate', () => ({
|
||||
startConfigUpdate: () => undefined,
|
||||
finishConfigUpdate: () => undefined,
|
||||
updateConfigUpdateMessage: () => undefined,
|
||||
}));
|
||||
|
||||
const { useSkillsCatalogStore } = await import('./useSkillsCatalogStore');
|
||||
|
||||
describe('skills catalog ClawHub label', () => {
|
||||
beforeEach(() => {
|
||||
useSkillsCatalogStore.setState({
|
||||
sources: useSkillsCatalogStore.getState().sources,
|
||||
});
|
||||
});
|
||||
|
||||
test('fallback sources label ClawHub correctly', () => {
|
||||
const clawhub = useSkillsCatalogStore.getState().sources.find((source) => source.id === 'clawdhub');
|
||||
expect(clawhub).toBeDefined();
|
||||
expect(clawhub?.label).toBe('ClawHub');
|
||||
});
|
||||
});
|
||||
@@ -30,11 +30,27 @@ const FALLBACK_SOURCES: SkillsCatalogSource[] = [
|
||||
sourceType: 'github',
|
||||
},
|
||||
{
|
||||
id: 'clawdhub',
|
||||
label: 'ClawHub',
|
||||
description: 'Community skill registry with vector search',
|
||||
source: 'clawdhub:registry',
|
||||
sourceType: 'clawdhub',
|
||||
id: 'openai',
|
||||
label: 'OpenAI',
|
||||
description: "OpenAI's curated skills",
|
||||
source: 'openai/skills',
|
||||
defaultSubpath: 'skills/.curated',
|
||||
sourceType: 'github',
|
||||
},
|
||||
{
|
||||
id: 'cursor',
|
||||
label: 'Cursor',
|
||||
description: "Cursor's plugin skills",
|
||||
source: 'cursor/plugins',
|
||||
defaultSubpath: 'pstack/skills',
|
||||
sourceType: 'github',
|
||||
},
|
||||
{
|
||||
id: 'mattpocock',
|
||||
label: 'Matt Pocock',
|
||||
description: 'Matt Pocock skills collection',
|
||||
source: 'mattpocock/skills',
|
||||
sourceType: 'github',
|
||||
},
|
||||
];
|
||||
|
||||
@@ -42,6 +58,8 @@ const SKILLS_CATALOG_LOAD_CACHE_TTL_MS = 5000;
|
||||
const DEFAULT_SKILLS_CATALOG_CACHE_KEY = '__default__';
|
||||
const skillsCatalogLastLoadedAt = new Map<string, number>();
|
||||
const skillsCatalogLoadInFlight = new Map<string, Promise<boolean>>();
|
||||
const sourceLoadInFlight = new Map<string, Promise<boolean>>();
|
||||
let activeSourceLoads = 0;
|
||||
|
||||
const getSkillsCatalogCacheKey = (directory: string | null): string => {
|
||||
return directory?.trim() || DEFAULT_SKILLS_CATALOG_CACHE_KEY;
|
||||
@@ -71,13 +89,10 @@ export interface SkillsCatalogState {
|
||||
sources: SkillsCatalogSource[];
|
||||
itemsBySource: Record<string, SkillsCatalogItem[]>;
|
||||
selectedSourceId: string | null;
|
||||
pageInfoBySource: Record<string, { nextCursor?: string | null }>;
|
||||
loadedSourceIds: Record<string, boolean>;
|
||||
clawdhubHasMoreBySource: Record<string, boolean>;
|
||||
|
||||
isLoadingCatalog: boolean;
|
||||
isLoadingSource: boolean;
|
||||
isLoadingMore: boolean;
|
||||
isScanning: boolean;
|
||||
isInstalling: boolean;
|
||||
|
||||
@@ -91,7 +106,6 @@ export interface SkillsCatalogState {
|
||||
|
||||
loadCatalog: (options?: { refresh?: boolean }) => Promise<boolean>;
|
||||
loadSource: (sourceId: string, options?: { refresh?: boolean }) => Promise<boolean>;
|
||||
loadMoreClawdHub: () => Promise<boolean>;
|
||||
scanRepo: (request: SkillsRepoScanRequest) => Promise<SkillsRepoScanResponse>;
|
||||
installSkills: (request: SkillsInstallRequest, options?: { directory?: string | null }) => Promise<SkillsInstallResponse>;
|
||||
}
|
||||
@@ -102,13 +116,10 @@ export const useSkillsCatalogStore = create<SkillsCatalogState>()(
|
||||
sources: FALLBACK_SOURCES,
|
||||
itemsBySource: {},
|
||||
selectedSourceId: FALLBACK_SOURCES[0]?.id ?? null,
|
||||
pageInfoBySource: {},
|
||||
loadedSourceIds: {},
|
||||
clawdhubHasMoreBySource: {},
|
||||
|
||||
isLoadingCatalog: false,
|
||||
isLoadingSource: false,
|
||||
isLoadingMore: false,
|
||||
isScanning: false,
|
||||
isInstalling: false,
|
||||
|
||||
@@ -141,9 +152,7 @@ export const useSkillsCatalogStore = create<SkillsCatalogState>()(
|
||||
const previous = {
|
||||
sources: get().sources,
|
||||
itemsBySource: get().itemsBySource,
|
||||
pageInfoBySource: get().pageInfoBySource,
|
||||
loadedSourceIds: get().loadedSourceIds,
|
||||
clawdhubHasMoreBySource: get().clawdhubHasMoreBySource,
|
||||
};
|
||||
|
||||
let lastError: SkillsCatalogResponse['error'] | null = null;
|
||||
@@ -168,9 +177,7 @@ export const useSkillsCatalogStore = create<SkillsCatalogState>()(
|
||||
|
||||
const sources = (payload.sources && payload.sources.length > 0) ? payload.sources : previous.sources;
|
||||
const itemsBySource = options?.refresh ? {} : (get().itemsBySource || {});
|
||||
const pageInfoBySource = options?.refresh ? {} : (get().pageInfoBySource || {});
|
||||
const loadedSourceIds = options?.refresh ? {} : (get().loadedSourceIds || {});
|
||||
const clawdhubHasMoreBySource = options?.refresh ? {} : (get().clawdhubHasMoreBySource || {});
|
||||
const currentSelected = get().selectedSourceId;
|
||||
const selectedSourceId =
|
||||
(currentSelected && sources.some((s) => s.id === currentSelected))
|
||||
@@ -180,9 +187,7 @@ export const useSkillsCatalogStore = create<SkillsCatalogState>()(
|
||||
set({
|
||||
sources,
|
||||
itemsBySource,
|
||||
pageInfoBySource,
|
||||
loadedSourceIds,
|
||||
clawdhubHasMoreBySource,
|
||||
selectedSourceId,
|
||||
});
|
||||
|
||||
@@ -197,9 +202,7 @@ export const useSkillsCatalogStore = create<SkillsCatalogState>()(
|
||||
set({
|
||||
sources: previous.sources,
|
||||
itemsBySource: previous.itemsBySource,
|
||||
pageInfoBySource: previous.pageInfoBySource,
|
||||
loadedSourceIds: previous.loadedSourceIds,
|
||||
clawdhubHasMoreBySource: previous.clawdhubHasMoreBySource,
|
||||
lastCatalogError: lastError || { kind: 'unknown', message: 'Failed to load catalog' },
|
||||
});
|
||||
|
||||
@@ -222,136 +225,83 @@ export const useSkillsCatalogStore = create<SkillsCatalogState>()(
|
||||
return false;
|
||||
}
|
||||
|
||||
// Deduplicate concurrent loads of the same source: the background
|
||||
// loader effect can restart while a request for this source is
|
||||
// already in flight.
|
||||
if (!options?.refresh) {
|
||||
const inFlight = sourceLoadInFlight.get(sourceId);
|
||||
if (inFlight) {
|
||||
return inFlight;
|
||||
}
|
||||
}
|
||||
|
||||
activeSourceLoads += 1;
|
||||
set({ isLoadingSource: true, lastCatalogError: null });
|
||||
|
||||
try {
|
||||
const currentDirectory = getRequestDirectory();
|
||||
const refresh = options?.refresh ? '&refresh=true' : '';
|
||||
const queryParams = currentDirectory
|
||||
? `?directory=${encodeURIComponent(currentDirectory)}&sourceId=${encodeURIComponent(sourceId)}${refresh}`
|
||||
: `?sourceId=${encodeURIComponent(sourceId)}${refresh}`;
|
||||
const request = (async () => {
|
||||
try {
|
||||
const currentDirectory = getRequestDirectory();
|
||||
const refresh = options?.refresh ? '&refresh=true' : '';
|
||||
const queryParams = currentDirectory
|
||||
? `?directory=${encodeURIComponent(currentDirectory)}&sourceId=${encodeURIComponent(sourceId)}${refresh}`
|
||||
: `?sourceId=${encodeURIComponent(sourceId)}${refresh}`;
|
||||
|
||||
const response = await runtimeFetch(`/api/config/skills/catalog/source${queryParams}`, {
|
||||
method: 'GET',
|
||||
headers: { Accept: 'application/json' },
|
||||
});
|
||||
|
||||
const payload = (await response.json().catch(() => null)) as SkillsCatalogSourceResponse | null;
|
||||
const hasItems = Array.isArray((payload as SkillsCatalogSourceResponse | null)?.items);
|
||||
if (!response.ok || (!payload?.ok && !hasItems)) {
|
||||
const fallback = await runtimeFetch(`/api/config/skills/catalog${queryParams}`, {
|
||||
const response = await runtimeFetch(`/api/config/skills/catalog/source${queryParams}`, {
|
||||
method: 'GET',
|
||||
headers: { Accept: 'application/json' },
|
||||
});
|
||||
const fallbackPayload = (await fallback.json().catch(() => null)) as SkillsCatalogResponse | null;
|
||||
const fallbackItems = fallbackPayload?.itemsBySource?.[sourceId];
|
||||
if (fallback.ok && fallbackPayload?.ok && Array.isArray(fallbackItems)) {
|
||||
set((state) => ({
|
||||
itemsBySource: { ...state.itemsBySource, [sourceId]: fallbackItems },
|
||||
pageInfoBySource: { ...state.pageInfoBySource, [sourceId]: { nextCursor: null } },
|
||||
loadedSourceIds: { ...state.loadedSourceIds, [sourceId]: true },
|
||||
clawdhubHasMoreBySource: { ...state.clawdhubHasMoreBySource, [sourceId]: false },
|
||||
}));
|
||||
return true;
|
||||
|
||||
const payload = (await response.json().catch(() => null)) as SkillsCatalogSourceResponse | null;
|
||||
const hasItems = Array.isArray((payload as SkillsCatalogSourceResponse | null)?.items);
|
||||
if (!response.ok || (!payload?.ok && !hasItems)) {
|
||||
const fallback = await runtimeFetch(`/api/config/skills/catalog${queryParams}`, {
|
||||
method: 'GET',
|
||||
headers: { Accept: 'application/json' },
|
||||
});
|
||||
const fallbackPayload = (await fallback.json().catch(() => null)) as SkillsCatalogResponse | null;
|
||||
const fallbackItems = fallbackPayload?.itemsBySource?.[sourceId];
|
||||
if (fallback.ok && fallbackPayload?.ok && Array.isArray(fallbackItems)) {
|
||||
set((state) => ({
|
||||
itemsBySource: { ...state.itemsBySource, [sourceId]: fallbackItems },
|
||||
loadedSourceIds: { ...state.loadedSourceIds, [sourceId]: true },
|
||||
}));
|
||||
return true;
|
||||
}
|
||||
|
||||
set({
|
||||
lastCatalogError: payload?.error || { kind: 'unknown', message: `Failed to load source (${response.status})` },
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
const items = payload?.items || [];
|
||||
|
||||
set((state) => ({
|
||||
itemsBySource: { ...state.itemsBySource, [sourceId]: items },
|
||||
loadedSourceIds: { ...state.loadedSourceIds, [sourceId]: true },
|
||||
}));
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
set({
|
||||
lastCatalogError: payload?.error || { kind: 'unknown', message: `Failed to load source (${response.status})` },
|
||||
lastCatalogError: { kind: 'unknown', message: error instanceof Error ? error.message : String(error) },
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
const items = payload?.items || [];
|
||||
const nextCursor = payload?.nextCursor ?? null;
|
||||
|
||||
set((state) => ({
|
||||
itemsBySource: { ...state.itemsBySource, [sourceId]: items },
|
||||
pageInfoBySource: { ...state.pageInfoBySource, [sourceId]: { nextCursor } },
|
||||
loadedSourceIds: { ...state.loadedSourceIds, [sourceId]: true },
|
||||
clawdhubHasMoreBySource: {
|
||||
...state.clawdhubHasMoreBySource,
|
||||
[sourceId]: items.length > 0,
|
||||
},
|
||||
}));
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
set({
|
||||
lastCatalogError: { kind: 'unknown', message: error instanceof Error ? error.message : String(error) },
|
||||
});
|
||||
return false;
|
||||
} finally {
|
||||
set({ isLoadingSource: false });
|
||||
}
|
||||
},
|
||||
|
||||
loadMoreClawdHub: async () => {
|
||||
const selectedSourceId = get().selectedSourceId;
|
||||
if (!selectedSourceId) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const pageInfo = get().pageInfoBySource[selectedSourceId];
|
||||
const cursor = pageInfo?.nextCursor || null;
|
||||
|
||||
set({ isLoadingMore: true });
|
||||
try {
|
||||
const currentDirectory = getRequestDirectory();
|
||||
const parts = [`sourceId=${encodeURIComponent(selectedSourceId)}`];
|
||||
if (currentDirectory) {
|
||||
parts.push(`directory=${encodeURIComponent(currentDirectory)}`);
|
||||
}
|
||||
if (cursor) {
|
||||
parts.push(`cursor=${encodeURIComponent(cursor)}`);
|
||||
}
|
||||
const queryParams = `?${parts.join('&')}`;
|
||||
|
||||
const response = await runtimeFetch(`/api/config/skills/catalog/source${queryParams}`, {
|
||||
method: 'GET',
|
||||
headers: { Accept: 'application/json' },
|
||||
});
|
||||
|
||||
const payload = (await response.json().catch(() => null)) as SkillsCatalogSourceResponse | null;
|
||||
if (!response.ok || !payload?.ok) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const nextCursor = payload.nextCursor ?? null;
|
||||
const currentItems = get().itemsBySource[selectedSourceId] || [];
|
||||
const items = payload.items || [];
|
||||
const merged = new Map(currentItems.map((item) => [`${item.sourceId}:${item.skillDir}`, item]));
|
||||
let newCount = 0;
|
||||
|
||||
for (const item of items) {
|
||||
const key = `${item.sourceId}:${item.skillDir}`;
|
||||
if (!merged.has(key)) {
|
||||
newCount += 1;
|
||||
} finally {
|
||||
activeSourceLoads -= 1;
|
||||
if (activeSourceLoads === 0) {
|
||||
set({ isLoadingSource: false });
|
||||
}
|
||||
merged.set(key, item);
|
||||
}
|
||||
})();
|
||||
|
||||
const noMore = items.length === 0 || newCount === 0;
|
||||
|
||||
set((state) => ({
|
||||
itemsBySource: {
|
||||
...state.itemsBySource,
|
||||
[selectedSourceId]: Array.from(merged.values()),
|
||||
},
|
||||
pageInfoBySource: {
|
||||
...state.pageInfoBySource,
|
||||
[selectedSourceId]: { nextCursor },
|
||||
},
|
||||
clawdhubHasMoreBySource: {
|
||||
...state.clawdhubHasMoreBySource,
|
||||
[selectedSourceId]: !noMore,
|
||||
},
|
||||
}));
|
||||
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
sourceLoadInFlight.set(sourceId, request);
|
||||
try {
|
||||
return await request;
|
||||
} finally {
|
||||
set({ isLoadingMore: false });
|
||||
if (sourceLoadInFlight.get(sourceId) === request) {
|
||||
sourceLoadInFlight.delete(sourceId);
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
Reference in New Issue
Block a user