tui: parallelize skill downloads for faster loading

Refactored skill discovery to download skills in parallel instead of sequentially,

reducing load times when multiple skills need to be fetched from remote URLs.

Also updated AGENTS.md with guidance on using dev branch for diffs.
This commit is contained in:
Dax Raad
2026-02-06 00:36:08 -05:00
parent 266de27a0b
commit c35bd39829
3 changed files with 67 additions and 49 deletions

View File

@@ -1,6 +1,7 @@
- To regenerate the JavaScript SDK, run `./packages/sdk/js/script/build.ts`. - To regenerate the JavaScript SDK, run `./packages/sdk/js/script/build.ts`.
- ALWAYS USE PARALLEL TOOLS WHEN APPLICABLE. - ALWAYS USE PARALLEL TOOLS WHEN APPLICABLE.
- The default branch in this repo is `dev`. - The default branch in this repo is `dev`.
- Local `main` ref may not exist; use `dev` or `origin/dev` for diffs.
- Prefer automation: execute requested actions without confirmation unless blocked by missing info or safety/irreversibility. - Prefer automation: execute requested actions without confirmation unless blocked by missing info or safety/irreversibility.
## Style Guide ## Style Guide

View File

@@ -1,7 +1,7 @@
import path from "path" import path from "path"
import { mkdir } from "fs/promises" import { mkdir } from "fs/promises"
import { Log } from "../util/log" import { Log } from "../util/log"
import { Global } from "@/global" import { Global } from "../global"
export namespace Discovery { export namespace Discovery {
const log = Log.create({ service: "skill-discovery" }) const log = Log.create({ service: "skill-discovery" })
@@ -20,60 +20,77 @@ export namespace Discovery {
async function get(url: string, dest: string): Promise<boolean> { async function get(url: string, dest: string): Promise<boolean> {
if (await Bun.file(dest).exists()) return true if (await Bun.file(dest).exists()) return true
try { return fetch(url)
const response = await fetch(url) .then(async (response) => {
if (!response.ok) { if (!response.ok) {
log.error("failed to download", { url, status: response.status }) log.error("failed to download", { url, status: response.status })
return false return false
} }
const content = await response.text() await Bun.write(dest, await response.text())
await Bun.write(dest, content)
return true return true
} catch (err) { })
.catch((err) => {
log.error("failed to download", { url, err }) log.error("failed to download", { url, err })
return false return false
} })
} }
export async function pull(url: string): Promise<string[]> { export async function pull(url: string): Promise<string[]> {
const result: string[] = [] const result: string[] = []
const indexUrl = new URL("index.json", url.endsWith("/") ? url : `${url}/`).href const base = url.endsWith("/") ? url : `${url}/`
const cacheDir = dir() const index = new URL("index.json", base).href
const cache = dir()
const host = base.slice(0, -1)
try { log.info("fetching index", { url: index })
log.info("fetching index", { url: indexUrl }) const data = await fetch(index)
const response = await fetch(indexUrl) .then(async (response) => {
if (!response.ok) { if (!response.ok) {
log.error("failed to fetch index", { url: indexUrl, status: response.status }) log.error("failed to fetch index", { url: index, status: response.status })
return undefined
}
return response
.json()
.then((json) => json as Index)
.catch((err) => {
log.error("failed to parse index", { url: index, err })
return undefined
})
})
.catch((err) => {
log.error("failed to fetch index", { url: index, err })
return undefined
})
if (!data?.skills || !Array.isArray(data.skills)) {
log.warn("invalid index format", { url: index })
return result return result
} }
const index = (await response.json()) as Index const list = data.skills.filter((skill) => {
if (!index.skills || !Array.isArray(index.skills)) { if (!skill?.name || !Array.isArray(skill.files)) {
log.warn("invalid index format", { url: indexUrl }) log.warn("invalid skill entry", { url: index, skill })
return result return false
} }
return true
})
for (const skill of index.skills) { await Promise.all(
if (!skill.name || !skill.files || !Array.isArray(skill.files)) { list.map(async (skill) => {
log.warn("invalid skill entry", { url: indexUrl, skill }) const root = path.join(cache, skill.name)
continue await Promise.all(
} skill.files.map(async (file) => {
const link = new URL(file, `${host}/${skill.name}/`).href
const dest = path.join(root, file)
await mkdir(path.dirname(dest), { recursive: true })
await get(link, dest)
}),
)
const skillDir = path.join(cacheDir, skill.name) const md = path.join(root, "SKILL.md")
for (const file of skill.files) { if (await Bun.file(md).exists()) result.push(root)
const fileUrl = new URL(file, `${url.replace(/\/$/, "")}/${skill.name}/`).href }),
const localPath = path.join(skillDir, file) )
await mkdir(path.dirname(localPath), { recursive: true })
await get(fileUrl, localPath)
}
const skillMd = path.join(skillDir, "SKILL.md")
if (await Bun.file(skillMd).exists()) result.push(skillDir)
}
} catch (err) {
log.error("failed to fetch from URL", { url, err })
}
return result return result
} }

View File

@@ -153,9 +153,9 @@ export namespace Skill {
} }
// Download and load skills from URLs // Download and load skills from URLs
for (const skillUrl of config.skills?.urls ?? []) { for (const url of config.skills?.urls ?? []) {
const downloadedDirs = await Discovery.pull(skillUrl) const list = await Discovery.pull(url)
for (const dir of downloadedDirs) { for (const dir of list) {
dirs.add(dir) dirs.add(dir)
for await (const match of SKILL_GLOB.scan({ for await (const match of SKILL_GLOB.scan({
cwd: dir, cwd: dir,