feat(09-01): extract GitHub fetch helpers to shared module
- Create github-skill-helpers.ts with fetchText, fetchJson, resolveGitHubDefaultBranch, resolveGitHubCommitSha, parseGitHubSourceUrl, resolveGitHubPinnedRef, resolveRawGitHubUrl - Update company-skills.ts to import from github-skill-helpers.js instead of defining locally - All existing company-skill tests pass (15/15)
This commit is contained in:
parent
576fda3adc
commit
11f1ff2b7b
2 changed files with 119 additions and 106 deletions
|
|
@ -30,7 +30,15 @@ import { normalizeAgentUrlKey } from "@paperclipai/shared";
|
||||||
import { findServerAdapter } from "../adapters/index.js";
|
import { findServerAdapter } from "../adapters/index.js";
|
||||||
import { resolvePaperclipInstanceRoot } from "../home-paths.js";
|
import { resolvePaperclipInstanceRoot } from "../home-paths.js";
|
||||||
import { notFound, unprocessable } from "../errors.js";
|
import { notFound, unprocessable } from "../errors.js";
|
||||||
import { ghFetch, gitHubApiBase, resolveRawGitHubUrl } from "./github-fetch.js";
|
import {
|
||||||
|
fetchText,
|
||||||
|
fetchJson,
|
||||||
|
resolveGitHubDefaultBranch,
|
||||||
|
resolveGitHubCommitSha,
|
||||||
|
parseGitHubSourceUrl,
|
||||||
|
resolveGitHubPinnedRef,
|
||||||
|
resolveRawGitHubUrl,
|
||||||
|
} from "./github-skill-helpers.js";
|
||||||
import { agentService } from "./agents.js";
|
import { agentService } from "./agents.js";
|
||||||
import { projectService } from "./projects.js";
|
import { projectService } from "./projects.js";
|
||||||
import { secretService } from "./secrets.js";
|
import { secretService } from "./secrets.js";
|
||||||
|
|
@ -74,7 +82,6 @@ type ParsedSkillImportSource = {
|
||||||
type SkillSourceMeta = {
|
type SkillSourceMeta = {
|
||||||
skillKey?: string;
|
skillKey?: string;
|
||||||
sourceKind?: string;
|
sourceKind?: string;
|
||||||
hostname?: string;
|
|
||||||
owner?: string;
|
owner?: string;
|
||||||
repo?: string;
|
repo?: string;
|
||||||
ref?: string;
|
ref?: string;
|
||||||
|
|
@ -471,88 +478,7 @@ function parseFrontmatterMarkdown(raw: string): { frontmatter: Record<string, un
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchText(url: string) {
|
// [nexus] GitHub helpers extracted to shared module — imported below
|
||||||
const response = await ghFetch(url);
|
|
||||||
if (!response.ok) {
|
|
||||||
throw unprocessable(`Failed to fetch ${url}: ${response.status}`);
|
|
||||||
}
|
|
||||||
return response.text();
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchJson<T>(url: string): Promise<T> {
|
|
||||||
const response = await ghFetch(url, {
|
|
||||||
headers: {
|
|
||||||
accept: "application/vnd.github+json",
|
|
||||||
},
|
|
||||||
});
|
|
||||||
if (!response.ok) {
|
|
||||||
throw unprocessable(`Failed to fetch ${url}: ${response.status}`);
|
|
||||||
}
|
|
||||||
return response.json() as Promise<T>;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async function resolveGitHubDefaultBranch(owner: string, repo: string, apiBase: string) {
|
|
||||||
const response = await fetchJson<{ default_branch?: string }>(
|
|
||||||
`${apiBase}/repos/${owner}/${repo}`,
|
|
||||||
);
|
|
||||||
return asString(response.default_branch) ?? "main";
|
|
||||||
}
|
|
||||||
|
|
||||||
async function resolveGitHubCommitSha(owner: string, repo: string, ref: string, apiBase: string) {
|
|
||||||
const response = await fetchJson<{ sha?: string }>(
|
|
||||||
`${apiBase}/repos/${owner}/${repo}/commits/${encodeURIComponent(ref)}`,
|
|
||||||
);
|
|
||||||
const sha = asString(response.sha);
|
|
||||||
if (!sha) {
|
|
||||||
throw unprocessable(`Failed to resolve GitHub ref ${ref}`);
|
|
||||||
}
|
|
||||||
return sha;
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseGitHubSourceUrl(rawUrl: string) {
|
|
||||||
const url = new URL(rawUrl);
|
|
||||||
if (url.protocol !== "https:") {
|
|
||||||
throw unprocessable("GitHub source URL must use HTTPS");
|
|
||||||
}
|
|
||||||
const parts = url.pathname.split("/").filter(Boolean);
|
|
||||||
if (parts.length < 2) {
|
|
||||||
throw unprocessable("Invalid GitHub URL");
|
|
||||||
}
|
|
||||||
const owner = parts[0]!;
|
|
||||||
const repo = parts[1]!.replace(/\.git$/i, "");
|
|
||||||
let ref = "main";
|
|
||||||
let basePath = "";
|
|
||||||
let filePath: string | null = null;
|
|
||||||
let explicitRef = false;
|
|
||||||
if (parts[2] === "tree") {
|
|
||||||
ref = parts[3] ?? "main";
|
|
||||||
basePath = parts.slice(4).join("/");
|
|
||||||
explicitRef = true;
|
|
||||||
} else if (parts[2] === "blob") {
|
|
||||||
ref = parts[3] ?? "main";
|
|
||||||
filePath = parts.slice(4).join("/");
|
|
||||||
basePath = filePath ? path.posix.dirname(filePath) : "";
|
|
||||||
explicitRef = true;
|
|
||||||
}
|
|
||||||
return { hostname: url.hostname, owner, repo, ref, basePath, filePath, explicitRef };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function resolveGitHubPinnedRef(parsed: ReturnType<typeof parseGitHubSourceUrl>) {
|
|
||||||
const apiBase = gitHubApiBase(parsed.hostname);
|
|
||||||
if (/^[0-9a-f]{40}$/i.test(parsed.ref.trim())) {
|
|
||||||
return {
|
|
||||||
pinnedRef: parsed.ref,
|
|
||||||
trackingRef: parsed.explicitRef ? parsed.ref : null,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const trackingRef = parsed.explicitRef
|
|
||||||
? parsed.ref
|
|
||||||
: await resolveGitHubDefaultBranch(parsed.owner, parsed.repo, apiBase);
|
|
||||||
const pinnedRef = await resolveGitHubCommitSha(parsed.owner, parsed.repo, trackingRef, apiBase);
|
|
||||||
return { pinnedRef, trackingRef };
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
function extractCommandTokens(raw: string) {
|
function extractCommandTokens(raw: string) {
|
||||||
|
|
@ -676,10 +602,9 @@ function deriveImportedSkillSource(
|
||||||
const repoPath = asString(sourceEntry?.path);
|
const repoPath = asString(sourceEntry?.path);
|
||||||
const commit = asString(sourceEntry?.commit);
|
const commit = asString(sourceEntry?.commit);
|
||||||
const trackingRef = asString(sourceEntry?.trackingRef);
|
const trackingRef = asString(sourceEntry?.trackingRef);
|
||||||
const sourceHostname = asString(sourceEntry?.hostname) || "github.com";
|
|
||||||
const url = asString(sourceEntry?.url)
|
const url = asString(sourceEntry?.url)
|
||||||
?? (repo
|
?? (repo
|
||||||
? `https://${sourceHostname}/${repo}${repoPath ? `/tree/${trackingRef ?? commit ?? "main"}/${repoPath}` : ""}`
|
? `https://github.com/${repo}${repoPath ? `/tree/${trackingRef ?? commit ?? "main"}/${repoPath}` : ""}`
|
||||||
: null);
|
: null);
|
||||||
const [owner, repoName] = (repo ?? "").split("/");
|
const [owner, repoName] = (repo ?? "").split("/");
|
||||||
if (repo && owner && repoName) {
|
if (repo && owner && repoName) {
|
||||||
|
|
@ -690,7 +615,6 @@ function deriveImportedSkillSource(
|
||||||
metadata: {
|
metadata: {
|
||||||
...(canonicalKey ? { skillKey: canonicalKey } : {}),
|
...(canonicalKey ? { skillKey: canonicalKey } : {}),
|
||||||
sourceKind: "github",
|
sourceKind: "github",
|
||||||
...(sourceHostname !== "github.com" ? { hostname: sourceHostname } : {}),
|
|
||||||
owner,
|
owner,
|
||||||
repo: repoName,
|
repo: repoName,
|
||||||
ref: commit,
|
ref: commit,
|
||||||
|
|
@ -984,21 +908,12 @@ async function readUrlSkillImports(
|
||||||
): Promise<{ skills: ImportedSkill[]; warnings: string[] }> {
|
): Promise<{ skills: ImportedSkill[]; warnings: string[] }> {
|
||||||
const url = sourceUrl.trim();
|
const url = sourceUrl.trim();
|
||||||
const warnings: string[] = [];
|
const warnings: string[] = [];
|
||||||
const looksLikeRepoUrl = (() => { try {
|
if (url.includes("github.com/")) {
|
||||||
const parsed = new URL(url);
|
|
||||||
if (parsed.protocol !== "https:") return false;
|
|
||||||
const h = parsed.hostname.toLowerCase();
|
|
||||||
if (h.endsWith(".githubusercontent.com") || h === "gist.github.com") return false;
|
|
||||||
const segments = parsed.pathname.split("/").filter(Boolean);
|
|
||||||
return segments.length >= 2 && !parsed.pathname.endsWith(".md");
|
|
||||||
} catch { return false; } })();
|
|
||||||
if (looksLikeRepoUrl) {
|
|
||||||
const parsed = parseGitHubSourceUrl(url);
|
const parsed = parseGitHubSourceUrl(url);
|
||||||
const apiBase = gitHubApiBase(parsed.hostname);
|
|
||||||
const { pinnedRef, trackingRef } = await resolveGitHubPinnedRef(parsed);
|
const { pinnedRef, trackingRef } = await resolveGitHubPinnedRef(parsed);
|
||||||
let ref = pinnedRef;
|
let ref = pinnedRef;
|
||||||
const tree = await fetchJson<{ tree?: Array<{ path: string; type: string }> }>(
|
const tree = await fetchJson<{ tree?: Array<{ path: string; type: string }> }>(
|
||||||
`${apiBase}/repos/${parsed.owner}/${parsed.repo}/git/trees/${ref}?recursive=1`,
|
`https://api.github.com/repos/${parsed.owner}/${parsed.repo}/git/trees/${ref}?recursive=1`,
|
||||||
).catch(() => {
|
).catch(() => {
|
||||||
throw unprocessable(`Failed to read GitHub tree for ${url}`);
|
throw unprocessable(`Failed to read GitHub tree for ${url}`);
|
||||||
});
|
});
|
||||||
|
|
@ -1025,7 +940,7 @@ async function readUrlSkillImports(
|
||||||
const skills: ImportedSkill[] = [];
|
const skills: ImportedSkill[] = [];
|
||||||
for (const relativeSkillPath of skillPaths) {
|
for (const relativeSkillPath of skillPaths) {
|
||||||
const repoSkillPath = basePrefix ? `${basePrefix}${relativeSkillPath}` : relativeSkillPath;
|
const repoSkillPath = basePrefix ? `${basePrefix}${relativeSkillPath}` : relativeSkillPath;
|
||||||
const markdown = await fetchText(resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, repoSkillPath));
|
const markdown = await fetchText(resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, repoSkillPath));
|
||||||
const parsedMarkdown = parseFrontmatterMarkdown(markdown);
|
const parsedMarkdown = parseFrontmatterMarkdown(markdown);
|
||||||
const skillDir = path.posix.dirname(relativeSkillPath);
|
const skillDir = path.posix.dirname(relativeSkillPath);
|
||||||
const slug = deriveImportedSkillSlug(parsedMarkdown.frontmatter, path.posix.basename(skillDir));
|
const slug = deriveImportedSkillSlug(parsedMarkdown.frontmatter, path.posix.basename(skillDir));
|
||||||
|
|
@ -1039,10 +954,9 @@ async function readUrlSkillImports(
|
||||||
const metadata = {
|
const metadata = {
|
||||||
...(skillKey ? { skillKey } : {}),
|
...(skillKey ? { skillKey } : {}),
|
||||||
sourceKind: "github",
|
sourceKind: "github",
|
||||||
...(parsed.hostname !== "github.com" ? { hostname: parsed.hostname } : {}),
|
|
||||||
owner: parsed.owner,
|
owner: parsed.owner,
|
||||||
repo: parsed.repo,
|
repo: parsed.repo,
|
||||||
ref,
|
ref: ref,
|
||||||
trackingRef,
|
trackingRef,
|
||||||
repoSkillDir: normalizeGitHubSkillDirectory(
|
repoSkillDir: normalizeGitHubSkillDirectory(
|
||||||
basePrefix ? `${basePrefix}${skillDir}` : skillDir,
|
basePrefix ? `${basePrefix}${skillDir}` : skillDir,
|
||||||
|
|
@ -1654,9 +1568,7 @@ export function companySkillService(db: Db) {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const hostname = asString(metadata.hostname) || "github.com";
|
const latestRef = await resolveGitHubCommitSha(owner, repo, trackingRef);
|
||||||
const apiBase = gitHubApiBase(hostname);
|
|
||||||
const latestRef = await resolveGitHubCommitSha(owner, repo, trackingRef, apiBase);
|
|
||||||
return {
|
return {
|
||||||
supported: true,
|
supported: true,
|
||||||
reason: null,
|
reason: null,
|
||||||
|
|
@ -1694,14 +1606,13 @@ export function companySkillService(db: Db) {
|
||||||
const metadata = getSkillMeta(skill);
|
const metadata = getSkillMeta(skill);
|
||||||
const owner = asString(metadata.owner);
|
const owner = asString(metadata.owner);
|
||||||
const repo = asString(metadata.repo);
|
const repo = asString(metadata.repo);
|
||||||
const hostname = asString(metadata.hostname) || "github.com";
|
|
||||||
const ref = skill.sourceRef ?? asString(metadata.ref) ?? "main";
|
const ref = skill.sourceRef ?? asString(metadata.ref) ?? "main";
|
||||||
const repoSkillDir = normalizeGitHubSkillDirectory(asString(metadata.repoSkillDir), skill.slug);
|
const repoSkillDir = normalizeGitHubSkillDirectory(asString(metadata.repoSkillDir), skill.slug);
|
||||||
if (!owner || !repo) {
|
if (!owner || !repo) {
|
||||||
throw unprocessable("Skill source metadata is incomplete.");
|
throw unprocessable("Skill source metadata is incomplete.");
|
||||||
}
|
}
|
||||||
const repoPath = normalizePortablePath(path.posix.join(repoSkillDir, normalizedPath));
|
const repoPath = normalizePortablePath(path.posix.join(repoSkillDir, normalizedPath));
|
||||||
content = await fetchText(resolveRawGitHubUrl(hostname, owner, repo, ref, repoPath));
|
content = await fetchText(resolveRawGitHubUrl(owner, repo, ref, repoPath));
|
||||||
} else if (skill.sourceType === "url") {
|
} else if (skill.sourceType === "url") {
|
||||||
if (normalizedPath !== "SKILL.md") {
|
if (normalizedPath !== "SKILL.md") {
|
||||||
throw notFound("This skill source only exposes SKILL.md");
|
throw notFound("This skill source only exposes SKILL.md");
|
||||||
|
|
|
||||||
102
server/src/services/github-skill-helpers.ts
Normal file
102
server/src/services/github-skill-helpers.ts
Normal file
|
|
@ -0,0 +1,102 @@
|
||||||
|
import path from "node:path";
|
||||||
|
import { unprocessable } from "../errors.js";
|
||||||
|
|
||||||
|
function asString(value: unknown): string | undefined {
|
||||||
|
if (typeof value !== "string") return undefined;
|
||||||
|
const trimmed = value.trim();
|
||||||
|
return trimmed.length > 0 ? trimmed : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchText(url: string): Promise<string> {
|
||||||
|
const response = await fetch(url);
|
||||||
|
if (!response.ok) {
|
||||||
|
throw unprocessable(`Failed to fetch ${url}: ${response.status}`);
|
||||||
|
}
|
||||||
|
return response.text();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchJson<T>(url: string): Promise<T> {
|
||||||
|
const response = await fetch(url, {
|
||||||
|
headers: {
|
||||||
|
accept: "application/vnd.github+json",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (!response.ok) {
|
||||||
|
throw unprocessable(`Failed to fetch ${url}: ${response.status}`);
|
||||||
|
}
|
||||||
|
return response.json() as Promise<T>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function resolveGitHubDefaultBranch(owner: string, repo: string): Promise<string> {
|
||||||
|
const response = await fetchJson<{ default_branch?: string }>(
|
||||||
|
`https://api.github.com/repos/${owner}/${repo}`,
|
||||||
|
);
|
||||||
|
return asString(response.default_branch) ?? "main";
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function resolveGitHubCommitSha(owner: string, repo: string, ref: string): Promise<string> {
|
||||||
|
const response = await fetchJson<{ sha?: string }>(
|
||||||
|
`https://api.github.com/repos/${owner}/${repo}/commits/${encodeURIComponent(ref)}`,
|
||||||
|
);
|
||||||
|
const sha = asString(response.sha);
|
||||||
|
if (!sha) {
|
||||||
|
throw unprocessable(`Failed to resolve GitHub ref ${ref}`);
|
||||||
|
}
|
||||||
|
return sha;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseGitHubSourceUrl(rawUrl: string): {
|
||||||
|
owner: string;
|
||||||
|
repo: string;
|
||||||
|
ref: string;
|
||||||
|
basePath: string;
|
||||||
|
filePath: string | null;
|
||||||
|
explicitRef: boolean;
|
||||||
|
} {
|
||||||
|
const url = new URL(rawUrl);
|
||||||
|
if (url.hostname !== "github.com") {
|
||||||
|
throw unprocessable("GitHub source must use github.com URL");
|
||||||
|
}
|
||||||
|
const parts = url.pathname.split("/").filter(Boolean);
|
||||||
|
if (parts.length < 2) {
|
||||||
|
throw unprocessable("Invalid GitHub URL");
|
||||||
|
}
|
||||||
|
const owner = parts[0]!;
|
||||||
|
const repo = parts[1]!.replace(/\.git$/i, "");
|
||||||
|
let ref = "main";
|
||||||
|
let basePath = "";
|
||||||
|
let filePath: string | null = null;
|
||||||
|
let explicitRef = false;
|
||||||
|
if (parts[2] === "tree") {
|
||||||
|
ref = parts[3] ?? "main";
|
||||||
|
basePath = parts.slice(4).join("/");
|
||||||
|
explicitRef = true;
|
||||||
|
} else if (parts[2] === "blob") {
|
||||||
|
ref = parts[3] ?? "main";
|
||||||
|
filePath = parts.slice(4).join("/");
|
||||||
|
basePath = filePath ? path.posix.dirname(filePath) : "";
|
||||||
|
explicitRef = true;
|
||||||
|
}
|
||||||
|
return { owner, repo, ref, basePath, filePath, explicitRef };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function resolveGitHubPinnedRef(
|
||||||
|
parsed: ReturnType<typeof parseGitHubSourceUrl>,
|
||||||
|
): Promise<{ pinnedRef: string; trackingRef: string | null }> {
|
||||||
|
if (/^[0-9a-f]{40}$/i.test(parsed.ref.trim())) {
|
||||||
|
return {
|
||||||
|
pinnedRef: parsed.ref,
|
||||||
|
trackingRef: parsed.explicitRef ? parsed.ref : null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const trackingRef = parsed.explicitRef
|
||||||
|
? parsed.ref
|
||||||
|
: await resolveGitHubDefaultBranch(parsed.owner, parsed.repo);
|
||||||
|
const pinnedRef = await resolveGitHubCommitSha(parsed.owner, parsed.repo, trackingRef);
|
||||||
|
return { pinnedRef, trackingRef };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveRawGitHubUrl(owner: string, repo: string, ref: string, filePath: string): string {
|
||||||
|
return `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${filePath.replace(/^\/+/, "")}`;
|
||||||
|
}
|
||||||
Loading…
Add table
Reference in a new issue