diff --git a/cli/src/__tests__/company-import-url.test.ts b/cli/src/__tests__/company-import-url.test.ts index abc96f7d..1f1548bd 100644 --- a/cli/src/__tests__/company-import-url.test.ts +++ b/cli/src/__tests__/company-import-url.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; import { isGithubShorthand, - isGithubUrl, + looksLikeRepoUrl, isHttpUrl, normalizeGithubImportSource, } from "../commands/client/company.js"; @@ -21,17 +21,17 @@ describe("isHttpUrl", () => { }); }); -describe("isGithubUrl", () => { +describe("looksLikeRepoUrl", () => { it("matches GitHub URLs", () => { - expect(isGithubUrl("https://github.com/org/repo")).toBe(true); + expect(looksLikeRepoUrl("https://github.com/org/repo")).toBe(true); }); - it("rejects non-GitHub HTTP URLs", () => { - expect(isGithubUrl("https://example.com/foo")).toBe(false); + it("rejects URLs without owner/repo path", () => { + expect(looksLikeRepoUrl("https://example.com/foo")).toBe(false); }); it("rejects local paths", () => { - expect(isGithubUrl("/tmp/my-company")).toBe(false); + expect(looksLikeRepoUrl("/tmp/my-company")).toBe(false); }); }); diff --git a/cli/src/commands/client/company.ts b/cli/src/commands/client/company.ts index ac4fdc1c..571c1c1a 100644 --- a/cli/src/commands/client/company.ts +++ b/cli/src/commands/client/company.ts @@ -765,8 +765,15 @@ export function isHttpUrl(input: string): boolean { return /^https?:\/\//i.test(input.trim()); } -export function isGithubUrl(input: string): boolean { - return /^https?:\/\/github\.com\//i.test(input.trim()); +export function looksLikeRepoUrl(input: string): boolean { + try { + const url = new URL(input.trim()); + if (url.protocol !== "https:") return false; + const segments = url.pathname.split("/").filter(Boolean); + return segments.length >= 2; + } catch { + return false; + } } function isGithubSegment(input: string): boolean { @@ -797,13 +804,15 @@ function normalizeGithubImportPath(input: string | null | undefined): string | n } function buildGithubImportUrl(input: { + hostname?: string; owner: string; repo: string; ref?: string | null; path?: string | null; companyPath?: string | null; }): string { - const url = new URL(`https://github.com/${input.owner}/${input.repo.replace(/\.git$/i, "")}`); + const host = input.hostname || "github.com"; + const url = new URL(`https://${host}/${input.owner}/${input.repo.replace(/\.git$/i, "")}`); const ref = input.ref?.trim(); if (ref) { url.searchParams.set("ref", ref); @@ -834,14 +843,15 @@ export function normalizeGithubImportSource(input: string, refOverride?: string) }); } - if (!isGithubUrl(trimmed)) { - throw new Error("GitHub source must be a github.com URL or owner/repo[/path] shorthand."); + if (!looksLikeRepoUrl(trimmed)) { + throw new Error("GitHub source must be a GitHub or GitHub Enterprise URL, or owner/repo[/path] shorthand."); } if (!ref) { return trimmed; } const url = new URL(trimmed); + const hostname = url.hostname; const parts = url.pathname.split("/").filter(Boolean); if (parts.length < 2) { throw new Error("Invalid GitHub URL."); @@ -852,18 +862,18 @@ export function normalizeGithubImportSource(input: string, refOverride?: string) const existingPath = normalizeGithubImportPath(url.searchParams.get("path")); const existingCompanyPath = normalizeGithubImportPath(url.searchParams.get("companyPath")); if (existingCompanyPath) { - return buildGithubImportUrl({ owner, repo, ref, companyPath: existingCompanyPath }); + return buildGithubImportUrl({ hostname, owner, repo, ref, companyPath: existingCompanyPath }); } if (existingPath) { - return buildGithubImportUrl({ owner, repo, ref, path: existingPath }); + return buildGithubImportUrl({ hostname, owner, repo, ref, path: existingPath }); } if (parts[2] === "tree") { - return buildGithubImportUrl({ owner, repo, ref, path: parts.slice(4).join("/") }); + return buildGithubImportUrl({ hostname, owner, repo, ref, path: parts.slice(4).join("/") }); } if (parts[2] === "blob") { - return buildGithubImportUrl({ owner, repo, ref, companyPath: parts.slice(4).join("/") }); + return buildGithubImportUrl({ hostname, owner, repo, ref, companyPath: parts.slice(4).join("/") }); } - return buildGithubImportUrl({ owner, repo, ref }); + return buildGithubImportUrl({ hostname, owner, repo, ref }); } async function pathExists(inputPath: string): Promise { @@ -1208,13 +1218,13 @@ export function registerCompanyCommands(program: Command): void { | { type: "github"; url: string }; const treatAsLocalPath = !isHttpUrl(from) && await pathExists(from); - const isGithubSource = isGithubUrl(from) || (isGithubShorthand(from) && !treatAsLocalPath); + const isGithubSource = looksLikeRepoUrl(from) || (isGithubShorthand(from) && !treatAsLocalPath); if (isHttpUrl(from) || isGithubSource) { - if (!isGithubUrl(from) && !isGithubShorthand(from)) { + if (!looksLikeRepoUrl(from) && !isGithubShorthand(from)) { throw new Error( "Only GitHub URLs and local paths are supported for import. " + - "Generic HTTP URLs are not supported. Use a GitHub URL (https://github.com/...) or a local directory path.", + "Generic HTTP URLs are not supported. Use a GitHub or GitHub Enterprise URL (https://github.com/... or https://ghe.example.com/...) or a local directory path.", ); } sourcePayload = { type: "github", url: normalizeGithubImportSource(from, opts.ref) }; diff --git a/server/src/__tests__/company-portability.test.ts b/server/src/__tests__/company-portability.test.ts index a3410df6..7a9ff812 100644 --- a/server/src/__tests__/company-portability.test.ts +++ b/server/src/__tests__/company-portability.test.ts @@ -375,6 +375,7 @@ describe("company portability", () => { expect( parseGitHubSourceUrl("https://github.com/paperclipai/companies?ref=feature%2Fdemo&path=gstack"), ).toEqual({ + hostname: "github.com", owner: "paperclipai", repo: "companies", ref: "feature/demo", @@ -389,6 +390,7 @@ describe("company portability", () => { "https://github.com/paperclipai/companies?ref=abc123&companyPath=gstack%2FCOMPANY.md", ), ).toEqual({ + hostname: "github.com", owner: "paperclipai", repo: "companies", ref: "abc123", diff --git a/server/src/services/company-portability.ts b/server/src/services/company-portability.ts index db4be18a..b1bb7ed9 100644 --- a/server/src/services/company-portability.ts +++ b/server/src/services/company-portability.ts @@ -45,6 +45,7 @@ import { writePaperclipSkillSyncPreference, } from "@paperclipai/adapter-utils/server-utils"; import { notFound, unprocessable } from "../errors.js"; +import { ghFetch, gitHubApiBase, resolveRawGitHubUrl } from "./github-fetch.js"; import type { StorageService } from "../storage/types.js"; import { accessService } from "./access.js"; import { agentService } from "./agents.js"; @@ -2101,7 +2102,7 @@ function parseFrontmatterMarkdown(raw: string): MarkdownDoc { } async function fetchText(url: string) { - const response = await fetch(url); + const response = await ghFetch(url); if (!response.ok) { throw unprocessable(`Failed to fetch ${url}: ${response.status}`); } @@ -2109,7 +2110,7 @@ async function fetchText(url: string) { } async function fetchOptionalText(url: string) { - const response = await fetch(url); + const response = await ghFetch(url); if (response.status === 404) return null; if (!response.ok) { throw unprocessable(`Failed to fetch ${url}: ${response.status}`); @@ -2118,7 +2119,7 @@ async function fetchOptionalText(url: string) { } async function fetchBinary(url: string) { - const response = await fetch(url); + const response = await ghFetch(url); if (!response.ok) { throw unprocessable(`Failed to fetch ${url}: ${response.status}`); } @@ -2126,7 +2127,7 @@ async function fetchBinary(url: string) { } async function fetchJson(url: string): Promise { - const response = await fetch(url, { + const response = await ghFetch(url, { headers: { accept: "application/vnd.github+json", }, @@ -2411,14 +2412,16 @@ function buildManifestFromPackageFiles( const repoPath = asString(primarySource?.path); const commit = asString(primarySource?.commit); const trackingRef = asString(primarySource?.trackingRef); + const sourceHostname = asString(primarySource?.hostname) || "github.com"; const [owner, repoName] = (repo ?? "").split("/"); sourceType = "github"; sourceLocator = asString(primarySource?.url) - ?? (repo ? `https://github.com/${repo}${repoPath ? `/tree/${trackingRef ?? commit ?? "main"}/${repoPath}` : ""}` : null); + ?? (repo ? `https://${sourceHostname}/${repo}${repoPath ? `/tree/${trackingRef ?? commit ?? "main"}/${repoPath}` : ""}` : null); sourceRef = commit; normalizedMetadata = owner && repoName ? { sourceKind: "github", + ...(sourceHostname !== "github.com" ? { hostname: sourceHostname } : {}), owner, repo: repoName, ref: commit, @@ -2564,9 +2567,10 @@ function normalizeGitHubSourcePath(value: string | null | undefined) { export function parseGitHubSourceUrl(rawUrl: string) { const url = new URL(rawUrl); - if (url.hostname !== "github.com") { - throw unprocessable("GitHub source must use github.com URL"); + if (url.protocol !== "https:") { + throw unprocessable("GitHub source URL must use HTTPS"); } + const hostname = url.hostname; const parts = url.pathname.split("/").filter(Boolean); if (parts.length < 2) { throw unprocessable("Invalid GitHub URL"); @@ -2584,6 +2588,7 @@ export function parseGitHubSourceUrl(rawUrl: string) { if (basePath === ".") basePath = ""; } return { + hostname, owner, repo, ref: queryRef || "main", @@ -2607,13 +2612,9 @@ export function parseGitHubSourceUrl(rawUrl: string) { basePath = path.posix.dirname(blobPath); if (basePath === ".") basePath = ""; } - return { owner, repo, ref, basePath, companyPath }; + return { hostname, owner, repo, ref, basePath, companyPath }; } -function resolveRawGitHubUrl(owner: string, repo: string, ref: string, filePath: string) { - const normalizedFilePath = filePath.replace(/^\/+/, ""); - return `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${normalizedFilePath}`; -} export function companyPortabilityService(db: Db, storage?: StorageService) { const companies = companyService(db); @@ -2641,14 +2642,14 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { let companyMarkdown: string | null = null; try { companyMarkdown = await fetchOptionalText( - resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, companyRelativePath), + resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, companyRelativePath), ); } catch (err) { if (ref === "main") { ref = "master"; warnings.push("GitHub ref main not found; falling back to master."); companyMarkdown = await fetchOptionalText( - resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, companyRelativePath), + resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, companyRelativePath), ); } else { throw err; @@ -2664,8 +2665,9 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { const files: Record = { [companyPath]: companyMarkdown, }; + const apiBase = gitHubApiBase(parsed.hostname); const tree = await fetchJson<{ tree?: Array<{ path: string; type: string }> }>( - `https://api.github.com/repos/${parsed.owner}/${parsed.repo}/git/trees/${ref}?recursive=1`, + `${apiBase}/repos/${parsed.owner}/${parsed.repo}/git/trees/${ref}?recursive=1`, ).catch(() => ({ tree: [] })); const basePrefix = parsed.basePath ? `${parsed.basePath.replace(/^\/+|\/+$/g, "")}/` : ""; const candidatePaths = (tree.tree ?? []) @@ -2686,7 +2688,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { const relativePath = basePrefix ? repoPath.slice(basePrefix.length) : repoPath; if (files[relativePath] !== undefined) continue; files[normalizePortablePath(relativePath)] = await fetchText( - resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, repoPath), + resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, repoPath), ); } const companyDoc = parseFrontmatterMarkdown(companyMarkdown); @@ -2697,7 +2699,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { if (files[relativePath] !== undefined) continue; if (!(repoPath.endsWith(".md") || repoPath.endsWith(".yaml") || repoPath.endsWith(".yml"))) continue; files[relativePath] = await fetchText( - resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, repoPath), + resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, repoPath), ); } @@ -2707,7 +2709,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { const repoPath = [parsed.basePath, companyLogoPath].filter(Boolean).join("/"); try { const binary = await fetchBinary( - resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, repoPath), + resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, repoPath), ); resolved.files[companyLogoPath] = bufferToPortableBinaryFile(binary, inferContentTypeFromPath(companyLogoPath)); } catch (err) { diff --git a/server/src/services/company-skills.ts b/server/src/services/company-skills.ts index 2b97da20..fae77e5f 100644 --- a/server/src/services/company-skills.ts +++ b/server/src/services/company-skills.ts @@ -30,6 +30,7 @@ import { normalizeAgentUrlKey } from "@paperclipai/shared"; import { findServerAdapter } from "../adapters/index.js"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; import { notFound, unprocessable } from "../errors.js"; +import { ghFetch, gitHubApiBase, resolveRawGitHubUrl } from "./github-fetch.js"; import { agentService } from "./agents.js"; import { projectService } from "./projects.js"; import { secretService } from "./secrets.js"; @@ -73,6 +74,7 @@ type ParsedSkillImportSource = { type SkillSourceMeta = { skillKey?: string; sourceKind?: string; + hostname?: string; owner?: string; repo?: string; ref?: string; @@ -470,7 +472,7 @@ function parseFrontmatterMarkdown(raw: string): { frontmatter: Record(url: string): Promise { - const response = await fetch(url, { + const response = await ghFetch(url, { headers: { accept: "application/vnd.github+json", }, @@ -489,16 +491,17 @@ async function fetchJson(url: string): Promise { return response.json() as Promise; } -async function resolveGitHubDefaultBranch(owner: string, repo: string) { + +async function resolveGitHubDefaultBranch(owner: string, repo: string, apiBase: string) { const response = await fetchJson<{ default_branch?: string }>( - `https://api.github.com/repos/${owner}/${repo}`, + `${apiBase}/repos/${owner}/${repo}`, ); return asString(response.default_branch) ?? "main"; } -async function resolveGitHubCommitSha(owner: string, repo: string, ref: string) { +async function resolveGitHubCommitSha(owner: string, repo: string, ref: string, apiBase: string) { const response = await fetchJson<{ sha?: string }>( - `https://api.github.com/repos/${owner}/${repo}/commits/${encodeURIComponent(ref)}`, + `${apiBase}/repos/${owner}/${repo}/commits/${encodeURIComponent(ref)}`, ); const sha = asString(response.sha); if (!sha) { @@ -509,8 +512,8 @@ async function resolveGitHubCommitSha(owner: string, repo: string, ref: string) function parseGitHubSourceUrl(rawUrl: string) { const url = new URL(rawUrl); - if (url.hostname !== "github.com") { - throw unprocessable("GitHub source must use github.com URL"); + if (url.protocol !== "https:") { + throw unprocessable("GitHub source URL must use HTTPS"); } const parts = url.pathname.split("/").filter(Boolean); if (parts.length < 2) { @@ -532,10 +535,11 @@ function parseGitHubSourceUrl(rawUrl: string) { basePath = filePath ? path.posix.dirname(filePath) : ""; explicitRef = true; } - return { owner, repo, ref, basePath, filePath, explicitRef }; + return { hostname: url.hostname, owner, repo, ref, basePath, filePath, explicitRef }; } async function resolveGitHubPinnedRef(parsed: ReturnType) { + const apiBase = gitHubApiBase(parsed.hostname); if (/^[0-9a-f]{40}$/i.test(parsed.ref.trim())) { return { pinnedRef: parsed.ref, @@ -545,14 +549,11 @@ async function resolveGitHubPinnedRef(parsed: ReturnType { const url = sourceUrl.trim(); const warnings: string[] = []; - if (url.includes("github.com/")) { + const looksLikeRepoUrl = (() => { try { + const parsed = new URL(url); + if (parsed.protocol !== "https:") return false; + const h = parsed.hostname.toLowerCase(); + if (h.endsWith(".githubusercontent.com") || h === "gist.github.com") return false; + const segments = parsed.pathname.split("/").filter(Boolean); + return segments.length >= 2 && !parsed.pathname.endsWith(".md"); + } catch { return false; } })(); + if (looksLikeRepoUrl) { const parsed = parseGitHubSourceUrl(url); + const apiBase = gitHubApiBase(parsed.hostname); const { pinnedRef, trackingRef } = await resolveGitHubPinnedRef(parsed); let ref = pinnedRef; const tree = await fetchJson<{ tree?: Array<{ path: string; type: string }> }>( - `https://api.github.com/repos/${parsed.owner}/${parsed.repo}/git/trees/${ref}?recursive=1`, + `${apiBase}/repos/${parsed.owner}/${parsed.repo}/git/trees/${ref}?recursive=1`, ).catch(() => { throw unprocessable(`Failed to read GitHub tree for ${url}`); }); @@ -1013,7 +1025,7 @@ async function readUrlSkillImports( const skills: ImportedSkill[] = []; for (const relativeSkillPath of skillPaths) { const repoSkillPath = basePrefix ? `${basePrefix}${relativeSkillPath}` : relativeSkillPath; - const markdown = await fetchText(resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, repoSkillPath)); + const markdown = await fetchText(resolveRawGitHubUrl(parsed.hostname, parsed.owner, parsed.repo, ref, repoSkillPath)); const parsedMarkdown = parseFrontmatterMarkdown(markdown); const skillDir = path.posix.dirname(relativeSkillPath); const slug = deriveImportedSkillSlug(parsedMarkdown.frontmatter, path.posix.basename(skillDir)); @@ -1027,9 +1039,10 @@ async function readUrlSkillImports( const metadata = { ...(skillKey ? { skillKey } : {}), sourceKind: "github", + ...(parsed.hostname !== "github.com" ? { hostname: parsed.hostname } : {}), owner: parsed.owner, repo: parsed.repo, - ref: ref, + ref, trackingRef, repoSkillDir: normalizeGitHubSkillDirectory( basePrefix ? `${basePrefix}${skillDir}` : skillDir, @@ -1641,7 +1654,9 @@ export function companySkillService(db: Db) { }; } - const latestRef = await resolveGitHubCommitSha(owner, repo, trackingRef); + const hostname = asString(metadata.hostname) || "github.com"; + const apiBase = gitHubApiBase(hostname); + const latestRef = await resolveGitHubCommitSha(owner, repo, trackingRef, apiBase); return { supported: true, reason: null, @@ -1679,13 +1694,14 @@ export function companySkillService(db: Db) { const metadata = getSkillMeta(skill); const owner = asString(metadata.owner); const repo = asString(metadata.repo); + const hostname = asString(metadata.hostname) || "github.com"; const ref = skill.sourceRef ?? asString(metadata.ref) ?? "main"; const repoSkillDir = normalizeGitHubSkillDirectory(asString(metadata.repoSkillDir), skill.slug); if (!owner || !repo) { throw unprocessable("Skill source metadata is incomplete."); } const repoPath = normalizePortablePath(path.posix.join(repoSkillDir, normalizedPath)); - content = await fetchText(resolveRawGitHubUrl(owner, repo, ref, repoPath)); + content = await fetchText(resolveRawGitHubUrl(hostname, owner, repo, ref, repoPath)); } else if (skill.sourceType === "url") { if (normalizedPath !== "SKILL.md") { throw notFound("This skill source only exposes SKILL.md"); diff --git a/server/src/services/github-fetch.ts b/server/src/services/github-fetch.ts new file mode 100644 index 00000000..787ae0ef --- /dev/null +++ b/server/src/services/github-fetch.ts @@ -0,0 +1,25 @@ +import { unprocessable } from "../errors.js"; + +function isGitHubDotCom(hostname: string) { + const h = hostname.toLowerCase(); + return h === "github.com" || h === "www.github.com"; +} + +export function gitHubApiBase(hostname: string) { + return isGitHubDotCom(hostname) ? "https://api.github.com" : `https://${hostname}/api/v3`; +} + +export function resolveRawGitHubUrl(hostname: string, owner: string, repo: string, ref: string, filePath: string) { + const p = filePath.replace(/^\/+/, ""); + return isGitHubDotCom(hostname) + ? `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${p}` + : `https://${hostname}/raw/${owner}/${repo}/${ref}/${p}`; +} + +export async function ghFetch(url: string, init?: RequestInit): Promise { + try { + return await fetch(url, init); + } catch { + throw unprocessable(`Could not connect to ${new URL(url).hostname} — ensure the URL points to a GitHub or GitHub Enterprise instance`); + } +} diff --git a/ui/src/components/NewProjectDialog.tsx b/ui/src/components/NewProjectDialog.tsx index afdb057a..367384ac 100644 --- a/ui/src/components/NewProjectDialog.tsx +++ b/ui/src/components/NewProjectDialog.tsx @@ -118,11 +118,10 @@ export function NewProjectDialog() { const isAbsolutePath = (value: string) => value.startsWith("/") || /^[A-Za-z]:[\\/]/.test(value); - const isGitHubRepoUrl = (value: string) => { + const looksLikeRepoUrl = (value: string) => { try { const parsed = new URL(value); - const host = parsed.hostname.toLowerCase(); - if (host !== "github.com" && host !== "www.github.com") return false; + if (parsed.protocol !== "https:") return false; const segments = parsed.pathname.split("/").filter(Boolean); return segments.length >= 2; } catch { @@ -156,8 +155,8 @@ export function NewProjectDialog() { setWorkspaceError("Local folder must be a full absolute path."); return; } - if (repoUrl && !isGitHubRepoUrl(repoUrl)) { - setWorkspaceError("Repo must use a valid GitHub repo URL."); + if (repoUrl && !looksLikeRepoUrl(repoUrl)) { + setWorkspaceError("Repo must use a valid GitHub or GitHub Enterprise repo URL."); return; } diff --git a/ui/src/components/ProjectProperties.tsx b/ui/src/components/ProjectProperties.tsx index 36caf20f..0e0ea33e 100644 --- a/ui/src/components/ProjectProperties.tsx +++ b/ui/src/components/ProjectProperties.tsx @@ -343,11 +343,10 @@ export function ProjectProperties({ project, onUpdate, onFieldUpdate, getFieldSa const isAbsolutePath = (value: string) => value.startsWith("/") || /^[A-Za-z]:[\\/]/.test(value); - const isGitHubRepoUrl = (value: string) => { + const looksLikeRepoUrl = (value: string) => { try { const parsed = new URL(value); - const host = parsed.hostname.toLowerCase(); - if (host !== "github.com" && host !== "www.github.com") return false; + if (parsed.protocol !== "https:") return false; const segments = parsed.pathname.split("/").filter(Boolean); return segments.length >= 2; } catch { @@ -432,8 +431,8 @@ export function ProjectProperties({ project, onUpdate, onFieldUpdate, getFieldSa persistCodebase({ repoUrl: null }); return; } - if (!isGitHubRepoUrl(repoUrl)) { - setWorkspaceError("Repo must use a valid GitHub repo URL."); + if (!looksLikeRepoUrl(repoUrl)) { + setWorkspaceError("Repo must use a valid GitHub or GitHub Enterprise repo URL."); return; } setWorkspaceError(null);