diff --git a/cli/src/__tests__/company-import-export-e2e.test.ts b/cli/src/__tests__/company-import-export-e2e.test.ts index 9c141a13..56171c71 100644 --- a/cli/src/__tests__/company-import-export-e2e.test.ts +++ b/cli/src/__tests__/company-import-export-e2e.test.ts @@ -1,5 +1,5 @@ import { execFile, spawn } from "node:child_process"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import net from "node:net"; import os from "node:os"; import path from "node:path"; @@ -182,6 +182,107 @@ function createCliEnv() { return env; } +function writeUint16(target: Uint8Array, offset: number, value: number) { + target[offset] = value & 0xff; + target[offset + 1] = (value >>> 8) & 0xff; +} + +function writeUint32(target: Uint8Array, offset: number, value: number) { + target[offset] = value & 0xff; + target[offset + 1] = (value >>> 8) & 0xff; + target[offset + 2] = (value >>> 16) & 0xff; + target[offset + 3] = (value >>> 24) & 0xff; +} + +function crc32(bytes: Uint8Array) { + let crc = 0xffffffff; + for (const byte of bytes) { + crc ^= byte; + for (let bit = 0; bit < 8; bit += 1) { + crc = (crc & 1) === 1 ? (crc >>> 1) ^ 0xedb88320 : crc >>> 1; + } + } + return (crc ^ 0xffffffff) >>> 0; +} + +function collectTextFiles(root: string, current: string, files: Record) { + for (const entry of readdirSync(current, { withFileTypes: true })) { + const absolutePath = path.join(current, entry.name); + if (entry.isDirectory()) { + collectTextFiles(root, absolutePath, files); + continue; + } + if (!entry.isFile()) continue; + const relativePath = path.relative(root, absolutePath).replace(/\\/g, "/"); + files[relativePath] = readFileSync(absolutePath, "utf8"); + } +} + +function createStoredZipArchive(files: Record, rootPath: string) { + const encoder = new TextEncoder(); + const localChunks: Uint8Array[] = []; + const centralChunks: Uint8Array[] = []; + let localOffset = 0; + let entryCount = 0; + + for (const [relativePath, content] of Object.entries(files).sort(([left], [right]) => left.localeCompare(right))) { + const fileName = encoder.encode(`${rootPath}/${relativePath}`); + const body = encoder.encode(content); + const checksum = crc32(body); + + const localHeader = new Uint8Array(30 + fileName.length); + writeUint32(localHeader, 0, 0x04034b50); + writeUint16(localHeader, 4, 20); + writeUint16(localHeader, 6, 0x0800); + writeUint16(localHeader, 8, 0); + writeUint32(localHeader, 14, checksum); + writeUint32(localHeader, 18, body.length); + writeUint32(localHeader, 22, body.length); + writeUint16(localHeader, 26, fileName.length); + localHeader.set(fileName, 30); + + const centralHeader = new Uint8Array(46 + fileName.length); + writeUint32(centralHeader, 0, 0x02014b50); + writeUint16(centralHeader, 4, 20); + writeUint16(centralHeader, 6, 20); + writeUint16(centralHeader, 8, 0x0800); + writeUint16(centralHeader, 10, 0); + writeUint32(centralHeader, 16, checksum); + writeUint32(centralHeader, 20, body.length); + writeUint32(centralHeader, 24, body.length); + writeUint16(centralHeader, 28, fileName.length); + writeUint32(centralHeader, 42, localOffset); + centralHeader.set(fileName, 46); + + localChunks.push(localHeader, body); + centralChunks.push(centralHeader); + localOffset += localHeader.length + body.length; + entryCount += 1; + } + + const centralDirectoryLength = centralChunks.reduce((sum, chunk) => sum + chunk.length, 0); + const archive = new Uint8Array( + localChunks.reduce((sum, chunk) => sum + chunk.length, 0) + centralDirectoryLength + 22, + ); + let offset = 0; + for (const chunk of localChunks) { + archive.set(chunk, offset); + offset += chunk.length; + } + const centralDirectoryOffset = offset; + for (const chunk of centralChunks) { + archive.set(chunk, offset); + offset += chunk.length; + } + writeUint32(archive, offset, 0x06054b50); + writeUint16(archive, offset + 8, entryCount); + writeUint16(archive, offset + 10, entryCount); + writeUint32(archive, offset + 12, centralDirectoryLength); + writeUint32(archive, offset + 16, centralDirectoryOffset); + + return archive; +} + async function stopServerProcess(child: ServerProcess | null) { if (!child || child.exitCode !== null) return; child.kill("SIGTERM"); @@ -345,6 +446,8 @@ describe("paperclipai company import/export e2e", () => { }, ); + const largeIssueDescription = `Round-trip the company package through the CLI.\n\n${"portable-data ".repeat(12_000)}`; + const sourceIssue = await api<{ id: string; title: string; identifier: string }>( apiBase, `/api/companies/${sourceCompany.id}/issues`, @@ -353,7 +456,7 @@ describe("paperclipai company import/export e2e", () => { headers: { "content-type": "application/json" }, body: JSON.stringify({ title: "Validate company import/export", - description: "Round-trip the company package through the CLI.", + description: largeIssueDescription, status: "todo", projectId: sourceProject.id, assigneeAgentId: sourceAgent.id, @@ -397,6 +500,7 @@ describe("paperclipai company import/export e2e", () => { `Imported ${sourceCompany.name}`, "--include", "company,agents,projects,issues", + "--yes", ], { apiBase, configPath }, ); @@ -470,6 +574,7 @@ describe("paperclipai company import/export e2e", () => { "company,agents,projects,issues", "--collision", "rename", + "--yes", ], { apiBase, configPath }, ); @@ -494,5 +599,32 @@ describe("paperclipai company import/export e2e", () => { expect(new Set(twiceImportedAgents.map((agent) => agent.name)).size).toBe(2); expect(twiceImportedProjects).toHaveLength(2); expect(twiceImportedIssues).toHaveLength(2); + + const zipPath = path.join(tempRoot, "exported-company.zip"); + const portableFiles: Record = {}; + collectTextFiles(exportDir, exportDir, portableFiles); + writeFileSync(zipPath, createStoredZipArchive(portableFiles, "paperclip-demo")); + + const importedFromZip = await runCliJson<{ + company: { id: string; name: string; action: string }; + agents: Array<{ id: string | null; action: string; name: string }>; + }>( + [ + "company", + "import", + zipPath, + "--target", + "new", + "--new-company-name", + `Zip Imported ${sourceCompany.name}`, + "--include", + "company,agents,projects,issues", + "--yes", + ], + { apiBase, configPath }, + ); + + expect(importedFromZip.company.action).toBe("created"); + expect(importedFromZip.agents.some((agent) => agent.action === "created")).toBe(true); }, 60_000); }); diff --git a/cli/src/__tests__/company-import-zip.test.ts b/cli/src/__tests__/company-import-zip.test.ts new file mode 100644 index 00000000..0c48a24a --- /dev/null +++ b/cli/src/__tests__/company-import-zip.test.ts @@ -0,0 +1,133 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { resolveInlineSourceFromPath } from "../commands/client/company.js"; + +function writeUint16(target: Uint8Array, offset: number, value: number) { + target[offset] = value & 0xff; + target[offset + 1] = (value >>> 8) & 0xff; +} + +function writeUint32(target: Uint8Array, offset: number, value: number) { + target[offset] = value & 0xff; + target[offset + 1] = (value >>> 8) & 0xff; + target[offset + 2] = (value >>> 16) & 0xff; + target[offset + 3] = (value >>> 24) & 0xff; +} + +function crc32(bytes: Uint8Array) { + let crc = 0xffffffff; + for (const byte of bytes) { + crc ^= byte; + for (let bit = 0; bit < 8; bit += 1) { + crc = (crc & 1) === 1 ? (crc >>> 1) ^ 0xedb88320 : crc >>> 1; + } + } + return (crc ^ 0xffffffff) >>> 0; +} + +function createStoredZipArchive(files: Record, rootPath: string) { + const encoder = new TextEncoder(); + const localChunks: Uint8Array[] = []; + const centralChunks: Uint8Array[] = []; + let localOffset = 0; + let entryCount = 0; + + for (const [relativePath, content] of Object.entries(files).sort(([left], [right]) => left.localeCompare(right))) { + const fileName = encoder.encode(`${rootPath}/${relativePath}`); + const body = encoder.encode(content); + const checksum = crc32(body); + + const localHeader = new Uint8Array(30 + fileName.length); + writeUint32(localHeader, 0, 0x04034b50); + writeUint16(localHeader, 4, 20); + writeUint16(localHeader, 6, 0x0800); + writeUint16(localHeader, 8, 0); + writeUint32(localHeader, 14, checksum); + writeUint32(localHeader, 18, body.length); + writeUint32(localHeader, 22, body.length); + writeUint16(localHeader, 26, fileName.length); + localHeader.set(fileName, 30); + + const centralHeader = new Uint8Array(46 + fileName.length); + writeUint32(centralHeader, 0, 0x02014b50); + writeUint16(centralHeader, 4, 20); + writeUint16(centralHeader, 6, 20); + writeUint16(centralHeader, 8, 0x0800); + writeUint16(centralHeader, 10, 0); + writeUint32(centralHeader, 16, checksum); + writeUint32(centralHeader, 20, body.length); + writeUint32(centralHeader, 24, body.length); + writeUint16(centralHeader, 28, fileName.length); + writeUint32(centralHeader, 42, localOffset); + centralHeader.set(fileName, 46); + + localChunks.push(localHeader, body); + centralChunks.push(centralHeader); + localOffset += localHeader.length + body.length; + entryCount += 1; + } + + const centralDirectoryLength = centralChunks.reduce((sum, chunk) => sum + chunk.length, 0); + const archive = new Uint8Array( + localChunks.reduce((sum, chunk) => sum + chunk.length, 0) + centralDirectoryLength + 22, + ); + + let offset = 0; + for (const chunk of localChunks) { + archive.set(chunk, offset); + offset += chunk.length; + } + const centralDirectoryOffset = offset; + for (const chunk of centralChunks) { + archive.set(chunk, offset); + offset += chunk.length; + } + + writeUint32(archive, offset, 0x06054b50); + writeUint16(archive, offset + 8, entryCount); + writeUint16(archive, offset + 10, entryCount); + writeUint32(archive, offset + 12, centralDirectoryLength); + writeUint32(archive, offset + 16, centralDirectoryOffset); + + return archive; +} + +const tempDirs: string[] = []; + +afterEach(async () => { + for (const dir of tempDirs.splice(0)) { + await rm(dir, { recursive: true, force: true }); + } +}); + +describe("resolveInlineSourceFromPath", () => { + it("imports portable files from a zip archive instead of scanning the parent directory", async () => { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-company-import-zip-")); + tempDirs.push(tempDir); + + const archivePath = path.join(tempDir, "paperclip-demo.zip"); + const archive = createStoredZipArchive( + { + "COMPANY.md": "# Company\n", + ".paperclip.yaml": "schema: paperclip/v1\n", + "agents/ceo/AGENT.md": "# CEO\n", + "notes/todo.txt": "ignore me\n", + }, + "paperclip-demo", + ); + await writeFile(archivePath, archive); + + const resolved = await resolveInlineSourceFromPath(archivePath); + + expect(resolved).toEqual({ + rootPath: "paperclip-demo", + files: { + "COMPANY.md": "# Company\n", + ".paperclip.yaml": "schema: paperclip/v1\n", + "agents/ceo/AGENT.md": "# CEO\n", + }, + }); + }); +}); diff --git a/cli/src/commands/client/company.ts b/cli/src/commands/client/company.ts index dbf52890..013f8e56 100644 --- a/cli/src/commands/client/company.ts +++ b/cli/src/commands/client/company.ts @@ -12,6 +12,7 @@ import type { CompanyPortabilityImportResult, } from "@paperclipai/shared"; import { ApiRequestError } from "../../client/http.js"; +import { readZipArchive } from "./zip.js"; import { addCommonClientOptions, formatInlineRecord, @@ -184,6 +185,14 @@ function normalizePortablePath(filePath: string): string { return filePath.replace(/\\/g, "/"); } +function shouldIncludePortableFile(filePath: string): boolean { + const baseName = path.basename(filePath); + const isMarkdown = baseName.endsWith(".md"); + const isPaperclipYaml = baseName === ".paperclip.yaml" || baseName === ".paperclip.yml"; + const contentType = binaryContentTypeByExtension[path.extname(baseName).toLowerCase()]; + return isMarkdown || isPaperclipYaml || Boolean(contentType); +} + function findPortableExtensionPath(files: Record): string | null { if (files[".paperclip.yaml"] !== undefined) return ".paperclip.yaml"; if (files[".paperclip.yml"] !== undefined) return ".paperclip.yml"; @@ -853,21 +862,29 @@ async function collectPackageFiles( continue; } if (!entry.isFile()) continue; - const isMarkdown = entry.name.endsWith(".md"); - const isPaperclipYaml = entry.name === ".paperclip.yaml" || entry.name === ".paperclip.yml"; - const contentType = binaryContentTypeByExtension[path.extname(entry.name).toLowerCase()]; - if (!isMarkdown && !isPaperclipYaml && !contentType) continue; const relativePath = path.relative(root, absolutePath).replace(/\\/g, "/"); + if (!shouldIncludePortableFile(relativePath)) continue; files[relativePath] = readPortableFileEntry(relativePath, await readFile(absolutePath)); } } -async function resolveInlineSourceFromPath(inputPath: string): Promise<{ +export async function resolveInlineSourceFromPath(inputPath: string): Promise<{ rootPath: string; files: Record; }> { const resolved = path.resolve(inputPath); const resolvedStat = await stat(resolved); + if (resolvedStat.isFile() && path.extname(resolved).toLowerCase() === ".zip") { + const archive = await readZipArchive(await readFile(resolved)); + const filteredFiles = Object.fromEntries( + Object.entries(archive.files).filter(([relativePath]) => shouldIncludePortableFile(relativePath)), + ); + return { + rootPath: archive.rootPath ?? path.basename(resolved, ".zip"), + files: filteredFiles, + }; + } + const rootDir = resolvedStat.isDirectory() ? resolved : path.dirname(resolved); const files: Record = {}; await collectPackageFiles(rootDir, rootDir, files); diff --git a/cli/src/commands/client/zip.ts b/cli/src/commands/client/zip.ts new file mode 100644 index 00000000..ff1eb669 --- /dev/null +++ b/cli/src/commands/client/zip.ts @@ -0,0 +1,129 @@ +import { inflateRawSync } from "node:zlib"; +import path from "node:path"; +import type { CompanyPortabilityFileEntry } from "@paperclipai/shared"; + +const textDecoder = new TextDecoder(); + +const binaryContentTypeByExtension: Record = { + ".gif": "image/gif", + ".jpeg": "image/jpeg", + ".jpg": "image/jpeg", + ".png": "image/png", + ".svg": "image/svg+xml", + ".webp": "image/webp", +}; + +function normalizeArchivePath(pathValue: string) { + return pathValue + .replace(/\\/g, "/") + .split("/") + .filter(Boolean) + .join("/"); +} + +function readUint16(source: Uint8Array, offset: number) { + return source[offset]! | (source[offset + 1]! << 8); +} + +function readUint32(source: Uint8Array, offset: number) { + return ( + source[offset]! | + (source[offset + 1]! << 8) | + (source[offset + 2]! << 16) | + (source[offset + 3]! << 24) + ) >>> 0; +} + +function sharedArchiveRoot(paths: string[]) { + if (paths.length === 0) return null; + const firstSegments = paths + .map((entry) => normalizeArchivePath(entry).split("/").filter(Boolean)) + .filter((parts) => parts.length > 0); + if (firstSegments.length === 0) return null; + const candidate = firstSegments[0]![0]!; + return firstSegments.every((parts) => parts.length > 1 && parts[0] === candidate) + ? candidate + : null; +} + +function bytesToPortableFileEntry(pathValue: string, bytes: Uint8Array): CompanyPortabilityFileEntry { + const contentType = binaryContentTypeByExtension[path.extname(pathValue).toLowerCase()]; + if (!contentType) return textDecoder.decode(bytes); + return { + encoding: "base64", + data: Buffer.from(bytes).toString("base64"), + contentType, + }; +} + +async function inflateZipEntry(compressionMethod: number, bytes: Uint8Array) { + if (compressionMethod === 0) return bytes; + if (compressionMethod !== 8) { + throw new Error("Unsupported zip archive: only STORE and DEFLATE entries are supported."); + } + return new Uint8Array(inflateRawSync(bytes)); +} + +export async function readZipArchive(source: ArrayBuffer | Uint8Array): Promise<{ + rootPath: string | null; + files: Record; +}> { + const bytes = source instanceof Uint8Array ? source : new Uint8Array(source); + const entries: Array<{ path: string; body: CompanyPortabilityFileEntry }> = []; + let offset = 0; + + while (offset + 4 <= bytes.length) { + const signature = readUint32(bytes, offset); + if (signature === 0x02014b50 || signature === 0x06054b50) break; + if (signature !== 0x04034b50) { + throw new Error("Invalid zip archive: unsupported local file header."); + } + + if (offset + 30 > bytes.length) { + throw new Error("Invalid zip archive: truncated local file header."); + } + + const generalPurposeFlag = readUint16(bytes, offset + 6); + const compressionMethod = readUint16(bytes, offset + 8); + const compressedSize = readUint32(bytes, offset + 18); + const fileNameLength = readUint16(bytes, offset + 26); + const extraFieldLength = readUint16(bytes, offset + 28); + + if ((generalPurposeFlag & 0x0008) !== 0) { + throw new Error("Unsupported zip archive: data descriptors are not supported."); + } + + const nameOffset = offset + 30; + const bodyOffset = nameOffset + fileNameLength + extraFieldLength; + const bodyEnd = bodyOffset + compressedSize; + if (bodyEnd > bytes.length) { + throw new Error("Invalid zip archive: truncated file contents."); + } + + const rawArchivePath = textDecoder.decode(bytes.slice(nameOffset, nameOffset + fileNameLength)); + const archivePath = normalizeArchivePath(rawArchivePath); + const isDirectoryEntry = /\/$/.test(rawArchivePath.replace(/\\/g, "/")); + if (archivePath && !isDirectoryEntry) { + const entryBytes = await inflateZipEntry(compressionMethod, bytes.slice(bodyOffset, bodyEnd)); + entries.push({ + path: archivePath, + body: bytesToPortableFileEntry(archivePath, entryBytes), + }); + } + + offset = bodyEnd; + } + + const rootPath = sharedArchiveRoot(entries.map((entry) => entry.path)); + const files: Record = {}; + for (const entry of entries) { + const normalizedPath = + rootPath && entry.path.startsWith(`${rootPath}/`) + ? entry.path.slice(rootPath.length + 1) + : entry.path; + if (!normalizedPath) continue; + files[normalizedPath] = entry.body; + } + + return { rootPath, files }; +} diff --git a/server/src/app.ts b/server/src/app.ts index 55a4e53b..5535ab3d 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -79,6 +79,8 @@ export async function createApp( const app = express(); app.use(express.json({ + // Company import/export payloads can inline full portable packages. + limit: "10mb", verify: (req, _res, buf) => { (req as unknown as { rawBody: Buffer }).rawBody = buf; },