Fix company zip imports

Co-Authored-By: Paperclip <noreply@paperclip.ing>

cli/src/__tests__/company-import-export-e2e.test.ts

@@ -1,5 +1,5 @@
 import { execFile, spawn } from "node:child_process";
-import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
 import net from "node:net";
 import os from "node:os";
 import path from "node:path";
@@ -182,6 +182,107 @@ function createCliEnv() {
   return env;
 }
 
+function writeUint16(target: Uint8Array, offset: number, value: number) {
+  target[offset] = value & 0xff;
+  target[offset + 1] = (value >>> 8) & 0xff;
+}
+
+function writeUint32(target: Uint8Array, offset: number, value: number) {
+  target[offset] = value & 0xff;
+  target[offset + 1] = (value >>> 8) & 0xff;
+  target[offset + 2] = (value >>> 16) & 0xff;
+  target[offset + 3] = (value >>> 24) & 0xff;
+}
+
+function crc32(bytes: Uint8Array) {
+  let crc = 0xffffffff;
+  for (const byte of bytes) {
+    crc ^= byte;
+    for (let bit = 0; bit < 8; bit += 1) {
+      crc = (crc & 1) === 1 ? (crc >>> 1) ^ 0xedb88320 : crc >>> 1;
+    }
+  }
+  return (crc ^ 0xffffffff) >>> 0;
+}
+
+function collectTextFiles(root: string, current: string, files: Record<string, string>) {
+  for (const entry of readdirSync(current, { withFileTypes: true })) {
+    const absolutePath = path.join(current, entry.name);
+    if (entry.isDirectory()) {
+      collectTextFiles(root, absolutePath, files);
+      continue;
+    }
+    if (!entry.isFile()) continue;
+    const relativePath = path.relative(root, absolutePath).replace(/\\/g, "/");
+    files[relativePath] = readFileSync(absolutePath, "utf8");
+  }
+}
+
+function createStoredZipArchive(files: Record<string, string>, rootPath: string) {
+  const encoder = new TextEncoder();
+  const localChunks: Uint8Array[] = [];
+  const centralChunks: Uint8Array[] = [];
+  let localOffset = 0;
+  let entryCount = 0;
+
+  for (const [relativePath, content] of Object.entries(files).sort(([left], [right]) => left.localeCompare(right))) {
+    const fileName = encoder.encode(`${rootPath}/${relativePath}`);
+    const body = encoder.encode(content);
+    const checksum = crc32(body);
+
+    const localHeader = new Uint8Array(30 + fileName.length);
+    writeUint32(localHeader, 0, 0x04034b50);
+    writeUint16(localHeader, 4, 20);
+    writeUint16(localHeader, 6, 0x0800);
+    writeUint16(localHeader, 8, 0);
+    writeUint32(localHeader, 14, checksum);
+    writeUint32(localHeader, 18, body.length);
+    writeUint32(localHeader, 22, body.length);
+    writeUint16(localHeader, 26, fileName.length);
+    localHeader.set(fileName, 30);
+
+    const centralHeader = new Uint8Array(46 + fileName.length);
+    writeUint32(centralHeader, 0, 0x02014b50);
+    writeUint16(centralHeader, 4, 20);
+    writeUint16(centralHeader, 6, 20);
+    writeUint16(centralHeader, 8, 0x0800);
+    writeUint16(centralHeader, 10, 0);
+    writeUint32(centralHeader, 16, checksum);
+    writeUint32(centralHeader, 20, body.length);
+    writeUint32(centralHeader, 24, body.length);
+    writeUint16(centralHeader, 28, fileName.length);
+    writeUint32(centralHeader, 42, localOffset);
+    centralHeader.set(fileName, 46);
+
+    localChunks.push(localHeader, body);
+    centralChunks.push(centralHeader);
+    localOffset += localHeader.length + body.length;
+    entryCount += 1;
+  }
+
+  const centralDirectoryLength = centralChunks.reduce((sum, chunk) => sum + chunk.length, 0);
+  const archive = new Uint8Array(
+    localChunks.reduce((sum, chunk) => sum + chunk.length, 0) + centralDirectoryLength + 22,
+  );
+  let offset = 0;
+  for (const chunk of localChunks) {
+    archive.set(chunk, offset);
+    offset += chunk.length;
+  }
+  const centralDirectoryOffset = offset;
+  for (const chunk of centralChunks) {
+    archive.set(chunk, offset);
+    offset += chunk.length;
+  }
+  writeUint32(archive, offset, 0x06054b50);
+  writeUint16(archive, offset + 8, entryCount);
+  writeUint16(archive, offset + 10, entryCount);
+  writeUint32(archive, offset + 12, centralDirectoryLength);
+  writeUint32(archive, offset + 16, centralDirectoryOffset);
+
+  return archive;
+}
+
 async function stopServerProcess(child: ServerProcess | null) {
   if (!child || child.exitCode !== null) return;
   child.kill("SIGTERM");
@@ -345,6 +446,8 @@ describe("paperclipai company import/export e2e", () => {
       },
     );
 
+    const largeIssueDescription = `Round-trip the company package through the CLI.\n\n${"portable-data ".repeat(12_000)}`;
+
     const sourceIssue = await api<{ id: string; title: string; identifier: string }>(
       apiBase,
       `/api/companies/${sourceCompany.id}/issues`,
@@ -353,7 +456,7 @@ describe("paperclipai company import/export e2e", () => {
         headers: { "content-type": "application/json" },
         body: JSON.stringify({
           title: "Validate company import/export",
-          description: "Round-trip the company package through the CLI.",
+          description: largeIssueDescription,
           status: "todo",
           projectId: sourceProject.id,
           assigneeAgentId: sourceAgent.id,
@@ -397,6 +500,7 @@ describe("paperclipai company import/export e2e", () => {
         `Imported ${sourceCompany.name}`,
         "--include",
         "company,agents,projects,issues",
+        "--yes",
       ],
       { apiBase, configPath },
     );
@@ -470,6 +574,7 @@ describe("paperclipai company import/export e2e", () => {
         "company,agents,projects,issues",
         "--collision",
         "rename",
+        "--yes",
       ],
       { apiBase, configPath },
     );
@@ -494,5 +599,32 @@ describe("paperclipai company import/export e2e", () => {
     expect(new Set(twiceImportedAgents.map((agent) => agent.name)).size).toBe(2);
     expect(twiceImportedProjects).toHaveLength(2);
     expect(twiceImportedIssues).toHaveLength(2);
+
+    const zipPath = path.join(tempRoot, "exported-company.zip");
+    const portableFiles: Record<string, string> = {};
+    collectTextFiles(exportDir, exportDir, portableFiles);
+    writeFileSync(zipPath, createStoredZipArchive(portableFiles, "paperclip-demo"));
+
+    const importedFromZip = await runCliJson<{
+      company: { id: string; name: string; action: string };
+      agents: Array<{ id: string | null; action: string; name: string }>;
+    }>(
+      [
+        "company",
+        "import",
+        zipPath,
+        "--target",
+        "new",
+        "--new-company-name",
+        `Zip Imported ${sourceCompany.name}`,
+        "--include",
+        "company,agents,projects,issues",
+        "--yes",
+      ],
+      { apiBase, configPath },
+    );
+
+    expect(importedFromZip.company.action).toBe("created");
+    expect(importedFromZip.agents.some((agent) => agent.action === "created")).toBe(true);
   }, 60_000);
 });

cli/src/__tests__/company-import-zip.test.ts

@@ -0,0 +1,133 @@
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { resolveInlineSourceFromPath } from "../commands/client/company.js";
+
+function writeUint16(target: Uint8Array, offset: number, value: number) {
+  target[offset] = value & 0xff;
+  target[offset + 1] = (value >>> 8) & 0xff;
+}
+
+function writeUint32(target: Uint8Array, offset: number, value: number) {
+  target[offset] = value & 0xff;
+  target[offset + 1] = (value >>> 8) & 0xff;
+  target[offset + 2] = (value >>> 16) & 0xff;
+  target[offset + 3] = (value >>> 24) & 0xff;
+}
+
+function crc32(bytes: Uint8Array) {
+  let crc = 0xffffffff;
+  for (const byte of bytes) {
+    crc ^= byte;
+    for (let bit = 0; bit < 8; bit += 1) {
+      crc = (crc & 1) === 1 ? (crc >>> 1) ^ 0xedb88320 : crc >>> 1;
+    }
+  }
+  return (crc ^ 0xffffffff) >>> 0;
+}
+
+function createStoredZipArchive(files: Record<string, string>, rootPath: string) {
+  const encoder = new TextEncoder();
+  const localChunks: Uint8Array[] = [];
+  const centralChunks: Uint8Array[] = [];
+  let localOffset = 0;
+  let entryCount = 0;
+
+  for (const [relativePath, content] of Object.entries(files).sort(([left], [right]) => left.localeCompare(right))) {
+    const fileName = encoder.encode(`${rootPath}/${relativePath}`);
+    const body = encoder.encode(content);
+    const checksum = crc32(body);
+
+    const localHeader = new Uint8Array(30 + fileName.length);
+    writeUint32(localHeader, 0, 0x04034b50);
+    writeUint16(localHeader, 4, 20);
+    writeUint16(localHeader, 6, 0x0800);
+    writeUint16(localHeader, 8, 0);
+    writeUint32(localHeader, 14, checksum);
+    writeUint32(localHeader, 18, body.length);
+    writeUint32(localHeader, 22, body.length);
+    writeUint16(localHeader, 26, fileName.length);
+    localHeader.set(fileName, 30);
+
+    const centralHeader = new Uint8Array(46 + fileName.length);
+    writeUint32(centralHeader, 0, 0x02014b50);
+    writeUint16(centralHeader, 4, 20);
+    writeUint16(centralHeader, 6, 20);
+    writeUint16(centralHeader, 8, 0x0800);
+    writeUint16(centralHeader, 10, 0);
+    writeUint32(centralHeader, 16, checksum);
+    writeUint32(centralHeader, 20, body.length);
+    writeUint32(centralHeader, 24, body.length);
+    writeUint16(centralHeader, 28, fileName.length);
+    writeUint32(centralHeader, 42, localOffset);
+    centralHeader.set(fileName, 46);
+
+    localChunks.push(localHeader, body);
+    centralChunks.push(centralHeader);
+    localOffset += localHeader.length + body.length;
+    entryCount += 1;
+  }
+
+  const centralDirectoryLength = centralChunks.reduce((sum, chunk) => sum + chunk.length, 0);
+  const archive = new Uint8Array(
+    localChunks.reduce((sum, chunk) => sum + chunk.length, 0) + centralDirectoryLength + 22,
+  );
+
+  let offset = 0;
+  for (const chunk of localChunks) {
+    archive.set(chunk, offset);
+    offset += chunk.length;
+  }
+  const centralDirectoryOffset = offset;
+  for (const chunk of centralChunks) {
+    archive.set(chunk, offset);
+    offset += chunk.length;
+  }
+
+  writeUint32(archive, offset, 0x06054b50);
+  writeUint16(archive, offset + 8, entryCount);
+  writeUint16(archive, offset + 10, entryCount);
+  writeUint32(archive, offset + 12, centralDirectoryLength);
+  writeUint32(archive, offset + 16, centralDirectoryOffset);
+
+  return archive;
+}
+
+const tempDirs: string[] = [];
+
+afterEach(async () => {
+  for (const dir of tempDirs.splice(0)) {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+describe("resolveInlineSourceFromPath", () => {
+  it("imports portable files from a zip archive instead of scanning the parent directory", async () => {
+    const tempDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-company-import-zip-"));
+    tempDirs.push(tempDir);
+
+    const archivePath = path.join(tempDir, "paperclip-demo.zip");
+    const archive = createStoredZipArchive(
+      {
+        "COMPANY.md": "# Company\n",
+        ".paperclip.yaml": "schema: paperclip/v1\n",
+        "agents/ceo/AGENT.md": "# CEO\n",
+        "notes/todo.txt": "ignore me\n",
+      },
+      "paperclip-demo",
+    );
+    await writeFile(archivePath, archive);
+
+    const resolved = await resolveInlineSourceFromPath(archivePath);
+
+    expect(resolved).toEqual({
+      rootPath: "paperclip-demo",
+      files: {
+        "COMPANY.md": "# Company\n",
+        ".paperclip.yaml": "schema: paperclip/v1\n",
+        "agents/ceo/AGENT.md": "# CEO\n",
+      },
+    });
+  });
+});

cli/src/commands/client/company.ts

@@ -12,6 +12,7 @@ import type {
   CompanyPortabilityImportResult,
 } from "@paperclipai/shared";
 import { ApiRequestError } from "../../client/http.js";
+import { readZipArchive } from "./zip.js";
 import {
   addCommonClientOptions,
   formatInlineRecord,
@@ -184,6 +185,14 @@ function normalizePortablePath(filePath: string): string {
   return filePath.replace(/\\/g, "/");
 }
 
+function shouldIncludePortableFile(filePath: string): boolean {
+  const baseName = path.basename(filePath);
+  const isMarkdown = baseName.endsWith(".md");
+  const isPaperclipYaml = baseName === ".paperclip.yaml" || baseName === ".paperclip.yml";
+  const contentType = binaryContentTypeByExtension[path.extname(baseName).toLowerCase()];
+  return isMarkdown || isPaperclipYaml || Boolean(contentType);
+}
+
 function findPortableExtensionPath(files: Record<string, CompanyPortabilityFileEntry>): string | null {
   if (files[".paperclip.yaml"] !== undefined) return ".paperclip.yaml";
   if (files[".paperclip.yml"] !== undefined) return ".paperclip.yml";
@@ -853,21 +862,29 @@ async function collectPackageFiles(
       continue;
     }
     if (!entry.isFile()) continue;
-    const isMarkdown = entry.name.endsWith(".md");
-    const isPaperclipYaml = entry.name === ".paperclip.yaml" || entry.name === ".paperclip.yml";
-    const contentType = binaryContentTypeByExtension[path.extname(entry.name).toLowerCase()];
-    if (!isMarkdown && !isPaperclipYaml && !contentType) continue;
     const relativePath = path.relative(root, absolutePath).replace(/\\/g, "/");
+    if (!shouldIncludePortableFile(relativePath)) continue;
     files[relativePath] = readPortableFileEntry(relativePath, await readFile(absolutePath));
   }
 }
 
-async function resolveInlineSourceFromPath(inputPath: string): Promise<{
+export async function resolveInlineSourceFromPath(inputPath: string): Promise<{
   rootPath: string;
   files: Record<string, CompanyPortabilityFileEntry>;
 }> {
   const resolved = path.resolve(inputPath);
   const resolvedStat = await stat(resolved);
+  if (resolvedStat.isFile() && path.extname(resolved).toLowerCase() === ".zip") {
+    const archive = await readZipArchive(await readFile(resolved));
+    const filteredFiles = Object.fromEntries(
+      Object.entries(archive.files).filter(([relativePath]) => shouldIncludePortableFile(relativePath)),
+    );
+    return {
+      rootPath: archive.rootPath ?? path.basename(resolved, ".zip"),
+      files: filteredFiles,
+    };
+  }
+
   const rootDir = resolvedStat.isDirectory() ? resolved : path.dirname(resolved);
   const files: Record<string, CompanyPortabilityFileEntry> = {};
   await collectPackageFiles(rootDir, rootDir, files);

cli/src/commands/client/zip.ts

@@ -0,0 +1,129 @@
+import { inflateRawSync } from "node:zlib";
+import path from "node:path";
+import type { CompanyPortabilityFileEntry } from "@paperclipai/shared";
+
+const textDecoder = new TextDecoder();
+
+const binaryContentTypeByExtension: Record<string, string> = {
+  ".gif": "image/gif",
+  ".jpeg": "image/jpeg",
+  ".jpg": "image/jpeg",
+  ".png": "image/png",
+  ".svg": "image/svg+xml",
+  ".webp": "image/webp",
+};
+
+function normalizeArchivePath(pathValue: string) {
+  return pathValue
+    .replace(/\\/g, "/")
+    .split("/")
+    .filter(Boolean)
+    .join("/");
+}
+
+function readUint16(source: Uint8Array, offset: number) {
+  return source[offset]! | (source[offset + 1]! << 8);
+}
+
+function readUint32(source: Uint8Array, offset: number) {
+  return (
+    source[offset]! |
+    (source[offset + 1]! << 8) |
+    (source[offset + 2]! << 16) |
+    (source[offset + 3]! << 24)
+  ) >>> 0;
+}
+
+function sharedArchiveRoot(paths: string[]) {
+  if (paths.length === 0) return null;
+  const firstSegments = paths
+    .map((entry) => normalizeArchivePath(entry).split("/").filter(Boolean))
+    .filter((parts) => parts.length > 0);
+  if (firstSegments.length === 0) return null;
+  const candidate = firstSegments[0]![0]!;
+  return firstSegments.every((parts) => parts.length > 1 && parts[0] === candidate)
+    ? candidate
+    : null;
+}
+
+function bytesToPortableFileEntry(pathValue: string, bytes: Uint8Array): CompanyPortabilityFileEntry {
+  const contentType = binaryContentTypeByExtension[path.extname(pathValue).toLowerCase()];
+  if (!contentType) return textDecoder.decode(bytes);
+  return {
+    encoding: "base64",
+    data: Buffer.from(bytes).toString("base64"),
+    contentType,
+  };
+}
+
+async function inflateZipEntry(compressionMethod: number, bytes: Uint8Array) {
+  if (compressionMethod === 0) return bytes;
+  if (compressionMethod !== 8) {
+    throw new Error("Unsupported zip archive: only STORE and DEFLATE entries are supported.");
+  }
+  return new Uint8Array(inflateRawSync(bytes));
+}
+
+export async function readZipArchive(source: ArrayBuffer | Uint8Array): Promise<{
+  rootPath: string | null;
+  files: Record<string, CompanyPortabilityFileEntry>;
+}> {
+  const bytes = source instanceof Uint8Array ? source : new Uint8Array(source);
+  const entries: Array<{ path: string; body: CompanyPortabilityFileEntry }> = [];
+  let offset = 0;
+
+  while (offset + 4 <= bytes.length) {
+    const signature = readUint32(bytes, offset);
+    if (signature === 0x02014b50 || signature === 0x06054b50) break;
+    if (signature !== 0x04034b50) {
+      throw new Error("Invalid zip archive: unsupported local file header.");
+    }
+
+    if (offset + 30 > bytes.length) {
+      throw new Error("Invalid zip archive: truncated local file header.");
+    }
+
+    const generalPurposeFlag = readUint16(bytes, offset + 6);
+    const compressionMethod = readUint16(bytes, offset + 8);
+    const compressedSize = readUint32(bytes, offset + 18);
+    const fileNameLength = readUint16(bytes, offset + 26);
+    const extraFieldLength = readUint16(bytes, offset + 28);
+
+    if ((generalPurposeFlag & 0x0008) !== 0) {
+      throw new Error("Unsupported zip archive: data descriptors are not supported.");
+    }
+
+    const nameOffset = offset + 30;
+    const bodyOffset = nameOffset + fileNameLength + extraFieldLength;
+    const bodyEnd = bodyOffset + compressedSize;
+    if (bodyEnd > bytes.length) {
+      throw new Error("Invalid zip archive: truncated file contents.");
+    }
+
+    const rawArchivePath = textDecoder.decode(bytes.slice(nameOffset, nameOffset + fileNameLength));
+    const archivePath = normalizeArchivePath(rawArchivePath);
+    const isDirectoryEntry = /\/$/.test(rawArchivePath.replace(/\\/g, "/"));
+    if (archivePath && !isDirectoryEntry) {
+      const entryBytes = await inflateZipEntry(compressionMethod, bytes.slice(bodyOffset, bodyEnd));
+      entries.push({
+        path: archivePath,
+        body: bytesToPortableFileEntry(archivePath, entryBytes),
+      });
+    }
+
+    offset = bodyEnd;
+  }
+
+  const rootPath = sharedArchiveRoot(entries.map((entry) => entry.path));
+  const files: Record<string, CompanyPortabilityFileEntry> = {};
+  for (const entry of entries) {
+    const normalizedPath =
+      rootPath && entry.path.startsWith(`${rootPath}/`)
+        ? entry.path.slice(rootPath.length + 1)
+        : entry.path;
+    if (!normalizedPath) continue;
+    files[normalizedPath] = entry.body;
+  }
+
+  return { rootPath, files };
+}

server/src/app.ts

@@ -79,6 +79,8 @@ export async function createApp(
   const app = express();
 
   app.use(express.json({
+    // Company import/export payloads can inline full portable packages.
+    limit: "10mb",
     verify: (req, _res, buf) => {
       (req as unknown as { rawBody: Buffer }).rawBody = buf;
     },