Merge pull request #1834 from paperclipai/fix/project-description-mentions

fix: improve embedded Postgres bootstrap and worktree init

cli/src/commands/worktree.ts

@@ -41,6 +41,8 @@ import {
   projects,
   runDatabaseBackup,
   runDatabaseRestore,
+  createEmbeddedPostgresLogBuffer,
+  formatEmbeddedPostgresError,
 } from "@paperclipai/db";
 import type { Command } from "commander";
 import { ensureAgentJwtSecret, loadPaperclipEnvFile, mergePaperclipEnvEntries, readPaperclipEnvEntries, resolvePaperclipEnvFile } from "../config/env.js";
@@ -806,6 +808,7 @@ async function ensureEmbeddedPostgres(dataDir: string, preferredPort: number): P
   }
 
   const port = await findAvailablePort(preferredPort);
+  const logBuffer = createEmbeddedPostgresLogBuffer();
   const instance = new EmbeddedPostgres({
     databaseDir: dataDir,
     user: "paperclip",
@@ -813,17 +816,31 @@ async function ensureEmbeddedPostgres(dataDir: string, preferredPort: number): P
     port,
     persistent: true,
     initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
-    onLog: () => {},
-    onError: () => {},
+    onLog: logBuffer.append,
+    onError: logBuffer.append,
   });
 
   if (!existsSync(path.resolve(dataDir, "PG_VERSION"))) {
-    await instance.initialise();
+    try {
+      await instance.initialise();
+    } catch (error) {
+      throw formatEmbeddedPostgresError(error, {
+        fallbackMessage: `Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${port}`,
+        recentLogs: logBuffer.getRecentLogs(),
+      });
+    }
   }
   if (existsSync(postmasterPidFile)) {
     rmSync(postmasterPidFile, { force: true });
   }
-  await instance.start();
+  try {
+    await instance.start();
+  } catch (error) {
+    throw formatEmbeddedPostgresError(error, {
+      fallbackMessage: `Failed to start embedded PostgreSQL on port ${port}`,
+      recentLogs: logBuffer.getRecentLogs(),
+    });
+  }
 
   return {
     port,

doc/DEVELOPING.md

@@ -206,6 +206,17 @@ paperclipai worktree init --from-data-dir ~/.paperclip
 paperclipai worktree init --force
 ```
 
+Repair an already-created repo-managed worktree and reseed its isolated instance from the main default install:
+
+```sh
+cd ~/.paperclip/worktrees/PAP-884-ai-commits-component
+pnpm paperclipai worktree init --force --seed-mode minimal \
+  --name PAP-884-ai-commits-component \
+  --from-config ~/.paperclip/instances/default/config.json
+```
+
+That rewrites the worktree-local `.paperclip/config.json` + `.paperclip/.env`, recreates the isolated instance under `~/.paperclip-worktrees/instances/<worktree-id>/`, and preserves the git worktree contents themselves.
+
 **`pnpm paperclipai worktree:make <name> [options]`** — Create `~/NAME` as a git worktree, then initialize an isolated Paperclip instance inside it. This combines `git worktree add` with `worktree init` in a single step.
 
 | Option | Description |

packages/db/src/embedded-postgres-error.test.ts

@@ -0,0 +1,28 @@
+import { describe, expect, it } from "vitest";
+import { createEmbeddedPostgresLogBuffer, formatEmbeddedPostgresError } from "./embedded-postgres-error.js";
+
+describe("formatEmbeddedPostgresError", () => {
+  it("adds a shared-memory hint when initdb logs expose the real cause", () => {
+    const error = formatEmbeddedPostgresError("Postgres init script exited with code 1.", {
+      fallbackMessage: "Failed to initialize embedded PostgreSQL cluster",
+      recentLogs: [
+        "running bootstrap script ...",
+        "FATAL:  could not create shared memory segment: Cannot allocate memory",
+        "DETAIL:  Failed system call was shmget(key=123, size=56, 03600).",
+      ],
+    });
+
+    expect(error.message).toContain("could not allocate shared memory");
+    expect(error.message).toContain("kern.sysv.shm");
+    expect(error.message).toContain("could not create shared memory segment");
+  });
+
+  it("keeps only recent non-empty log lines in the collector", () => {
+    const buffer = createEmbeddedPostgresLogBuffer(2);
+    buffer.append("line one\n\n");
+    buffer.append("line two");
+    buffer.append("line three");
+
+    expect(buffer.getRecentLogs()).toEqual(["line two", "line three"]);
+  });
+});

packages/db/src/embedded-postgres-error.ts

@@ -0,0 +1,89 @@
+const DEFAULT_RECENT_LOG_LIMIT = 40;
+const RECENT_LOG_SUMMARY_LINES = 8;
+
+function toError(error: unknown, fallbackMessage: string): Error {
+  if (error instanceof Error) return error;
+  if (error === undefined) return new Error(fallbackMessage);
+  if (typeof error === "string") return new Error(`${fallbackMessage}: ${error}`);
+
+  try {
+    return new Error(`${fallbackMessage}: ${JSON.stringify(error)}`);
+  } catch {
+    return new Error(`${fallbackMessage}: ${String(error)}`);
+  }
+}
+
+function summarizeRecentLogs(recentLogs: string[]): string | null {
+  if (recentLogs.length === 0) return null;
+  return recentLogs
+    .slice(-RECENT_LOG_SUMMARY_LINES)
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0)
+    .join(" | ");
+}
+
+function detectEmbeddedPostgresHint(recentLogs: string[]): string | null {
+  const haystack = recentLogs.join("\n").toLowerCase();
+  if (!haystack.includes("could not create shared memory segment")) {
+    return null;
+  }
+
+  return (
+    "Embedded PostgreSQL bootstrap could not allocate shared memory. " +
+    "On macOS, this usually means the host's kern.sysv.shm* limits are too low for another local PostgreSQL cluster. " +
+    "Stop other local PostgreSQL servers or raise the shared-memory sysctls, then retry."
+  );
+}
+
+export function createEmbeddedPostgresLogBuffer(limit = DEFAULT_RECENT_LOG_LIMIT): {
+  append(message: unknown): void;
+  getRecentLogs(): string[];
+} {
+  const recentLogs: string[] = [];
+
+  return {
+    append(message: unknown) {
+      const text =
+        typeof message === "string"
+          ? message
+          : message instanceof Error
+            ? message.message
+            : String(message ?? "");
+
+      for (const rawLine of text.split(/\r?\n/)) {
+        const line = rawLine.trim();
+        if (!line) continue;
+        recentLogs.push(line);
+        if (recentLogs.length > limit) {
+          recentLogs.splice(0, recentLogs.length - limit);
+        }
+      }
+    },
+    getRecentLogs() {
+      return [...recentLogs];
+    },
+  };
+}
+
+export function formatEmbeddedPostgresError(
+  error: unknown,
+  input: {
+    fallbackMessage: string;
+    recentLogs?: string[];
+  },
+): Error {
+  const baseError = toError(error, input.fallbackMessage);
+  const recentLogs = input.recentLogs ?? [];
+  const parts = [baseError.message];
+  const hint = detectEmbeddedPostgresHint(recentLogs);
+  const recentSummary = summarizeRecentLogs(recentLogs);
+
+  if (hint) {
+    parts.push(hint);
+  }
+  if (recentSummary) {
+    parts.push(`Recent embedded Postgres logs: ${recentSummary}`);
+  }
+
+  return new Error(parts.join(" "));
+}

packages/db/src/index.ts

@@ -25,4 +25,8 @@ export {
   type RunDatabaseBackupResult,
   type RunDatabaseRestoreOptions,
 } from "./backup-lib.js";
+export {
+  createEmbeddedPostgresLogBuffer,
+  formatEmbeddedPostgresError,
+} from "./embedded-postgres-error.js";
 export * from "./schema/index.js";

packages/db/src/migration-runtime.ts

@@ -2,6 +2,7 @@ import { existsSync, readFileSync, rmSync } from "node:fs";
 import { createServer } from "node:net";
 import path from "node:path";
 import { ensurePostgresDatabase, getPostgresDataDirectory } from "./client.js";
+import { createEmbeddedPostgresLogBuffer, formatEmbeddedPostgresError } from "./embedded-postgres-error.js";
 import { resolveDatabaseTarget } from "./runtime-config.js";
 
 type EmbeddedPostgresInstance = {
@@ -27,18 +28,6 @@ export type MigrationConnection = {
   stop: () => Promise<void>;
 };
 
-function toError(error: unknown, fallbackMessage: string): Error {
-  if (error instanceof Error) return error;
-  if (error === undefined) return new Error(fallbackMessage);
-  if (typeof error === "string") return new Error(`${fallbackMessage}: ${error}`);
-
-  try {
-    return new Error(`${fallbackMessage}: ${JSON.stringify(error)}`);
-  } catch {
-    return new Error(`${fallbackMessage}: ${String(error)}`);
-  }
-}
-
 function readRunningPostmasterPid(postmasterPidFile: string): number | null {
   if (!existsSync(postmasterPidFile)) return null;
   try {
@@ -109,6 +98,7 @@ async function ensureEmbeddedPostgresConnection(
   const runningPid = readRunningPostmasterPid(postmasterPidFile);
   const runningPort = readPidFilePort(postmasterPidFile);
   const preferredAdminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${preferredPort}/postgres`;
+  const logBuffer = createEmbeddedPostgresLogBuffer();
 
   if (!runningPid && existsSync(pgVersionFile)) {
     try {
@@ -151,18 +141,19 @@ async function ensureEmbeddedPostgresConnection(
     port: selectedPort,
     persistent: true,
     initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
-    onLog: () => {},
-    onError: () => {},
+    onLog: logBuffer.append,
+    onError: logBuffer.append,
   });
 
   if (!existsSync(path.resolve(dataDir, "PG_VERSION"))) {
     try {
       await instance.initialise();
     } catch (error) {
-      throw toError(
-        error,
-        `Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${selectedPort}`,
-      );
+      throw formatEmbeddedPostgresError(error, {
+        fallbackMessage:
+          `Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${selectedPort}`,
+        recentLogs: logBuffer.getRecentLogs(),
+      });
     }
   }
   if (existsSync(postmasterPidFile)) {
@@ -171,7 +162,10 @@ async function ensureEmbeddedPostgresConnection(
   try {
     await instance.start();
   } catch (error) {
-    throw toError(error, `Failed to start embedded PostgreSQL on port ${selectedPort}`);
+    throw formatEmbeddedPostgresError(error, {
+      fallbackMessage: `Failed to start embedded PostgreSQL on port ${selectedPort}`,
+      recentLogs: logBuffer.getRecentLogs(),
+    });
   }
 
   const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${selectedPort}/postgres`;

scripts/provision-worktree.sh

@@ -32,13 +32,13 @@ source_env_path="$(dirname "$source_config_path")/.env"
 mkdir -p "$paperclip_dir"
 
 run_isolated_worktree_init() {
-  if command -v paperclipai >/dev/null 2>&1; then
-    paperclipai worktree init --force --name "$worktree_name" --from-config "$source_config_path"
+  if command -v pnpm >/dev/null 2>&1 && pnpm paperclipai --help >/dev/null 2>&1; then
+    pnpm paperclipai worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path"
     return 0
   fi
 
-  if command -v pnpm >/dev/null 2>&1 && pnpm paperclipai --help >/dev/null 2>&1; then
-    pnpm paperclipai worktree init --force --name "$worktree_name" --from-config "$source_config_path"
+  if command -v paperclipai >/dev/null 2>&1; then
+    paperclipai worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path"
     return 0
   fi
 

server/scripts/dev-watch.ts

@@ -7,7 +7,7 @@ import { resolveServerDevWatchIgnorePaths } from "../src/dev-watch-ignore.ts";
 const require = createRequire(import.meta.url);
 const tsxCliPath = require.resolve("tsx/dist/cli.mjs");
 const serverRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
-const ignoreArgs = resolveServerDevWatchIgnorePaths(serverRoot).flatMap((ignorePath) => ["--ignore", ignorePath]);
+const ignoreArgs = resolveServerDevWatchIgnorePaths(serverRoot).flatMap((ignorePath) => ["--exclude", ignorePath]);
 
 const child = spawn(
   process.execPath,

server/src/__tests__/dev-watch-ignore.test.ts

@@ -25,10 +25,18 @@ describe("resolveServerDevWatchIgnorePaths", () => {
     const ignorePaths = resolveServerDevWatchIgnorePaths(serverRoot);
 
     expect(ignorePaths).toContain(path.join(worktreeUiRoot, "node_modules"));
+    expect(ignorePaths).toContain(`${path.join(worktreeUiRoot, "node_modules").replaceAll(path.sep, "/")}/**`);
     expect(ignorePaths).toContain(fs.realpathSync(path.join(sharedUiRoot, "node_modules")));
+    expect(ignorePaths).toContain(`${fs.realpathSync(path.join(sharedUiRoot, "node_modules")).replaceAll(path.sep, "/")}/**`);
+    expect(ignorePaths).toContain(path.join(worktreeUiRoot, "node_modules", ".vite-temp"));
+    expect(ignorePaths).toContain(
+      `${path.join(worktreeUiRoot, "node_modules", ".vite-temp").replaceAll(path.sep, "/")}/**`,
+    );
     expect(ignorePaths).toContain(path.join(worktreeUiRoot, ".vite"));
     expect(ignorePaths).toContain(fs.realpathSync(path.join(sharedUiRoot, ".vite")));
     expect(ignorePaths).toContain(path.join(worktreeUiRoot, "dist"));
     expect(ignorePaths).toContain(fs.realpathSync(path.join(sharedUiRoot, "dist")));
+    expect(ignorePaths).toContain("**/{node_modules,bower_components,vendor}/**");
+    expect(ignorePaths).toContain("**/.vite-temp/**");
   });
 });

server/src/dev-watch-ignore.ts

@@ -1,19 +1,34 @@
 import fs from "node:fs";
 import path from "node:path";
 
+function toGlobstarPath(candidate: string): string {
+  return `${candidate.replaceAll(path.sep, "/")}/**`;
+}
+
 function addIgnorePath(target: Set<string>, candidate: string): void {
   target.add(candidate);
+  target.add(toGlobstarPath(candidate));
   try {
-    target.add(fs.realpathSync(candidate));
+    const realPath = fs.realpathSync(candidate);
+    target.add(realPath);
+    target.add(toGlobstarPath(realPath));
   } catch {
     // Ignore paths that do not exist in the current checkout.
   }
 }
 
 export function resolveServerDevWatchIgnorePaths(serverRoot: string): string[] {
-  const ignorePaths = new Set<string>();
+  const ignorePaths = new Set<string>([
+    "**/{node_modules,bower_components,vendor}/**",
+    "**/.vite-temp/**",
+  ]);
 
-  for (const relativePath of ["../ui/node_modules", "../ui/.vite", "../ui/dist"]) {
+  for (const relativePath of [
+    "../ui/node_modules",
+    "../ui/node_modules/.vite-temp",
+    "../ui/.vite",
+    "../ui/dist",
+  ]) {
     addIgnorePath(ignorePaths, path.resolve(serverRoot, relativePath));
   }
 

server/src/index.ts

@@ -10,9 +10,11 @@ import { and, eq } from "drizzle-orm";
 import {
   createDb,
   ensurePostgresDatabase,
+  formatEmbeddedPostgresError,
   getPostgresDataDirectory,
   inspectMigrations,
   applyPendingMigrations,
+  createEmbeddedPostgresLogBuffer,
   reconcilePendingMigrationHistory,
   formatDatabaseBackupResult,
   runDatabaseBackup,
@@ -272,29 +274,31 @@ export async function startServer(): Promise<StartedServer> {
     const dataDir = resolve(config.embeddedPostgresDataDir);
     const configuredPort = config.embeddedPostgresPort;
     let port = configuredPort;
-    const embeddedPostgresLogBuffer: string[] = [];
-    const EMBEDDED_POSTGRES_LOG_BUFFER_LIMIT = 120;
+    const logBuffer = createEmbeddedPostgresLogBuffer(120);
     const verboseEmbeddedPostgresLogs = process.env.PAPERCLIP_EMBEDDED_POSTGRES_VERBOSE === "true";
     const appendEmbeddedPostgresLog = (message: unknown) => {
-      const text = typeof message === "string" ? message : message instanceof Error ? message.message : String(message ?? "");
-      for (const lineRaw of text.split(/\r?\n/)) {
+      logBuffer.append(message);
+      if (!verboseEmbeddedPostgresLogs) {
+        return;
+      }
+      const lines = typeof message === "string"
+        ? message.split(/\r?\n/)
+        : message instanceof Error
+          ? [message.message]
+          : [String(message ?? "")];
+      for (const lineRaw of lines) {
         const line = lineRaw.trim();
         if (!line) continue;
-        embeddedPostgresLogBuffer.push(line);
-        if (embeddedPostgresLogBuffer.length > EMBEDDED_POSTGRES_LOG_BUFFER_LIMIT) {
-          embeddedPostgresLogBuffer.splice(0, embeddedPostgresLogBuffer.length - EMBEDDED_POSTGRES_LOG_BUFFER_LIMIT);
-        }
-        if (verboseEmbeddedPostgresLogs) {
-          logger.info({ embeddedPostgresLog: line }, "embedded-postgres");
-        }
+        logger.info({ embeddedPostgresLog: line }, "embedded-postgres");
       }
     };
     const logEmbeddedPostgresFailure = (phase: "initialise" | "start", err: unknown) => {
-      if (embeddedPostgresLogBuffer.length > 0) {
+      const recentLogs = logBuffer.getRecentLogs();
+      if (recentLogs.length > 0) {
         logger.error(
           {
             phase,
-            recentLogs: embeddedPostgresLogBuffer,
+            recentLogs,
             err,
           },
           "Embedded PostgreSQL failed; showing buffered startup logs",
@@ -371,7 +375,10 @@ export async function startServer(): Promise<StartedServer> {
             await embeddedPostgres.initialise();
           } catch (err) {
             logEmbeddedPostgresFailure("initialise", err);
-            throw err;
+            throw formatEmbeddedPostgresError(err, {
+              fallbackMessage: `Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${port}`,
+              recentLogs: logBuffer.getRecentLogs(),
+            });
           }
         } else {
           logger.info(`Embedded PostgreSQL cluster already exists (${clusterVersionFile}); skipping init`);
@@ -385,7 +392,10 @@ export async function startServer(): Promise<StartedServer> {
           await embeddedPostgres.start();
         } catch (err) {
           logEmbeddedPostgresFailure("start", err);
-          throw err;
+          throw formatEmbeddedPostgresError(err, {
+            fallbackMessage: `Failed to start embedded PostgreSQL on port ${port}`,
+            recentLogs: logBuffer.getRecentLogs(),
+          });
         }
         embeddedPostgresStartedByThisProcess = true;
       }