Improve embedded Postgres bootstrap errors

Co-Authored-By: Paperclip <noreply@paperclip.ing>

cli/src/commands/worktree.ts

@@ -41,6 +41,8 @@ import {
   projects,
   runDatabaseBackup,
   runDatabaseRestore,
+  createEmbeddedPostgresLogBuffer,
+  formatEmbeddedPostgresError,
 } from "@paperclipai/db";
 import type { Command } from "commander";
 import { ensureAgentJwtSecret, loadPaperclipEnvFile, mergePaperclipEnvEntries, readPaperclipEnvEntries, resolvePaperclipEnvFile } from "../config/env.js";
@@ -806,6 +808,7 @@ async function ensureEmbeddedPostgres(dataDir: string, preferredPort: number): P
   }
 
   const port = await findAvailablePort(preferredPort);
+  const logBuffer = createEmbeddedPostgresLogBuffer();
   const instance = new EmbeddedPostgres({
     databaseDir: dataDir,
     user: "paperclip",
@@ -813,17 +816,31 @@ async function ensureEmbeddedPostgres(dataDir: string, preferredPort: number): P
     port,
     persistent: true,
     initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
-    onLog: () => {},
-    onError: () => {},
+    onLog: logBuffer.append,
+    onError: logBuffer.append,
   });
 
   if (!existsSync(path.resolve(dataDir, "PG_VERSION"))) {
-    await instance.initialise();
+    try {
+      await instance.initialise();
+    } catch (error) {
+      throw formatEmbeddedPostgresError(error, {
+        fallbackMessage: `Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${port}`,
+        recentLogs: logBuffer.getRecentLogs(),
+      });
+    }
   }
   if (existsSync(postmasterPidFile)) {
     rmSync(postmasterPidFile, { force: true });
   }
-  await instance.start();
+  try {
+    await instance.start();
+  } catch (error) {
+    throw formatEmbeddedPostgresError(error, {
+      fallbackMessage: `Failed to start embedded PostgreSQL on port ${port}`,
+      recentLogs: logBuffer.getRecentLogs(),
+    });
+  }
 
   return {
     port,

packages/db/src/embedded-postgres-error.test.ts

@@ -0,0 +1,28 @@
+import { describe, expect, it } from "vitest";
+import { createEmbeddedPostgresLogBuffer, formatEmbeddedPostgresError } from "./embedded-postgres-error.js";
+
+describe("formatEmbeddedPostgresError", () => {
+  it("adds a shared-memory hint when initdb logs expose the real cause", () => {
+    const error = formatEmbeddedPostgresError("Postgres init script exited with code 1.", {
+      fallbackMessage: "Failed to initialize embedded PostgreSQL cluster",
+      recentLogs: [
+        "running bootstrap script ...",
+        "FATAL:  could not create shared memory segment: Cannot allocate memory",
+        "DETAIL:  Failed system call was shmget(key=123, size=56, 03600).",
+      ],
+    });
+
+    expect(error.message).toContain("could not allocate shared memory");
+    expect(error.message).toContain("kern.sysv.shm");
+    expect(error.message).toContain("could not create shared memory segment");
+  });
+
+  it("keeps only recent non-empty log lines in the collector", () => {
+    const buffer = createEmbeddedPostgresLogBuffer(2);
+    buffer.append("line one\n\n");
+    buffer.append("line two");
+    buffer.append("line three");
+
+    expect(buffer.getRecentLogs()).toEqual(["line two", "line three"]);
+  });
+});

packages/db/src/embedded-postgres-error.ts

@@ -0,0 +1,89 @@
+const DEFAULT_RECENT_LOG_LIMIT = 40;
+const RECENT_LOG_SUMMARY_LINES = 8;
+
+function toError(error: unknown, fallbackMessage: string): Error {
+  if (error instanceof Error) return error;
+  if (error === undefined) return new Error(fallbackMessage);
+  if (typeof error === "string") return new Error(`${fallbackMessage}: ${error}`);
+
+  try {
+    return new Error(`${fallbackMessage}: ${JSON.stringify(error)}`);
+  } catch {
+    return new Error(`${fallbackMessage}: ${String(error)}`);
+  }
+}
+
+function summarizeRecentLogs(recentLogs: string[]): string | null {
+  if (recentLogs.length === 0) return null;
+  return recentLogs
+    .slice(-RECENT_LOG_SUMMARY_LINES)
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0)
+    .join(" | ");
+}
+
+function detectEmbeddedPostgresHint(recentLogs: string[]): string | null {
+  const haystack = recentLogs.join("\n").toLowerCase();
+  if (!haystack.includes("could not create shared memory segment")) {
+    return null;
+  }
+
+  return (
+    "Embedded PostgreSQL bootstrap could not allocate shared memory. " +
+    "On macOS, this usually means the host's kern.sysv.shm* limits are too low for another local PostgreSQL cluster. " +
+    "Stop other local PostgreSQL servers or raise the shared-memory sysctls, then retry."
+  );
+}
+
+export function createEmbeddedPostgresLogBuffer(limit = DEFAULT_RECENT_LOG_LIMIT): {
+  append(message: unknown): void;
+  getRecentLogs(): string[];
+} {
+  const recentLogs: string[] = [];
+
+  return {
+    append(message: unknown) {
+      const text =
+        typeof message === "string"
+          ? message
+          : message instanceof Error
+            ? message.message
+            : String(message ?? "");
+
+      for (const rawLine of text.split(/\r?\n/)) {
+        const line = rawLine.trim();
+        if (!line) continue;
+        recentLogs.push(line);
+        if (recentLogs.length > limit) {
+          recentLogs.splice(0, recentLogs.length - limit);
+        }
+      }
+    },
+    getRecentLogs() {
+      return [...recentLogs];
+    },
+  };
+}
+
+export function formatEmbeddedPostgresError(
+  error: unknown,
+  input: {
+    fallbackMessage: string;
+    recentLogs?: string[];
+  },
+): Error {
+  const baseError = toError(error, input.fallbackMessage);
+  const recentLogs = input.recentLogs ?? [];
+  const parts = [baseError.message];
+  const hint = detectEmbeddedPostgresHint(recentLogs);
+  const recentSummary = summarizeRecentLogs(recentLogs);
+
+  if (hint) {
+    parts.push(hint);
+  }
+  if (recentSummary) {
+    parts.push(`Recent embedded Postgres logs: ${recentSummary}`);
+  }
+
+  return new Error(parts.join(" "));
+}

packages/db/src/index.ts

@@ -25,4 +25,8 @@ export {
   type RunDatabaseBackupResult,
   type RunDatabaseRestoreOptions,
 } from "./backup-lib.js";
+export {
+  createEmbeddedPostgresLogBuffer,
+  formatEmbeddedPostgresError,
+} from "./embedded-postgres-error.js";
 export * from "./schema/index.js";

packages/db/src/migration-runtime.ts

@@ -2,6 +2,7 @@ import { existsSync, readFileSync, rmSync } from "node:fs";
 import { createServer } from "node:net";
 import path from "node:path";
 import { ensurePostgresDatabase, getPostgresDataDirectory } from "./client.js";
+import { createEmbeddedPostgresLogBuffer, formatEmbeddedPostgresError } from "./embedded-postgres-error.js";
 import { resolveDatabaseTarget } from "./runtime-config.js";
 
 type EmbeddedPostgresInstance = {
@@ -27,18 +28,6 @@ export type MigrationConnection = {
   stop: () => Promise<void>;
 };
 
-function toError(error: unknown, fallbackMessage: string): Error {
-  if (error instanceof Error) return error;
-  if (error === undefined) return new Error(fallbackMessage);
-  if (typeof error === "string") return new Error(`${fallbackMessage}: ${error}`);
-
-  try {
-    return new Error(`${fallbackMessage}: ${JSON.stringify(error)}`);
-  } catch {
-    return new Error(`${fallbackMessage}: ${String(error)}`);
-  }
-}
-
 function readRunningPostmasterPid(postmasterPidFile: string): number | null {
   if (!existsSync(postmasterPidFile)) return null;
   try {
@@ -109,6 +98,7 @@ async function ensureEmbeddedPostgresConnection(
   const runningPid = readRunningPostmasterPid(postmasterPidFile);
   const runningPort = readPidFilePort(postmasterPidFile);
   const preferredAdminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${preferredPort}/postgres`;
+  const logBuffer = createEmbeddedPostgresLogBuffer();
 
   if (!runningPid && existsSync(pgVersionFile)) {
     try {
@@ -151,18 +141,19 @@ async function ensureEmbeddedPostgresConnection(
     port: selectedPort,
     persistent: true,
     initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
-    onLog: () => {},
-    onError: () => {},
+    onLog: logBuffer.append,
+    onError: logBuffer.append,
   });
 
   if (!existsSync(path.resolve(dataDir, "PG_VERSION"))) {
     try {
       await instance.initialise();
     } catch (error) {
-      throw toError(
-        error,
-        `Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${selectedPort}`,
-      );
+      throw formatEmbeddedPostgresError(error, {
+        fallbackMessage:
+          `Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${selectedPort}`,
+        recentLogs: logBuffer.getRecentLogs(),
+      });
     }
   }
   if (existsSync(postmasterPidFile)) {
@@ -171,7 +162,10 @@ async function ensureEmbeddedPostgresConnection(
   try {
     await instance.start();
   } catch (error) {
-    throw toError(error, `Failed to start embedded PostgreSQL on port ${selectedPort}`);
+    throw formatEmbeddedPostgresError(error, {
+      fallbackMessage: `Failed to start embedded PostgreSQL on port ${selectedPort}`,
+      recentLogs: logBuffer.getRecentLogs(),
+    });
   }
 
   const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${selectedPort}/postgres`;

server/src/index.ts

@@ -10,9 +10,11 @@ import { and, eq } from "drizzle-orm";
 import {
   createDb,
   ensurePostgresDatabase,
+  formatEmbeddedPostgresError,
   getPostgresDataDirectory,
   inspectMigrations,
   applyPendingMigrations,
+  createEmbeddedPostgresLogBuffer,
   reconcilePendingMigrationHistory,
   formatDatabaseBackupResult,
   runDatabaseBackup,
@@ -272,29 +274,31 @@ export async function startServer(): Promise<StartedServer> {
     const dataDir = resolve(config.embeddedPostgresDataDir);
     const configuredPort = config.embeddedPostgresPort;
     let port = configuredPort;
-    const embeddedPostgresLogBuffer: string[] = [];
-    const EMBEDDED_POSTGRES_LOG_BUFFER_LIMIT = 120;
+    const logBuffer = createEmbeddedPostgresLogBuffer(120);
     const verboseEmbeddedPostgresLogs = process.env.PAPERCLIP_EMBEDDED_POSTGRES_VERBOSE === "true";
     const appendEmbeddedPostgresLog = (message: unknown) => {
-      const text = typeof message === "string" ? message : message instanceof Error ? message.message : String(message ?? "");
-      for (const lineRaw of text.split(/\r?\n/)) {
+      logBuffer.append(message);
+      if (!verboseEmbeddedPostgresLogs) {
+        return;
+      }
+      const lines = typeof message === "string"
+        ? message.split(/\r?\n/)
+        : message instanceof Error
+          ? [message.message]
+          : [String(message ?? "")];
+      for (const lineRaw of lines) {
         const line = lineRaw.trim();
         if (!line) continue;
-        embeddedPostgresLogBuffer.push(line);
-        if (embeddedPostgresLogBuffer.length > EMBEDDED_POSTGRES_LOG_BUFFER_LIMIT) {
-          embeddedPostgresLogBuffer.splice(0, embeddedPostgresLogBuffer.length - EMBEDDED_POSTGRES_LOG_BUFFER_LIMIT);
-        }
-        if (verboseEmbeddedPostgresLogs) {
-          logger.info({ embeddedPostgresLog: line }, "embedded-postgres");
-        }
+        logger.info({ embeddedPostgresLog: line }, "embedded-postgres");
       }
     };
     const logEmbeddedPostgresFailure = (phase: "initialise" | "start", err: unknown) => {
-      if (embeddedPostgresLogBuffer.length > 0) {
+      const recentLogs = logBuffer.getRecentLogs();
+      if (recentLogs.length > 0) {
         logger.error(
           {
             phase,
-            recentLogs: embeddedPostgresLogBuffer,
+            recentLogs,
             err,
           },
           "Embedded PostgreSQL failed; showing buffered startup logs",
@@ -371,7 +375,10 @@ export async function startServer(): Promise<StartedServer> {
             await embeddedPostgres.initialise();
           } catch (err) {
             logEmbeddedPostgresFailure("initialise", err);
-            throw err;
+            throw formatEmbeddedPostgresError(err, {
+              fallbackMessage: `Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${port}`,
+              recentLogs: logBuffer.getRecentLogs(),
+            });
           }
         } else {
           logger.info(`Embedded PostgreSQL cluster already exists (${clusterVersionFile}); skipping init`);
@@ -385,7 +392,10 @@ export async function startServer(): Promise<StartedServer> {
           await embeddedPostgres.start();
         } catch (err) {
           logEmbeddedPostgresFailure("start", err);
-          throw err;
+          throw formatEmbeddedPostgresError(err, {
+            fallbackMessage: `Failed to start embedded PostgreSQL on port ${port}`,
+            recentLogs: logBuffer.getRecentLogs(),
+          });
         }
         embeddedPostgresStartedByThisProcess = true;
       }