From 76a692c260ea3c90cff03486b93deee3af4e467f Mon Sep 17 00:00:00 2001 From: dotta Date: Thu, 26 Mar 2026 11:34:16 -0500 Subject: [PATCH 1/3] Improve embedded Postgres bootstrap errors Co-Authored-By: Paperclip --- cli/src/commands/worktree.ts | 25 +++++- .../db/src/embedded-postgres-error.test.ts | 28 ++++++ packages/db/src/embedded-postgres-error.ts | 89 +++++++++++++++++++ packages/db/src/index.ts | 4 + packages/db/src/migration-runtime.ts | 32 +++---- server/src/index.ts | 40 +++++---- 6 files changed, 180 insertions(+), 38 deletions(-) create mode 100644 packages/db/src/embedded-postgres-error.test.ts create mode 100644 packages/db/src/embedded-postgres-error.ts diff --git a/cli/src/commands/worktree.ts b/cli/src/commands/worktree.ts index a528bf5b..65e74849 100644 --- a/cli/src/commands/worktree.ts +++ b/cli/src/commands/worktree.ts @@ -41,6 +41,8 @@ import { projects, runDatabaseBackup, runDatabaseRestore, + createEmbeddedPostgresLogBuffer, + formatEmbeddedPostgresError, } from "@paperclipai/db"; import type { Command } from "commander"; import { ensureAgentJwtSecret, loadPaperclipEnvFile, mergePaperclipEnvEntries, readPaperclipEnvEntries, resolvePaperclipEnvFile } from "../config/env.js"; @@ -806,6 +808,7 @@ async function ensureEmbeddedPostgres(dataDir: string, preferredPort: number): P } const port = await findAvailablePort(preferredPort); + const logBuffer = createEmbeddedPostgresLogBuffer(); const instance = new EmbeddedPostgres({ databaseDir: dataDir, user: "paperclip", @@ -813,17 +816,31 @@ async function ensureEmbeddedPostgres(dataDir: string, preferredPort: number): P port, persistent: true, initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"], - onLog: () => {}, - onError: () => {}, + onLog: logBuffer.append, + onError: logBuffer.append, }); if (!existsSync(path.resolve(dataDir, "PG_VERSION"))) { - await instance.initialise(); + try { + await instance.initialise(); + } catch (error) { + throw formatEmbeddedPostgresError(error, { + fallbackMessage: `Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${port}`, + recentLogs: logBuffer.getRecentLogs(), + }); + } } if (existsSync(postmasterPidFile)) { rmSync(postmasterPidFile, { force: true }); } - await instance.start(); + try { + await instance.start(); + } catch (error) { + throw formatEmbeddedPostgresError(error, { + fallbackMessage: `Failed to start embedded PostgreSQL on port ${port}`, + recentLogs: logBuffer.getRecentLogs(), + }); + } return { port, diff --git a/packages/db/src/embedded-postgres-error.test.ts b/packages/db/src/embedded-postgres-error.test.ts new file mode 100644 index 00000000..dba1ad46 --- /dev/null +++ b/packages/db/src/embedded-postgres-error.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vitest"; +import { createEmbeddedPostgresLogBuffer, formatEmbeddedPostgresError } from "./embedded-postgres-error.js"; + +describe("formatEmbeddedPostgresError", () => { + it("adds a shared-memory hint when initdb logs expose the real cause", () => { + const error = formatEmbeddedPostgresError("Postgres init script exited with code 1.", { + fallbackMessage: "Failed to initialize embedded PostgreSQL cluster", + recentLogs: [ + "running bootstrap script ...", + "FATAL: could not create shared memory segment: Cannot allocate memory", + "DETAIL: Failed system call was shmget(key=123, size=56, 03600).", + ], + }); + + expect(error.message).toContain("could not allocate shared memory"); + expect(error.message).toContain("kern.sysv.shm"); + expect(error.message).toContain("could not create shared memory segment"); + }); + + it("keeps only recent non-empty log lines in the collector", () => { + const buffer = createEmbeddedPostgresLogBuffer(2); + buffer.append("line one\n\n"); + buffer.append("line two"); + buffer.append("line three"); + + expect(buffer.getRecentLogs()).toEqual(["line two", "line three"]); + }); +}); diff --git a/packages/db/src/embedded-postgres-error.ts b/packages/db/src/embedded-postgres-error.ts new file mode 100644 index 00000000..9862a0f3 --- /dev/null +++ b/packages/db/src/embedded-postgres-error.ts @@ -0,0 +1,89 @@ +const DEFAULT_RECENT_LOG_LIMIT = 40; +const RECENT_LOG_SUMMARY_LINES = 8; + +function toError(error: unknown, fallbackMessage: string): Error { + if (error instanceof Error) return error; + if (error === undefined) return new Error(fallbackMessage); + if (typeof error === "string") return new Error(`${fallbackMessage}: ${error}`); + + try { + return new Error(`${fallbackMessage}: ${JSON.stringify(error)}`); + } catch { + return new Error(`${fallbackMessage}: ${String(error)}`); + } +} + +function summarizeRecentLogs(recentLogs: string[]): string | null { + if (recentLogs.length === 0) return null; + return recentLogs + .slice(-RECENT_LOG_SUMMARY_LINES) + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .join(" | "); +} + +function detectEmbeddedPostgresHint(recentLogs: string[]): string | null { + const haystack = recentLogs.join("\n").toLowerCase(); + if (!haystack.includes("could not create shared memory segment")) { + return null; + } + + return ( + "Embedded PostgreSQL bootstrap could not allocate shared memory. " + + "On macOS, this usually means the host's kern.sysv.shm* limits are too low for another local PostgreSQL cluster. " + + "Stop other local PostgreSQL servers or raise the shared-memory sysctls, then retry." + ); +} + +export function createEmbeddedPostgresLogBuffer(limit = DEFAULT_RECENT_LOG_LIMIT): { + append(message: unknown): void; + getRecentLogs(): string[]; +} { + const recentLogs: string[] = []; + + return { + append(message: unknown) { + const text = + typeof message === "string" + ? message + : message instanceof Error + ? message.message + : String(message ?? ""); + + for (const rawLine of text.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line) continue; + recentLogs.push(line); + if (recentLogs.length > limit) { + recentLogs.splice(0, recentLogs.length - limit); + } + } + }, + getRecentLogs() { + return [...recentLogs]; + }, + }; +} + +export function formatEmbeddedPostgresError( + error: unknown, + input: { + fallbackMessage: string; + recentLogs?: string[]; + }, +): Error { + const baseError = toError(error, input.fallbackMessage); + const recentLogs = input.recentLogs ?? []; + const parts = [baseError.message]; + const hint = detectEmbeddedPostgresHint(recentLogs); + const recentSummary = summarizeRecentLogs(recentLogs); + + if (hint) { + parts.push(hint); + } + if (recentSummary) { + parts.push(`Recent embedded Postgres logs: ${recentSummary}`); + } + + return new Error(parts.join(" ")); +} diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index b5ccb5d4..6c45acbc 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -25,4 +25,8 @@ export { type RunDatabaseBackupResult, type RunDatabaseRestoreOptions, } from "./backup-lib.js"; +export { + createEmbeddedPostgresLogBuffer, + formatEmbeddedPostgresError, +} from "./embedded-postgres-error.js"; export * from "./schema/index.js"; diff --git a/packages/db/src/migration-runtime.ts b/packages/db/src/migration-runtime.ts index 921de612..5aa2b6a2 100644 --- a/packages/db/src/migration-runtime.ts +++ b/packages/db/src/migration-runtime.ts @@ -2,6 +2,7 @@ import { existsSync, readFileSync, rmSync } from "node:fs"; import { createServer } from "node:net"; import path from "node:path"; import { ensurePostgresDatabase, getPostgresDataDirectory } from "./client.js"; +import { createEmbeddedPostgresLogBuffer, formatEmbeddedPostgresError } from "./embedded-postgres-error.js"; import { resolveDatabaseTarget } from "./runtime-config.js"; type EmbeddedPostgresInstance = { @@ -27,18 +28,6 @@ export type MigrationConnection = { stop: () => Promise; }; -function toError(error: unknown, fallbackMessage: string): Error { - if (error instanceof Error) return error; - if (error === undefined) return new Error(fallbackMessage); - if (typeof error === "string") return new Error(`${fallbackMessage}: ${error}`); - - try { - return new Error(`${fallbackMessage}: ${JSON.stringify(error)}`); - } catch { - return new Error(`${fallbackMessage}: ${String(error)}`); - } -} - function readRunningPostmasterPid(postmasterPidFile: string): number | null { if (!existsSync(postmasterPidFile)) return null; try { @@ -109,6 +98,7 @@ async function ensureEmbeddedPostgresConnection( const runningPid = readRunningPostmasterPid(postmasterPidFile); const runningPort = readPidFilePort(postmasterPidFile); const preferredAdminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${preferredPort}/postgres`; + const logBuffer = createEmbeddedPostgresLogBuffer(); if (!runningPid && existsSync(pgVersionFile)) { try { @@ -151,18 +141,19 @@ async function ensureEmbeddedPostgresConnection( port: selectedPort, persistent: true, initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"], - onLog: () => {}, - onError: () => {}, + onLog: logBuffer.append, + onError: logBuffer.append, }); if (!existsSync(path.resolve(dataDir, "PG_VERSION"))) { try { await instance.initialise(); } catch (error) { - throw toError( - error, - `Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${selectedPort}`, - ); + throw formatEmbeddedPostgresError(error, { + fallbackMessage: + `Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${selectedPort}`, + recentLogs: logBuffer.getRecentLogs(), + }); } } if (existsSync(postmasterPidFile)) { @@ -171,7 +162,10 @@ async function ensureEmbeddedPostgresConnection( try { await instance.start(); } catch (error) { - throw toError(error, `Failed to start embedded PostgreSQL on port ${selectedPort}`); + throw formatEmbeddedPostgresError(error, { + fallbackMessage: `Failed to start embedded PostgreSQL on port ${selectedPort}`, + recentLogs: logBuffer.getRecentLogs(), + }); } const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${selectedPort}/postgres`; diff --git a/server/src/index.ts b/server/src/index.ts index c37157c0..7ebfa7d1 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -10,9 +10,11 @@ import { and, eq } from "drizzle-orm"; import { createDb, ensurePostgresDatabase, + formatEmbeddedPostgresError, getPostgresDataDirectory, inspectMigrations, applyPendingMigrations, + createEmbeddedPostgresLogBuffer, reconcilePendingMigrationHistory, formatDatabaseBackupResult, runDatabaseBackup, @@ -272,29 +274,31 @@ export async function startServer(): Promise { const dataDir = resolve(config.embeddedPostgresDataDir); const configuredPort = config.embeddedPostgresPort; let port = configuredPort; - const embeddedPostgresLogBuffer: string[] = []; - const EMBEDDED_POSTGRES_LOG_BUFFER_LIMIT = 120; + const logBuffer = createEmbeddedPostgresLogBuffer(120); const verboseEmbeddedPostgresLogs = process.env.PAPERCLIP_EMBEDDED_POSTGRES_VERBOSE === "true"; const appendEmbeddedPostgresLog = (message: unknown) => { - const text = typeof message === "string" ? message : message instanceof Error ? message.message : String(message ?? ""); - for (const lineRaw of text.split(/\r?\n/)) { + logBuffer.append(message); + if (!verboseEmbeddedPostgresLogs) { + return; + } + const lines = typeof message === "string" + ? message.split(/\r?\n/) + : message instanceof Error + ? [message.message] + : [String(message ?? "")]; + for (const lineRaw of lines) { const line = lineRaw.trim(); if (!line) continue; - embeddedPostgresLogBuffer.push(line); - if (embeddedPostgresLogBuffer.length > EMBEDDED_POSTGRES_LOG_BUFFER_LIMIT) { - embeddedPostgresLogBuffer.splice(0, embeddedPostgresLogBuffer.length - EMBEDDED_POSTGRES_LOG_BUFFER_LIMIT); - } - if (verboseEmbeddedPostgresLogs) { - logger.info({ embeddedPostgresLog: line }, "embedded-postgres"); - } + logger.info({ embeddedPostgresLog: line }, "embedded-postgres"); } }; const logEmbeddedPostgresFailure = (phase: "initialise" | "start", err: unknown) => { - if (embeddedPostgresLogBuffer.length > 0) { + const recentLogs = logBuffer.getRecentLogs(); + if (recentLogs.length > 0) { logger.error( { phase, - recentLogs: embeddedPostgresLogBuffer, + recentLogs, err, }, "Embedded PostgreSQL failed; showing buffered startup logs", @@ -371,7 +375,10 @@ export async function startServer(): Promise { await embeddedPostgres.initialise(); } catch (err) { logEmbeddedPostgresFailure("initialise", err); - throw err; + throw formatEmbeddedPostgresError(err, { + fallbackMessage: `Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${port}`, + recentLogs: logBuffer.getRecentLogs(), + }); } } else { logger.info(`Embedded PostgreSQL cluster already exists (${clusterVersionFile}); skipping init`); @@ -385,7 +392,10 @@ export async function startServer(): Promise { await embeddedPostgres.start(); } catch (err) { logEmbeddedPostgresFailure("start", err); - throw err; + throw formatEmbeddedPostgresError(err, { + fallbackMessage: `Failed to start embedded PostgreSQL on port ${port}`, + recentLogs: logBuffer.getRecentLogs(), + }); } embeddedPostgresStartedByThisProcess = true; } From a8894799e44258fb13a329b2d56b7689121b76fd Mon Sep 17 00:00:00 2001 From: dotta Date: Thu, 26 Mar 2026 12:12:13 -0500 Subject: [PATCH 2/3] Align worktree provision with worktree init Co-Authored-By: Paperclip --- doc/DEVELOPING.md | 11 +++++++++++ scripts/provision-worktree.sh | 8 ++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 7d98cd6d..7864b90e 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -206,6 +206,17 @@ paperclipai worktree init --from-data-dir ~/.paperclip paperclipai worktree init --force ``` +Repair an already-created repo-managed worktree and reseed its isolated instance from the main default install: + +```sh +cd ~/.paperclip/worktrees/PAP-884-ai-commits-component +pnpm paperclipai worktree init --force --seed-mode minimal \ + --name PAP-884-ai-commits-component \ + --from-config ~/.paperclip/instances/default/config.json +``` + +That rewrites the worktree-local `.paperclip/config.json` + `.paperclip/.env`, recreates the isolated instance under `~/.paperclip-worktrees/instances//`, and preserves the git worktree contents themselves. + **`pnpm paperclipai worktree:make [options]`** — Create `~/NAME` as a git worktree, then initialize an isolated Paperclip instance inside it. This combines `git worktree add` with `worktree init` in a single step. | Option | Description | diff --git a/scripts/provision-worktree.sh b/scripts/provision-worktree.sh index ea5e0e0f..19b0831e 100644 --- a/scripts/provision-worktree.sh +++ b/scripts/provision-worktree.sh @@ -32,13 +32,13 @@ source_env_path="$(dirname "$source_config_path")/.env" mkdir -p "$paperclip_dir" run_isolated_worktree_init() { - if command -v paperclipai >/dev/null 2>&1; then - paperclipai worktree init --force --name "$worktree_name" --from-config "$source_config_path" + if command -v pnpm >/dev/null 2>&1 && pnpm paperclipai --help >/dev/null 2>&1; then + pnpm paperclipai worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path" return 0 fi - if command -v pnpm >/dev/null 2>&1 && pnpm paperclipai --help >/dev/null 2>&1; then - pnpm paperclipai worktree init --force --name "$worktree_name" --from-config "$source_config_path" + if command -v paperclipai >/dev/null 2>&1; then + paperclipai worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path" return 0 fi From 9ddf960312d53052762474d592fe3af4b7c4b1ce Mon Sep 17 00:00:00 2001 From: dotta Date: Thu, 26 Mar 2026 12:27:17 -0500 Subject: [PATCH 3/3] Harden dev-watch excludes for nested UI outputs Co-Authored-By: Paperclip --- server/scripts/dev-watch.ts | 2 +- server/src/__tests__/dev-watch-ignore.test.ts | 8 +++++++ server/src/dev-watch-ignore.ts | 21 ++++++++++++++++--- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/server/scripts/dev-watch.ts b/server/scripts/dev-watch.ts index 69a85245..cfcb7a71 100644 --- a/server/scripts/dev-watch.ts +++ b/server/scripts/dev-watch.ts @@ -7,7 +7,7 @@ import { resolveServerDevWatchIgnorePaths } from "../src/dev-watch-ignore.ts"; const require = createRequire(import.meta.url); const tsxCliPath = require.resolve("tsx/dist/cli.mjs"); const serverRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); -const ignoreArgs = resolveServerDevWatchIgnorePaths(serverRoot).flatMap((ignorePath) => ["--ignore", ignorePath]); +const ignoreArgs = resolveServerDevWatchIgnorePaths(serverRoot).flatMap((ignorePath) => ["--exclude", ignorePath]); const child = spawn( process.execPath, diff --git a/server/src/__tests__/dev-watch-ignore.test.ts b/server/src/__tests__/dev-watch-ignore.test.ts index 0331f61b..4f54e609 100644 --- a/server/src/__tests__/dev-watch-ignore.test.ts +++ b/server/src/__tests__/dev-watch-ignore.test.ts @@ -25,10 +25,18 @@ describe("resolveServerDevWatchIgnorePaths", () => { const ignorePaths = resolveServerDevWatchIgnorePaths(serverRoot); expect(ignorePaths).toContain(path.join(worktreeUiRoot, "node_modules")); + expect(ignorePaths).toContain(`${path.join(worktreeUiRoot, "node_modules").replaceAll(path.sep, "/")}/**`); expect(ignorePaths).toContain(fs.realpathSync(path.join(sharedUiRoot, "node_modules"))); + expect(ignorePaths).toContain(`${fs.realpathSync(path.join(sharedUiRoot, "node_modules")).replaceAll(path.sep, "/")}/**`); + expect(ignorePaths).toContain(path.join(worktreeUiRoot, "node_modules", ".vite-temp")); + expect(ignorePaths).toContain( + `${path.join(worktreeUiRoot, "node_modules", ".vite-temp").replaceAll(path.sep, "/")}/**`, + ); expect(ignorePaths).toContain(path.join(worktreeUiRoot, ".vite")); expect(ignorePaths).toContain(fs.realpathSync(path.join(sharedUiRoot, ".vite"))); expect(ignorePaths).toContain(path.join(worktreeUiRoot, "dist")); expect(ignorePaths).toContain(fs.realpathSync(path.join(sharedUiRoot, "dist"))); + expect(ignorePaths).toContain("**/{node_modules,bower_components,vendor}/**"); + expect(ignorePaths).toContain("**/.vite-temp/**"); }); }); diff --git a/server/src/dev-watch-ignore.ts b/server/src/dev-watch-ignore.ts index 6e7d90ce..cd618f73 100644 --- a/server/src/dev-watch-ignore.ts +++ b/server/src/dev-watch-ignore.ts @@ -1,19 +1,34 @@ import fs from "node:fs"; import path from "node:path"; +function toGlobstarPath(candidate: string): string { + return `${candidate.replaceAll(path.sep, "/")}/**`; +} + function addIgnorePath(target: Set, candidate: string): void { target.add(candidate); + target.add(toGlobstarPath(candidate)); try { - target.add(fs.realpathSync(candidate)); + const realPath = fs.realpathSync(candidate); + target.add(realPath); + target.add(toGlobstarPath(realPath)); } catch { // Ignore paths that do not exist in the current checkout. } } export function resolveServerDevWatchIgnorePaths(serverRoot: string): string[] { - const ignorePaths = new Set(); + const ignorePaths = new Set([ + "**/{node_modules,bower_components,vendor}/**", + "**/.vite-temp/**", + ]); - for (const relativePath of ["../ui/node_modules", "../ui/.vite", "../ui/dist"]) { + for (const relativePath of [ + "../ui/node_modules", + "../ui/node_modules/.vite-temp", + "../ui/.vite", + "../ui/dist", + ]) { addIgnorePath(ignorePaths, path.resolve(serverRoot, relativePath)); }