Merge pull request #1829 from paperclipai/pr/pap-891-worktree-reliability

fix(worktree): harden provisioned worktree isolation and test fallback behavior
This commit is contained in:
Dotta 2026-03-26 12:01:47 -05:00 committed by GitHub
commit b944293eda
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 1873 additions and 469 deletions

View file

@ -234,16 +234,27 @@ See [doc/DEVELOPING.md](doc/DEVELOPING.md) for the full development guide.
## Roadmap
- ⚪ Get OpenClaw onboarding easier
- ⚪ Get cloud agents working e.g. Cursor / e2b agents
- ⚪ ClipMart - buy and sell entire agent companies
- ⚪ Easy agent configurations / easier to understand
- ⚪ Better support for harness engineering
- 🟢 Plugin system (e.g. if you want to add a knowledgebase, custom tracing, queues, etc)
- ⚪ Better docs
- ✅ Plugin system (e.g. add a knowledge base, custom tracing, queues, etc)
- ✅ Get OpenClaw / claw-style agent employees
- ✅ companies.sh - import and export entire organizations
- ✅ Easy AGENTS.md configurations
- ✅ Skills Manager
- ✅ Scheduled Routines
- ✅ Better Budgeting
- ⚪ Artifacts & Deployments
- ⚪ CEO Chat
- ⚪ MAXIMIZER MODE
- ⚪ Multiple Human Users
- ⚪ Cloud / Sandbox agents (e.g. Cursor / e2b agents)
- ⚪ Cloud deployments
- ⚪ Desktop App
<br/>
## Community & Plugins
Find Plugins and more at [awesome-paperclip](https://github.com/gsxdsm/awesome-paperclip)
## Contributing
We welcome contributions. See the [contributing guide](CONTRIBUTING.md) for details.

View file

@ -6,33 +6,15 @@ import path from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
import { createStoredZipArchive } from "./helpers/zip.js";
type EmbeddedPostgresInstance = {
initialise(): Promise<void>;
start(): Promise<void>;
stop(): Promise<void>;
};
type EmbeddedPostgresCtor = new (opts: {
databaseDir: string;
user: string;
password: string;
port: number;
persistent: boolean;
initdbFlags?: string[];
onLog?: (message: unknown) => void;
onError?: (message: unknown) => void;
}) => EmbeddedPostgresInstance;
const execFileAsync = promisify(execFile);
type ServerProcess = ReturnType<typeof spawn>;
async function getEmbeddedPostgresCtor(): Promise<EmbeddedPostgresCtor> {
const mod = await import("embedded-postgres");
return mod.default as EmbeddedPostgresCtor;
}
async function getAvailablePort(): Promise<number> {
return await new Promise((resolve, reject) => {
const server = net.createServer();
@ -53,30 +35,13 @@ async function getAvailablePort(): Promise<number> {
});
}
async function startTempDatabase() {
const dataDir = mkdtempSync(path.join(os.tmpdir(), "paperclip-company-cli-db-"));
const port = await getAvailablePort();
const EmbeddedPostgres = await getEmbeddedPostgresCtor();
const instance = new EmbeddedPostgres({
databaseDir: dataDir,
user: "paperclip",
password: "paperclip",
port,
persistent: true,
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
onLog: () => {},
onError: () => {},
});
await instance.initialise();
await instance.start();
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
const { applyPendingMigrations, ensurePostgresDatabase } = await import("@paperclipai/db");
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`;
await ensurePostgresDatabase(adminConnectionString, "paperclip");
const connectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
await applyPendingMigrations(connectionString);
return { connectionString, dataDir, instance };
if (!embeddedPostgresSupport.supported) {
console.warn(
`Skipping embedded Postgres company import/export e2e tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
);
}
function writeTestConfig(configPath: string, tempRoot: string, port: number, connectionString: string) {
@ -265,26 +230,23 @@ async function waitForServer(
);
}
describe("paperclipai company import/export e2e", () => {
describeEmbeddedPostgres("paperclipai company import/export e2e", () => {
let tempRoot = "";
let configPath = "";
let exportDir = "";
let apiBase = "";
let serverProcess: ServerProcess | null = null;
let dbDataDir = "";
let dbInstance: EmbeddedPostgresInstance | null = null;
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
beforeAll(async () => {
tempRoot = mkdtempSync(path.join(os.tmpdir(), "paperclip-company-cli-e2e-"));
configPath = path.join(tempRoot, "config", "config.json");
exportDir = path.join(tempRoot, "exported-company");
const db = await startTempDatabase();
dbDataDir = db.dataDir;
dbInstance = db.instance;
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-cli-db-");
const port = await getAvailablePort();
writeTestConfig(configPath, tempRoot, port, db.connectionString);
writeTestConfig(configPath, tempRoot, port, tempDb.connectionString);
apiBase = `http://127.0.0.1:${port}`;
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../..");
@ -294,7 +256,7 @@ describe("paperclipai company import/export e2e", () => {
["paperclipai", "run", "--config", configPath],
{
cwd: repoRoot,
env: createServerEnv(configPath, port, db.connectionString),
env: createServerEnv(configPath, port, tempDb.connectionString),
stdio: ["ignore", "pipe", "pipe"],
},
);
@ -311,10 +273,7 @@ describe("paperclipai company import/export e2e", () => {
afterAll(async () => {
await stopServerProcess(serverProcess);
await dbInstance?.stop();
if (dbDataDir) {
rmSync(dbDataDir, { recursive: true, force: true });
}
await tempDb?.cleanup();
if (tempRoot) {
rmSync(tempRoot, { recursive: true, force: true });
}

View file

@ -0,0 +1,6 @@
export {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
type EmbeddedPostgresTestDatabase,
type EmbeddedPostgresTestSupport,
} from "@paperclipai/db";

View file

@ -344,6 +344,87 @@ describe("worktree helpers", () => {
}
});
it("avoids ports already claimed by sibling worktree instance configs", async () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-claimed-ports-"));
const repoRoot = path.join(tempRoot, "repo");
const homeDir = path.join(tempRoot, ".paperclip-worktrees");
const siblingInstanceRoot = path.join(homeDir, "instances", "existing-worktree");
const originalCwd = process.cwd();
try {
fs.mkdirSync(repoRoot, { recursive: true });
fs.mkdirSync(siblingInstanceRoot, { recursive: true });
fs.writeFileSync(
path.join(siblingInstanceRoot, "config.json"),
JSON.stringify(
{
...buildSourceConfig(),
database: {
mode: "embedded-postgres",
embeddedPostgresDataDir: path.join(siblingInstanceRoot, "db"),
embeddedPostgresPort: 54330,
backup: {
enabled: true,
intervalMinutes: 60,
retentionDays: 30,
dir: path.join(siblingInstanceRoot, "backups"),
},
},
logging: {
mode: "file",
logDir: path.join(siblingInstanceRoot, "logs"),
},
server: {
deploymentMode: "authenticated",
exposure: "private",
host: "127.0.0.1",
port: 3101,
allowedHostnames: ["localhost"],
serveUi: true,
},
storage: {
provider: "local_disk",
localDisk: {
baseDir: path.join(siblingInstanceRoot, "storage"),
},
s3: {
bucket: "paperclip",
region: "us-east-1",
prefix: "",
forcePathStyle: false,
},
},
secrets: {
provider: "local_encrypted",
strictMode: false,
localEncrypted: {
keyFilePath: path.join(siblingInstanceRoot, "secrets", "master.key"),
},
},
},
null,
2,
) + "\n",
);
process.chdir(repoRoot);
await worktreeInitCommand({
seed: false,
fromConfig: path.join(tempRoot, "missing", "config.json"),
home: homeDir,
});
const config = JSON.parse(fs.readFileSync(path.join(repoRoot, ".paperclip", "config.json"), "utf8"));
expect(config.server.port).toBe(3102);
expect(config.database.embeddedPostgresPort).not.toBe(54330);
expect(config.database.embeddedPostgresPort).not.toBe(config.server.port);
expect(config.database.embeddedPostgresPort).toBeGreaterThan(54330);
} finally {
process.chdir(originalCwd);
fs.rmSync(tempRoot, { recursive: true, force: true });
}
});
it("defaults the seed source config to the current repo-local Paperclip config", () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-source-config-"));
const repoRoot = path.join(tempRoot, "repo");

View file

@ -465,6 +465,62 @@ async function findAvailablePort(preferredPort: number, reserved = new Set<numbe
return port;
}
function resolveRepoManagedWorktreesRoot(cwd: string): string | null {
const normalized = path.resolve(cwd);
const marker = `${path.sep}.paperclip${path.sep}worktrees${path.sep}`;
const index = normalized.indexOf(marker);
if (index === -1) return null;
const repoRoot = normalized.slice(0, index);
return path.resolve(repoRoot, ".paperclip", "worktrees");
}
function collectClaimedWorktreePorts(homeDir: string, currentInstanceId: string, cwd: string): {
serverPorts: Set<number>;
databasePorts: Set<number>;
} {
const serverPorts = new Set<number>();
const databasePorts = new Set<number>();
const configPaths = new Set<string>();
const instancesDir = path.resolve(homeDir, "instances");
if (existsSync(instancesDir)) {
for (const entry of readdirSync(instancesDir, { withFileTypes: true })) {
if (!entry.isDirectory() || entry.name === currentInstanceId) continue;
const configPath = path.resolve(instancesDir, entry.name, "config.json");
if (existsSync(configPath)) {
configPaths.add(configPath);
}
}
}
const repoManagedWorktreesRoot = resolveRepoManagedWorktreesRoot(cwd);
if (repoManagedWorktreesRoot && existsSync(repoManagedWorktreesRoot)) {
for (const entry of readdirSync(repoManagedWorktreesRoot, { withFileTypes: true })) {
if (!entry.isDirectory()) continue;
const configPath = path.resolve(repoManagedWorktreesRoot, entry.name, ".paperclip", "config.json");
if (existsSync(configPath)) {
configPaths.add(configPath);
}
}
}
for (const configPath of configPaths) {
try {
const config = readConfig(configPath);
if (config?.server.port) {
serverPorts.add(config.server.port);
}
if (config?.database.mode === "embedded-postgres") {
databasePorts.add(config.database.embeddedPostgresPort);
}
} catch {
// Ignore malformed sibling configs.
}
}
return { serverPorts, databasePorts };
}
function detectGitBranchName(cwd: string): string | null {
try {
const value = execFileSync("git", ["branch", "--show-current"], {
@ -886,10 +942,14 @@ async function runWorktreeInit(opts: WorktreeInitOptions): Promise<void> {
rmSync(paths.instanceRoot, { recursive: true, force: true });
}
const claimedPorts = collectClaimedWorktreePorts(paths.homeDir, paths.instanceId, paths.cwd);
const preferredServerPort = opts.serverPort ?? ((sourceConfig?.server.port ?? 3100) + 1);
const serverPort = await findAvailablePort(preferredServerPort);
const serverPort = await findAvailablePort(preferredServerPort, claimedPorts.serverPorts);
const preferredDbPort = opts.dbPort ?? ((sourceConfig?.database.embeddedPostgresPort ?? 54329) + 1);
const databasePort = await findAvailablePort(preferredDbPort, new Set([serverPort]));
const databasePort = await findAvailablePort(
preferredDbPort,
new Set([...claimedPorts.databasePorts, serverPort]),
);
const targetConfig = buildWorktreeConfig({
sourceConfig,
paths,

View file

@ -1,83 +1,24 @@
import { createHash } from "node:crypto";
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import postgres from "postgres";
import {
applyPendingMigrations,
ensurePostgresDatabase,
inspectMigrations,
} from "./client.js";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
} from "./test-embedded-postgres.js";
type EmbeddedPostgresInstance = {
initialise(): Promise<void>;
start(): Promise<void>;
stop(): Promise<void>;
};
type EmbeddedPostgresCtor = new (opts: {
databaseDir: string;
user: string;
password: string;
port: number;
persistent: boolean;
initdbFlags?: string[];
onLog?: (message: unknown) => void;
onError?: (message: unknown) => void;
}) => EmbeddedPostgresInstance;
const tempPaths: string[] = [];
const runningInstances: EmbeddedPostgresInstance[] = [];
async function getEmbeddedPostgresCtor(): Promise<EmbeddedPostgresCtor> {
const mod = await import("embedded-postgres");
return mod.default as EmbeddedPostgresCtor;
}
async function getAvailablePort(): Promise<number> {
return await new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on("error", reject);
server.listen(0, "127.0.0.1", () => {
const address = server.address();
if (!address || typeof address === "string") {
server.close(() => reject(new Error("Failed to allocate test port")));
return;
}
const { port } = address;
server.close((error) => {
if (error) reject(error);
else resolve(port);
});
});
});
}
const cleanups: Array<() => Promise<void>> = [];
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
async function createTempDatabase(): Promise<string> {
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-db-client-"));
tempPaths.push(dataDir);
const port = await getAvailablePort();
const EmbeddedPostgres = await getEmbeddedPostgresCtor();
const instance = new EmbeddedPostgres({
databaseDir: dataDir,
user: "paperclip",
password: "paperclip",
port,
persistent: true,
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
onLog: () => {},
onError: () => {},
});
await instance.initialise();
await instance.start();
runningInstances.push(instance);
const adminUrl = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`;
await ensurePostgresDatabase(adminUrl, "paperclip");
return `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
const db = await startEmbeddedPostgresTestDatabase("paperclip-db-client-");
cleanups.push(db.cleanup);
return db.connectionString;
}
async function migrationHash(migrationFile: string): Promise<string> {
@ -89,19 +30,19 @@ async function migrationHash(migrationFile: string): Promise<string> {
}
afterEach(async () => {
while (runningInstances.length > 0) {
const instance = runningInstances.pop();
if (!instance) continue;
await instance.stop();
}
while (tempPaths.length > 0) {
const tempPath = tempPaths.pop();
if (!tempPath) continue;
fs.rmSync(tempPath, { recursive: true, force: true });
while (cleanups.length > 0) {
const cleanup = cleanups.pop();
await cleanup?.();
}
});
describe("applyPendingMigrations", () => {
if (!embeddedPostgresSupport.supported) {
console.warn(
`Skipping embedded Postgres migration tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
);
}
describeEmbeddedPostgres("applyPendingMigrations", () => {
it(
"applies an inserted earlier migration without replaying later legacy migrations",
async () => {

View file

@ -11,6 +11,12 @@ export {
type MigrationBootstrapResult,
type Db,
} from "./client.js";
export {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
type EmbeddedPostgresTestDatabase,
type EmbeddedPostgresTestSupport,
} from "./test-embedded-postgres.js";
export {
runDatabaseBackup,
runDatabaseRestore,

View file

@ -0,0 +1,144 @@
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import { applyPendingMigrations, ensurePostgresDatabase } from "./client.js";
type EmbeddedPostgresInstance = {
initialise(): Promise<void>;
start(): Promise<void>;
stop(): Promise<void>;
};
type EmbeddedPostgresCtor = new (opts: {
databaseDir: string;
user: string;
password: string;
port: number;
persistent: boolean;
initdbFlags?: string[];
onLog?: (message: unknown) => void;
onError?: (message: unknown) => void;
}) => EmbeddedPostgresInstance;
export type EmbeddedPostgresTestSupport = {
supported: boolean;
reason?: string;
};
export type EmbeddedPostgresTestDatabase = {
connectionString: string;
cleanup(): Promise<void>;
};
let embeddedPostgresSupportPromise: Promise<EmbeddedPostgresTestSupport> | null = null;
async function getEmbeddedPostgresCtor(): Promise<EmbeddedPostgresCtor> {
const mod = await import("embedded-postgres");
return mod.default as EmbeddedPostgresCtor;
}
async function getAvailablePort(): Promise<number> {
return await new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on("error", reject);
server.listen(0, "127.0.0.1", () => {
const address = server.address();
if (!address || typeof address === "string") {
server.close(() => reject(new Error("Failed to allocate test port")));
return;
}
const { port } = address;
server.close((error) => {
if (error) reject(error);
else resolve(port);
});
});
});
}
function formatEmbeddedPostgresError(error: unknown): string {
if (error instanceof Error && error.message.length > 0) return error.message;
if (typeof error === "string" && error.length > 0) return error;
return "embedded Postgres startup failed";
}
async function probeEmbeddedPostgresSupport(): Promise<EmbeddedPostgresTestSupport> {
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-embedded-postgres-probe-"));
const port = await getAvailablePort();
const EmbeddedPostgres = await getEmbeddedPostgresCtor();
const instance = new EmbeddedPostgres({
databaseDir: dataDir,
user: "paperclip",
password: "paperclip",
port,
persistent: true,
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
onLog: () => {},
onError: () => {},
});
try {
await instance.initialise();
await instance.start();
return { supported: true };
} catch (error) {
return {
supported: false,
reason: formatEmbeddedPostgresError(error),
};
} finally {
await instance.stop().catch(() => {});
fs.rmSync(dataDir, { recursive: true, force: true });
}
}
export async function getEmbeddedPostgresTestSupport(): Promise<EmbeddedPostgresTestSupport> {
if (!embeddedPostgresSupportPromise) {
embeddedPostgresSupportPromise = probeEmbeddedPostgresSupport();
}
return await embeddedPostgresSupportPromise;
}
export async function startEmbeddedPostgresTestDatabase(
tempDirPrefix: string,
): Promise<EmbeddedPostgresTestDatabase> {
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), tempDirPrefix));
const port = await getAvailablePort();
const EmbeddedPostgres = await getEmbeddedPostgresCtor();
const instance = new EmbeddedPostgres({
databaseDir: dataDir,
user: "paperclip",
password: "paperclip",
port,
persistent: true,
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
onLog: () => {},
onError: () => {},
});
try {
await instance.initialise();
await instance.start();
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`;
await ensurePostgresDatabase(adminConnectionString, "paperclip");
const connectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
await applyPendingMigrations(connectionString);
return {
connectionString,
cleanup: async () => {
await instance.stop().catch(() => {});
fs.rmSync(dataDir, { recursive: true, force: true });
},
};
} catch (error) {
await instance.stop().catch(() => {});
fs.rmSync(dataDir, { recursive: true, force: true });
throw new Error(
`Failed to start embedded PostgreSQL test database: ${formatEmbeddedPostgresError(error)}`,
);
}
}

View file

@ -3,6 +3,12 @@ set -euo pipefail
base_cwd="${PAPERCLIP_WORKSPACE_BASE_CWD:?PAPERCLIP_WORKSPACE_BASE_CWD is required}"
worktree_cwd="${PAPERCLIP_WORKSPACE_CWD:?PAPERCLIP_WORKSPACE_CWD is required}"
paperclip_home="${PAPERCLIP_HOME:-$HOME/.paperclip}"
paperclip_instance_id="${PAPERCLIP_INSTANCE_ID:-default}"
paperclip_dir="$worktree_cwd/.paperclip"
worktree_config_path="$paperclip_dir/config.json"
worktree_env_path="$paperclip_dir/.env"
worktree_name="${PAPERCLIP_WORKSPACE_BRANCH:-$(basename "$worktree_cwd")}"
if [[ ! -d "$base_cwd" ]]; then
echo "Base workspace does not exist: $base_cwd" >&2
@ -14,6 +20,286 @@ if [[ ! -d "$worktree_cwd" ]]; then
exit 1
fi
source_config_path="${PAPERCLIP_CONFIG:-}"
if [[ -z "$source_config_path" && ( -e "$base_cwd/.paperclip/config.json" || -L "$base_cwd/.paperclip/config.json" ) ]]; then
source_config_path="$base_cwd/.paperclip/config.json"
fi
if [[ -z "$source_config_path" ]]; then
source_config_path="$paperclip_home/instances/$paperclip_instance_id/config.json"
fi
source_env_path="$(dirname "$source_config_path")/.env"
mkdir -p "$paperclip_dir"
run_isolated_worktree_init() {
if command -v paperclipai >/dev/null 2>&1; then
paperclipai worktree init --force --name "$worktree_name" --from-config "$source_config_path"
return 0
fi
if command -v pnpm >/dev/null 2>&1 && pnpm paperclipai --help >/dev/null 2>&1; then
pnpm paperclipai worktree init --force --name "$worktree_name" --from-config "$source_config_path"
return 0
fi
return 1
}
write_fallback_worktree_config() {
WORKTREE_NAME="$worktree_name" \
BASE_CWD="$base_cwd" \
WORKTREE_CWD="$worktree_cwd" \
PAPERCLIP_DIR="$paperclip_dir" \
SOURCE_CONFIG_PATH="$source_config_path" \
SOURCE_ENV_PATH="$source_env_path" \
PAPERCLIP_WORKTREES_DIR="${PAPERCLIP_WORKTREES_DIR:-}" \
node <<'EOF'
const fs = require("node:fs");
const os = require("node:os");
const path = require("node:path");
const net = require("node:net");
function expandHomePrefix(value) {
if (!value) return value;
if (value === "~") return os.homedir();
if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2));
return value;
}
function nonEmpty(value) {
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}
function sanitizeInstanceId(value) {
const trimmed = String(value ?? "").trim().toLowerCase();
const normalized = trimmed
.replace(/[^a-z0-9_-]+/g, "-")
.replace(/-+/g, "-")
.replace(/^[-_]+|[-_]+$/g, "");
return normalized || "worktree";
}
function parseEnvFile(contents) {
const entries = {};
for (const rawLine of contents.split(/\r?\n/)) {
const line = rawLine.trim();
if (!line || line.startsWith("#")) continue;
const match = rawLine.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$/);
if (!match) continue;
const [, key, rawValue] = match;
const value = rawValue.trim();
if (!value) {
entries[key] = "";
continue;
}
if (
(value.startsWith("\"") && value.endsWith("\"")) ||
(value.startsWith("'") && value.endsWith("'"))
) {
entries[key] = value.slice(1, -1);
continue;
}
entries[key] = value.replace(/\s+#.*$/, "").trim();
}
return entries;
}
async function findAvailablePort(preferredPort, reserved = new Set()) {
const startPort = Number.isFinite(preferredPort) && preferredPort > 0 ? Math.trunc(preferredPort) : 0;
if (startPort > 0) {
for (let port = startPort; port < startPort + 100; port += 1) {
if (reserved.has(port)) continue;
const available = await new Promise((resolve) => {
const server = net.createServer();
server.unref();
server.once("error", () => resolve(false));
server.listen(port, "127.0.0.1", () => {
server.close(() => resolve(true));
});
});
if (available) return port;
}
}
return await new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.once("error", reject);
server.listen(0, "127.0.0.1", () => {
const address = server.address();
if (!address || typeof address === "string") {
server.close(() => reject(new Error("Failed to allocate a port.")));
return;
}
const port = address.port;
server.close(() => resolve(port));
});
});
}
function isLoopbackHost(hostname) {
const value = hostname.trim().toLowerCase();
return value === "127.0.0.1" || value === "localhost" || value === "::1";
}
function rewriteLocalUrlPort(rawUrl, port) {
if (!rawUrl) return undefined;
try {
const parsed = new URL(rawUrl);
if (!isLoopbackHost(parsed.hostname)) return rawUrl;
parsed.port = String(port);
return parsed.toString();
} catch {
return rawUrl;
}
}
function resolveRuntimeLikePath(value, configPath) {
const expanded = expandHomePrefix(value);
if (path.isAbsolute(expanded)) return expanded;
return path.resolve(path.dirname(configPath), expanded);
}
async function main() {
const worktreeName = process.env.WORKTREE_NAME;
const paperclipDir = process.env.PAPERCLIP_DIR;
const sourceConfigPath = process.env.SOURCE_CONFIG_PATH;
const sourceEnvPath = process.env.SOURCE_ENV_PATH;
const worktreeHome = path.resolve(expandHomePrefix(nonEmpty(process.env.PAPERCLIP_WORKTREES_DIR) ?? "~/.paperclip-worktrees"));
const instanceId = sanitizeInstanceId(worktreeName);
const instanceRoot = path.resolve(worktreeHome, "instances", instanceId);
const configPath = path.resolve(paperclipDir, "config.json");
const envPath = path.resolve(paperclipDir, ".env");
let sourceConfig = null;
if (sourceConfigPath && fs.existsSync(sourceConfigPath)) {
sourceConfig = JSON.parse(fs.readFileSync(sourceConfigPath, "utf8"));
}
const sourceEnvEntries =
sourceEnvPath && fs.existsSync(sourceEnvPath)
? parseEnvFile(fs.readFileSync(sourceEnvPath, "utf8"))
: {};
const preferredServerPort = Number(sourceConfig?.server?.port ?? 3101) + 1;
const serverPort = await findAvailablePort(preferredServerPort);
const preferredDbPort = Number(sourceConfig?.database?.embeddedPostgresPort ?? 54329) + 1;
const databasePort = await findAvailablePort(preferredDbPort, new Set([serverPort]));
fs.rmSync(configPath, { force: true });
fs.mkdirSync(path.dirname(configPath), { recursive: true });
fs.mkdirSync(instanceRoot, { recursive: true });
const authPublicBaseUrl = rewriteLocalUrlPort(sourceConfig?.auth?.publicBaseUrl, serverPort);
const targetConfig = {
$meta: {
version: 1,
updatedAt: new Date().toISOString(),
source: "configure",
},
...(sourceConfig?.llm ? { llm: sourceConfig.llm } : {}),
database: {
mode: "embedded-postgres",
embeddedPostgresDataDir: path.resolve(instanceRoot, "db"),
embeddedPostgresPort: databasePort,
backup: {
enabled: sourceConfig?.database?.backup?.enabled ?? true,
intervalMinutes: sourceConfig?.database?.backup?.intervalMinutes ?? 60,
retentionDays: sourceConfig?.database?.backup?.retentionDays ?? 30,
dir: path.resolve(instanceRoot, "data", "backups"),
},
},
logging: {
mode: sourceConfig?.logging?.mode ?? "file",
logDir: path.resolve(instanceRoot, "logs"),
},
server: {
deploymentMode: sourceConfig?.server?.deploymentMode ?? "local_trusted",
exposure: sourceConfig?.server?.exposure ?? "private",
host: sourceConfig?.server?.host ?? "127.0.0.1",
port: serverPort,
allowedHostnames: sourceConfig?.server?.allowedHostnames ?? [],
serveUi: sourceConfig?.server?.serveUi ?? true,
},
auth: {
baseUrlMode: sourceConfig?.auth?.baseUrlMode ?? "auto",
...(authPublicBaseUrl ? { publicBaseUrl: authPublicBaseUrl } : {}),
disableSignUp: sourceConfig?.auth?.disableSignUp ?? false,
},
storage: {
provider: sourceConfig?.storage?.provider ?? "local_disk",
localDisk: {
baseDir: path.resolve(instanceRoot, "data", "storage"),
},
s3: {
bucket: sourceConfig?.storage?.s3?.bucket ?? "paperclip",
region: sourceConfig?.storage?.s3?.region ?? "us-east-1",
endpoint: sourceConfig?.storage?.s3?.endpoint,
prefix: sourceConfig?.storage?.s3?.prefix ?? "",
forcePathStyle: sourceConfig?.storage?.s3?.forcePathStyle ?? false,
},
},
secrets: {
provider: sourceConfig?.secrets?.provider ?? "local_encrypted",
strictMode: sourceConfig?.secrets?.strictMode ?? false,
localEncrypted: {
keyFilePath: path.resolve(instanceRoot, "secrets", "master.key"),
},
},
};
fs.writeFileSync(configPath, `${JSON.stringify(targetConfig, null, 2)}\n`, { mode: 0o600 });
const inlineMasterKey = nonEmpty(sourceEnvEntries.PAPERCLIP_SECRETS_MASTER_KEY);
if (inlineMasterKey) {
fs.mkdirSync(path.resolve(instanceRoot, "secrets"), { recursive: true });
fs.writeFileSync(targetConfig.secrets.localEncrypted.keyFilePath, inlineMasterKey, {
encoding: "utf8",
mode: 0o600,
});
} else {
const sourceKeyFilePath = nonEmpty(sourceEnvEntries.PAPERCLIP_SECRETS_MASTER_KEY_FILE)
? resolveRuntimeLikePath(sourceEnvEntries.PAPERCLIP_SECRETS_MASTER_KEY_FILE, sourceConfigPath)
: nonEmpty(sourceConfig?.secrets?.localEncrypted?.keyFilePath)
? resolveRuntimeLikePath(sourceConfig.secrets.localEncrypted.keyFilePath, sourceConfigPath)
: null;
if (sourceKeyFilePath && fs.existsSync(sourceKeyFilePath)) {
fs.mkdirSync(path.resolve(instanceRoot, "secrets"), { recursive: true });
fs.copyFileSync(sourceKeyFilePath, targetConfig.secrets.localEncrypted.keyFilePath);
fs.chmodSync(targetConfig.secrets.localEncrypted.keyFilePath, 0o600);
}
}
const envLines = [
"PAPERCLIP_HOME=" + JSON.stringify(worktreeHome),
"PAPERCLIP_INSTANCE_ID=" + JSON.stringify(instanceId),
"PAPERCLIP_CONFIG=" + JSON.stringify(configPath),
"PAPERCLIP_CONTEXT=" + JSON.stringify(path.resolve(worktreeHome, "context.json")),
"PAPERCLIP_IN_WORKTREE=true",
"PAPERCLIP_WORKTREE_NAME=" + JSON.stringify(worktreeName),
];
const agentJwtSecret = nonEmpty(sourceEnvEntries.PAPERCLIP_AGENT_JWT_SECRET);
if (agentJwtSecret) {
envLines.push("PAPERCLIP_AGENT_JWT_SECRET=" + JSON.stringify(agentJwtSecret));
}
fs.writeFileSync(envPath, `${envLines.join("\n")}\n`, { mode: 0o600 });
}
main().catch((error) => {
console.error(error instanceof Error ? error.message : String(error));
process.exit(1);
});
EOF
}
if ! run_isolated_worktree_init; then
echo "paperclipai CLI not available in this workspace; writing isolated fallback config without DB seeding." >&2
write_fallback_worktree_config
fi
while IFS= read -r relative_path; do
[[ -n "$relative_path" ]] || continue
source_path="$base_cwd/$relative_path"

View file

@ -33,7 +33,7 @@
],
"scripts": {
"dev": "tsx src/index.ts",
"dev:watch": "cross-env PAPERCLIP_MIGRATION_PROMPT=never PAPERCLIP_MIGRATION_AUTO_APPLY=true tsx watch --ignore ../ui/node_modules --ignore ../ui/.vite --ignore ../ui/dist src/index.ts",
"dev:watch": "cross-env PAPERCLIP_MIGRATION_PROMPT=never PAPERCLIP_MIGRATION_AUTO_APPLY=true tsx ./scripts/dev-watch.ts",
"prepare:ui-dist": "bash ../scripts/prepare-server-ui-dist.sh",
"build": "tsc && mkdir -p dist/onboarding-assets && cp -R src/onboarding-assets/. dist/onboarding-assets/",
"prepack": "pnpm run prepare:ui-dist",

View file

@ -0,0 +1,33 @@
import { spawn } from "node:child_process";
import { createRequire } from "node:module";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { resolveServerDevWatchIgnorePaths } from "../src/dev-watch-ignore.ts";
const require = createRequire(import.meta.url);
const tsxCliPath = require.resolve("tsx/dist/cli.mjs");
const serverRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const ignoreArgs = resolveServerDevWatchIgnorePaths(serverRoot).flatMap((ignorePath) => ["--ignore", ignorePath]);
const child = spawn(
process.execPath,
[tsxCliPath, "watch", ...ignoreArgs, "src/index.ts"],
{
cwd: serverRoot,
env: process.env,
stdio: "inherit",
},
);
child.on("exit", (code, signal) => {
if (signal) {
process.kill(process.pid, signal);
return;
}
process.exit(code ?? 0);
});
child.on("error", (error) => {
console.error(error);
process.exit(1);
});

View file

@ -0,0 +1,34 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { resolveServerDevWatchIgnorePaths } from "../dev-watch-ignore.js";
describe("resolveServerDevWatchIgnorePaths", () => {
it("includes both the worktree UI paths and their real shared targets", () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-dev-watch-"));
const sharedUiRoot = path.join(tempRoot, "shared-ui");
const worktreeRoot = path.join(tempRoot, "repo", ".paperclip", "worktrees", "PAP-884");
const serverRoot = path.join(worktreeRoot, "server");
const worktreeUiRoot = path.join(worktreeRoot, "ui");
fs.mkdirSync(path.join(sharedUiRoot, "node_modules"), { recursive: true });
fs.mkdirSync(path.join(sharedUiRoot, ".vite"), { recursive: true });
fs.mkdirSync(path.join(sharedUiRoot, "dist"), { recursive: true });
fs.mkdirSync(serverRoot, { recursive: true });
fs.mkdirSync(worktreeUiRoot, { recursive: true });
fs.symlinkSync(path.join(sharedUiRoot, "node_modules"), path.join(worktreeUiRoot, "node_modules"));
fs.symlinkSync(path.join(sharedUiRoot, ".vite"), path.join(worktreeUiRoot, ".vite"));
fs.symlinkSync(path.join(sharedUiRoot, "dist"), path.join(worktreeUiRoot, "dist"));
const ignorePaths = resolveServerDevWatchIgnorePaths(serverRoot);
expect(ignorePaths).toContain(path.join(worktreeUiRoot, "node_modules"));
expect(ignorePaths).toContain(fs.realpathSync(path.join(sharedUiRoot, "node_modules")));
expect(ignorePaths).toContain(path.join(worktreeUiRoot, ".vite"));
expect(ignorePaths).toContain(fs.realpathSync(path.join(sharedUiRoot, ".vite")));
expect(ignorePaths).toContain(path.join(worktreeUiRoot, "dist"));
expect(ignorePaths).toContain(fs.realpathSync(path.join(sharedUiRoot, "dist")));
});
});

View file

@ -1,89 +1,29 @@
import { randomUUID } from "node:crypto";
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import { spawn, type ChildProcess } from "node:child_process";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import {
applyPendingMigrations,
createDb,
ensurePostgresDatabase,
agents,
agentWakeupRequests,
companies,
createDb,
heartbeatRunEvents,
heartbeatRuns,
issues,
} from "@paperclipai/db";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
import { runningProcesses } from "../adapters/index.ts";
import { heartbeatService } from "../services/heartbeat.ts";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
type EmbeddedPostgresInstance = {
initialise(): Promise<void>;
start(): Promise<void>;
stop(): Promise<void>;
};
type EmbeddedPostgresCtor = new (opts: {
databaseDir: string;
user: string;
password: string;
port: number;
persistent: boolean;
initdbFlags?: string[];
onLog?: (message: unknown) => void;
onError?: (message: unknown) => void;
}) => EmbeddedPostgresInstance;
async function getEmbeddedPostgresCtor(): Promise<EmbeddedPostgresCtor> {
const mod = await import("embedded-postgres");
return mod.default as EmbeddedPostgresCtor;
}
async function getAvailablePort(): Promise<number> {
return await new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on("error", reject);
server.listen(0, "127.0.0.1", () => {
const address = server.address();
if (!address || typeof address === "string") {
server.close(() => reject(new Error("Failed to allocate test port")));
return;
}
const { port } = address;
server.close((error) => {
if (error) reject(error);
else resolve(port);
});
});
});
}
async function startTempDatabase() {
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-heartbeat-recovery-"));
const port = await getAvailablePort();
const EmbeddedPostgres = await getEmbeddedPostgresCtor();
const instance = new EmbeddedPostgres({
databaseDir: dataDir,
user: "paperclip",
password: "paperclip",
port,
persistent: true,
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
onLog: () => {},
onError: () => {},
});
await instance.initialise();
await instance.start();
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`;
await ensurePostgresDatabase(adminConnectionString, "paperclip");
const connectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
await applyPendingMigrations(connectionString);
return { connectionString, instance, dataDir };
if (!embeddedPostgresSupport.supported) {
console.warn(
`Skipping embedded Postgres heartbeat recovery tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
);
}
function spawnAliveProcess() {
@ -92,17 +32,14 @@ function spawnAliveProcess() {
});
}
describe("heartbeat orphaned process recovery", () => {
describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
let db!: ReturnType<typeof createDb>;
let instance: EmbeddedPostgresInstance | null = null;
let dataDir = "";
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
const childProcesses = new Set<ChildProcess>();
beforeAll(async () => {
const started = await startTempDatabase();
db = createDb(started.connectionString);
instance = started.instance;
dataDir = started.dataDir;
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-recovery-");
db = createDb(tempDb.connectionString);
}, 20_000);
afterEach(async () => {
@ -125,10 +62,7 @@ describe("heartbeat orphaned process recovery", () => {
}
childProcesses.clear();
runningProcesses.clear();
await instance?.stop();
if (dataDir) {
fs.rmSync(dataDir, { recursive: true, force: true });
}
await tempDb?.cleanup();
});
async function seedRunFixture(input?: {

View file

@ -0,0 +1,6 @@
export {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
type EmbeddedPostgresTestDatabase,
type EmbeddedPostgresTestSupport,
} from "@paperclipai/db";

View file

@ -1,99 +1,37 @@
import { randomUUID } from "node:crypto";
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import {
activityLog,
agents,
applyPendingMigrations,
companies,
createDb,
ensurePostgresDatabase,
issueComments,
issues,
} from "@paperclipai/db";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
import { issueService } from "../services/issues.ts";
type EmbeddedPostgresInstance = {
initialise(): Promise<void>;
start(): Promise<void>;
stop(): Promise<void>;
};
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
type EmbeddedPostgresCtor = new (opts: {
databaseDir: string;
user: string;
password: string;
port: number;
persistent: boolean;
initdbFlags?: string[];
onLog?: (message: unknown) => void;
onError?: (message: unknown) => void;
}) => EmbeddedPostgresInstance;
async function getEmbeddedPostgresCtor(): Promise<EmbeddedPostgresCtor> {
const mod = await import("embedded-postgres");
return mod.default as EmbeddedPostgresCtor;
if (!embeddedPostgresSupport.supported) {
console.warn(
`Skipping embedded Postgres issue service tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
);
}
async function getAvailablePort(): Promise<number> {
return await new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on("error", reject);
server.listen(0, "127.0.0.1", () => {
const address = server.address();
if (!address || typeof address === "string") {
server.close(() => reject(new Error("Failed to allocate test port")));
return;
}
const { port } = address;
server.close((error) => {
if (error) reject(error);
else resolve(port);
});
});
});
}
async function startTempDatabase() {
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-issues-service-"));
const port = await getAvailablePort();
const EmbeddedPostgres = await getEmbeddedPostgresCtor();
const instance = new EmbeddedPostgres({
databaseDir: dataDir,
user: "paperclip",
password: "paperclip",
port,
persistent: true,
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
onLog: () => {},
onError: () => {},
});
await instance.initialise();
await instance.start();
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`;
await ensurePostgresDatabase(adminConnectionString, "paperclip");
const connectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
await applyPendingMigrations(connectionString);
return { connectionString, dataDir, instance };
}
describe("issueService.list participantAgentId", () => {
describeEmbeddedPostgres("issueService.list participantAgentId", () => {
let db!: ReturnType<typeof createDb>;
let svc!: ReturnType<typeof issueService>;
let instance: EmbeddedPostgresInstance | null = null;
let dataDir = "";
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
beforeAll(async () => {
const started = await startTempDatabase();
db = createDb(started.connectionString);
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issues-service-");
db = createDb(tempDb.connectionString);
svc = issueService(db);
instance = started.instance;
dataDir = started.dataDir;
}, 20_000);
afterEach(async () => {
@ -105,10 +43,7 @@ describe("issueService.list participantAgentId", () => {
});
afterAll(async () => {
await instance?.stop();
if (dataDir) {
fs.rmSync(dataDir, { recursive: true, force: true });
}
await tempDb?.cleanup();
});
it("returns issues an agent participated in across the supported signals", async () => {

View file

@ -1,8 +1,4 @@
import { randomUUID } from "node:crypto";
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import { eq } from "drizzle-orm";
import express from "express";
import request from "supertest";
@ -11,11 +7,9 @@ import {
activityLog,
agentWakeupRequests,
agents,
applyPendingMigrations,
companies,
companyMemberships,
createDb,
ensurePostgresDatabase,
heartbeatRunEvents,
heartbeatRuns,
instanceSettings,
@ -26,6 +20,10 @@ import {
routines,
routineTriggers,
} from "@paperclipai/db";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
import { errorHandler } from "../middleware/index.js";
import { accessService } from "../services/access.js";
@ -78,82 +76,22 @@ vi.mock("../services/index.js", async () => {
};
});
type EmbeddedPostgresInstance = {
initialise(): Promise<void>;
start(): Promise<void>;
stop(): Promise<void>;
};
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
type EmbeddedPostgresCtor = new (opts: {
databaseDir: string;
user: string;
password: string;
port: number;
persistent: boolean;
initdbFlags?: string[];
onLog?: (message: unknown) => void;
onError?: (message: unknown) => void;
}) => EmbeddedPostgresInstance;
async function getEmbeddedPostgresCtor(): Promise<EmbeddedPostgresCtor> {
const mod = await import("embedded-postgres");
return mod.default as EmbeddedPostgresCtor;
if (!embeddedPostgresSupport.supported) {
console.warn(
`Skipping embedded Postgres routine route tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
);
}
async function getAvailablePort(): Promise<number> {
return await new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on("error", reject);
server.listen(0, "127.0.0.1", () => {
const address = server.address();
if (!address || typeof address === "string") {
server.close(() => reject(new Error("Failed to allocate test port")));
return;
}
const { port } = address;
server.close((error) => {
if (error) reject(error);
else resolve(port);
});
});
});
}
async function startTempDatabase() {
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-routines-e2e-"));
const port = await getAvailablePort();
const EmbeddedPostgres = await getEmbeddedPostgresCtor();
const instance = new EmbeddedPostgres({
databaseDir: dataDir,
user: "paperclip",
password: "paperclip",
port,
persistent: true,
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
onLog: () => {},
onError: () => {},
});
await instance.initialise();
await instance.start();
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`;
await ensurePostgresDatabase(adminConnectionString, "paperclip");
const connectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
await applyPendingMigrations(connectionString);
return { connectionString, dataDir, instance };
}
describe("routine routes end-to-end", () => {
describeEmbeddedPostgres("routine routes end-to-end", () => {
let db!: ReturnType<typeof createDb>;
let instance: EmbeddedPostgresInstance | null = null;
let dataDir = "";
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
beforeAll(async () => {
const started = await startTempDatabase();
db = createDb(started.connectionString);
instance = started.instance;
dataDir = started.dataDir;
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-routines-e2e-");
db = createDb(tempDb.connectionString);
}, 20_000);
afterEach(async () => {
@ -174,10 +112,7 @@ describe("routine routes end-to-end", () => {
});
afterAll(async () => {
await instance?.stop();
if (dataDir) {
fs.rmSync(dataDir, { recursive: true, force: true });
}
await tempDb?.cleanup();
});
async function createApp(actor: Record<string, unknown>) {

View file

@ -1,19 +1,13 @@
import { createHmac, randomUUID } from "node:crypto";
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import {
activityLog,
agents,
applyPendingMigrations,
companies,
companySecrets,
companySecretVersions,
createDb,
ensurePostgresDatabase,
heartbeatRuns,
issues,
projects,
@ -21,85 +15,29 @@ import {
routines,
routineTriggers,
} from "@paperclipai/db";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
import { issueService } from "../services/issues.ts";
import { routineService } from "../services/routines.ts";
type EmbeddedPostgresInstance = {
initialise(): Promise<void>;
start(): Promise<void>;
stop(): Promise<void>;
};
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
type EmbeddedPostgresCtor = new (opts: {
databaseDir: string;
user: string;
password: string;
port: number;
persistent: boolean;
initdbFlags?: string[];
onLog?: (message: unknown) => void;
onError?: (message: unknown) => void;
}) => EmbeddedPostgresInstance;
async function getEmbeddedPostgresCtor(): Promise<EmbeddedPostgresCtor> {
const mod = await import("embedded-postgres");
return mod.default as EmbeddedPostgresCtor;
if (!embeddedPostgresSupport.supported) {
console.warn(
`Skipping embedded Postgres routines service tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
);
}
async function getAvailablePort(): Promise<number> {
return await new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on("error", reject);
server.listen(0, "127.0.0.1", () => {
const address = server.address();
if (!address || typeof address === "string") {
server.close(() => reject(new Error("Failed to allocate test port")));
return;
}
const { port } = address;
server.close((error) => {
if (error) reject(error);
else resolve(port);
});
});
});
}
async function startTempDatabase() {
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-routines-service-"));
const port = await getAvailablePort();
const EmbeddedPostgres = await getEmbeddedPostgresCtor();
const instance = new EmbeddedPostgres({
databaseDir: dataDir,
user: "paperclip",
password: "paperclip",
port,
persistent: true,
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
onLog: () => {},
onError: () => {},
});
await instance.initialise();
await instance.start();
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`;
await ensurePostgresDatabase(adminConnectionString, "paperclip");
const connectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
await applyPendingMigrations(connectionString);
return { connectionString, dataDir, instance };
}
describe("routine service live-execution coalescing", () => {
describeEmbeddedPostgres("routine service live-execution coalescing", () => {
let db!: ReturnType<typeof createDb>;
let instance: EmbeddedPostgresInstance | null = null;
let dataDir = "";
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
beforeAll(async () => {
const started = await startTempDatabase();
db = createDb(started.connectionString);
instance = started.instance;
dataDir = started.dataDir;
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-routines-service-");
db = createDb(tempDb.connectionString);
}, 20_000);
afterEach(async () => {
@ -117,10 +55,7 @@ describe("routine service live-execution coalescing", () => {
});
afterAll(async () => {
await instance?.stop();
if (dataDir) {
fs.rmSync(dataDir, { recursive: true, force: true });
}
await tempDb?.cleanup();
});
async function seedFixture(opts?: {

View file

@ -2,6 +2,7 @@ import { execFile } from "node:child_process";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { afterEach, describe, expect, it } from "vitest";
import {
@ -13,6 +14,7 @@ import {
stopRuntimeServicesForExecutionWorkspace,
type RealizedExecutionWorkspace,
} from "../services/workspace-runtime.ts";
import { resolvePaperclipConfigPath } from "../paths.ts";
import type { WorkspaceOperation } from "@paperclipai/shared";
import type { WorkspaceOperationRecorder } from "../services/workspace-operations.ts";
@ -124,6 +126,7 @@ afterEach(async () => {
delete process.env.PAPERCLIP_CONFIG;
delete process.env.PAPERCLIP_HOME;
delete process.env.PAPERCLIP_INSTANCE_ID;
delete process.env.PAPERCLIP_WORKTREES_DIR;
delete process.env.DATABASE_URL;
});
@ -282,6 +285,156 @@ describe("realizeExecutionWorkspace", () => {
await expect(fs.readFile(path.join(reused.cwd, ".paperclip-provision-created"), "utf8")).resolves.toBe("false\n");
});
it("writes an isolated repo-local Paperclip config and worktree branding when provisioning", async () => {
const repoRoot = await createTempRepo();
const previousCwd = process.cwd();
const paperclipHome = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-worktree-home-"));
const isolatedWorktreeHome = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-worktrees-"));
const instanceId = "worktree-base";
const sharedConfigDir = path.join(paperclipHome, "instances", instanceId);
const sharedConfigPath = path.join(sharedConfigDir, "config.json");
const sharedEnvPath = path.join(sharedConfigDir, ".env");
process.env.PAPERCLIP_HOME = paperclipHome;
process.env.PAPERCLIP_INSTANCE_ID = instanceId;
process.env.PAPERCLIP_WORKTREES_DIR = isolatedWorktreeHome;
await fs.mkdir(sharedConfigDir, { recursive: true });
await fs.writeFile(
sharedConfigPath,
JSON.stringify(
{
$meta: {
version: 1,
updatedAt: "2026-03-26T00:00:00.000Z",
source: "doctor",
},
database: {
mode: "embedded-postgres",
embeddedPostgresDataDir: path.join(sharedConfigDir, "db"),
embeddedPostgresPort: 54329,
backup: {
enabled: true,
intervalMinutes: 60,
retentionDays: 30,
dir: path.join(sharedConfigDir, "backups"),
},
},
logging: {
mode: "file",
logDir: path.join(sharedConfigDir, "logs"),
},
server: {
deploymentMode: "local_trusted",
exposure: "private",
host: "127.0.0.1",
port: 3100,
allowedHostnames: [],
serveUi: true,
},
auth: {
baseUrlMode: "auto",
disableSignUp: false,
},
storage: {
provider: "local_disk",
localDisk: {
baseDir: path.join(sharedConfigDir, "storage"),
},
s3: {
bucket: "paperclip",
region: "us-east-1",
prefix: "",
forcePathStyle: false,
},
},
secrets: {
provider: "local_encrypted",
strictMode: false,
localEncrypted: {
keyFilePath: path.join(sharedConfigDir, "master.key"),
},
},
},
null,
2,
) + "\n",
"utf8",
);
await fs.writeFile(sharedEnvPath, 'DATABASE_URL="postgres://worktree:test@db.example.com:6543/paperclip"\n', "utf8");
await fs.mkdir(path.join(repoRoot, "scripts"), { recursive: true });
await fs.copyFile(
fileURLToPath(new URL("../../../scripts/provision-worktree.sh", import.meta.url)),
path.join(repoRoot, "scripts", "provision-worktree.sh"),
);
await runGit(repoRoot, ["add", "scripts/provision-worktree.sh"]);
await runGit(repoRoot, ["commit", "-m", "Add worktree provision script"]);
try {
const workspace = await realizeExecutionWorkspace({
base: {
baseCwd: repoRoot,
source: "project_primary",
projectId: "project-1",
workspaceId: "workspace-1",
repoUrl: null,
repoRef: "HEAD",
},
config: {
workspaceStrategy: {
type: "git_worktree",
branchTemplate: "{{issue.identifier}}-{{slug}}",
provisionCommand: "bash ./scripts/provision-worktree.sh",
},
},
issue: {
id: "issue-1",
identifier: "PAP-885",
title: "Show worktree banner",
},
agent: {
id: "agent-1",
name: "Codex Coder",
companyId: "company-1",
},
});
const configPath = path.join(workspace.cwd, ".paperclip", "config.json");
const envPath = path.join(workspace.cwd, ".paperclip", ".env");
const envContents = await fs.readFile(envPath, "utf8");
const configContents = JSON.parse(await fs.readFile(configPath, "utf8"));
const configStats = await fs.lstat(configPath);
const expectedInstanceId = "pap-885-show-worktree-banner";
const expectedInstanceRoot = path.join(
isolatedWorktreeHome,
"instances",
expectedInstanceId,
);
expect(configStats.isSymbolicLink()).toBe(false);
expect(configContents.database.embeddedPostgresDataDir).toBe(path.join(expectedInstanceRoot, "db"));
expect(configContents.database.embeddedPostgresDataDir).not.toBe(path.join(sharedConfigDir, "db"));
expect(configContents.server.port).not.toBe(3100);
expect(configContents.secrets.localEncrypted.keyFilePath).toBe(
path.join(expectedInstanceRoot, "secrets", "master.key"),
);
expect(envContents).not.toContain("DATABASE_URL=");
expect(envContents).toContain(`PAPERCLIP_HOME=${JSON.stringify(isolatedWorktreeHome)}`);
expect(envContents).toContain(`PAPERCLIP_INSTANCE_ID=${JSON.stringify(expectedInstanceId)}`);
expect(envContents).toContain(`PAPERCLIP_CONFIG=${JSON.stringify(configPath)}`);
expect(envContents).toContain("PAPERCLIP_IN_WORKTREE=true");
expect(envContents).toContain(
`PAPERCLIP_WORKTREE_NAME=${JSON.stringify("PAP-885-show-worktree-banner")}`,
);
process.chdir(workspace.cwd);
expect(resolvePaperclipConfigPath()).toBe(configPath);
} finally {
process.chdir(previousCwd);
}
});
it("records worktree setup and provision operations when a recorder is provided", async () => {
const repoRoot = await createTempRepo();
const { recorder, operations } = createWorkspaceOperationRecorderDouble();

View file

@ -0,0 +1,426 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import {
applyRuntimePortSelectionToConfig,
maybePersistWorktreeRuntimePorts,
maybeRepairLegacyWorktreeConfigAndEnvFiles,
} from "../worktree-config.js";
const ORIGINAL_ENV = { ...process.env };
const ORIGINAL_CWD = process.cwd();
afterEach(() => {
process.chdir(ORIGINAL_CWD);
for (const key of Object.keys(process.env)) {
if (!(key in ORIGINAL_ENV)) {
delete process.env[key];
}
}
for (const [key, value] of Object.entries(ORIGINAL_ENV)) {
process.env[key] = value;
}
});
function buildLegacyConfig(sharedRoot: string) {
return {
$meta: {
version: 1,
updatedAt: "2026-03-26T00:00:00.000Z",
source: "configure",
},
database: {
mode: "embedded-postgres" as const,
embeddedPostgresDataDir: path.join(sharedRoot, "db"),
embeddedPostgresPort: 54329,
backup: {
enabled: true,
intervalMinutes: 60,
retentionDays: 30,
dir: path.join(sharedRoot, "data", "backups"),
},
},
logging: {
mode: "file" as const,
logDir: path.join(sharedRoot, "logs"),
},
server: {
deploymentMode: "local_trusted" as const,
exposure: "private" as const,
host: "127.0.0.1",
port: 3100,
allowedHostnames: [],
serveUi: true,
},
auth: {
baseUrlMode: "explicit" as const,
publicBaseUrl: "http://127.0.0.1:3100",
disableSignUp: false,
},
storage: {
provider: "local_disk" as const,
localDisk: {
baseDir: path.join(sharedRoot, "data", "storage"),
},
s3: {
bucket: "paperclip",
region: "us-east-1",
prefix: "",
forcePathStyle: false,
},
},
secrets: {
provider: "local_encrypted" as const,
strictMode: false,
localEncrypted: {
keyFilePath: path.join(sharedRoot, "secrets", "master.key"),
},
},
};
}
describe("worktree config repair", () => {
it("repairs legacy repo-local worktree config and env files into an isolated instance", async () => {
const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-worktree-repair-"));
const worktreeRoot = path.join(tempRoot, "PAP-884-ai-commits-component");
const paperclipDir = path.join(worktreeRoot, ".paperclip");
const configPath = path.join(paperclipDir, "config.json");
const envPath = path.join(paperclipDir, ".env");
const sharedRoot = path.join(tempRoot, ".paperclip", "instances", "default");
const isolatedHome = path.join(tempRoot, ".paperclip-worktrees");
await fs.mkdir(paperclipDir, { recursive: true });
await fs.writeFile(configPath, JSON.stringify(buildLegacyConfig(sharedRoot), null, 2) + "\n", "utf8");
await fs.writeFile(
envPath,
[
"# Paperclip environment variables",
"PAPERCLIP_IN_WORKTREE=true",
"PAPERCLIP_WORKTREE_NAME=PAP-884-ai-commits-component",
"PAPERCLIP_AGENT_JWT_SECRET=shared-secret",
"",
].join("\n"),
"utf8",
);
process.chdir(worktreeRoot);
process.env.PAPERCLIP_IN_WORKTREE = "true";
process.env.PAPERCLIP_WORKTREE_NAME = "PAP-884-ai-commits-component";
process.env.PAPERCLIP_WORKTREES_DIR = isolatedHome;
delete process.env.PAPERCLIP_HOME;
delete process.env.PAPERCLIP_INSTANCE_ID;
delete process.env.PAPERCLIP_CONFIG;
delete process.env.PAPERCLIP_CONTEXT;
const result = maybeRepairLegacyWorktreeConfigAndEnvFiles();
expect(result).toEqual({
repairedConfig: true,
repairedEnv: true,
});
const repairedConfig = JSON.parse(await fs.readFile(configPath, "utf8"));
const repairedEnv = await fs.readFile(envPath, "utf8");
const instanceRoot = path.join(isolatedHome, "instances", "pap-884-ai-commits-component");
expect(repairedConfig.database.embeddedPostgresDataDir).toBe(path.join(instanceRoot, "db"));
expect(repairedConfig.database.backup.dir).toBe(path.join(instanceRoot, "data", "backups"));
expect(repairedConfig.logging.logDir).toBe(path.join(instanceRoot, "logs"));
expect(repairedConfig.storage.localDisk.baseDir).toBe(path.join(instanceRoot, "data", "storage"));
expect(repairedConfig.secrets.localEncrypted.keyFilePath).toBe(path.join(instanceRoot, "secrets", "master.key"));
expect(repairedEnv).toContain(`PAPERCLIP_HOME=${JSON.stringify(isolatedHome)}`);
expect(repairedEnv).toContain('PAPERCLIP_INSTANCE_ID="pap-884-ai-commits-component"');
expect(repairedEnv).toContain(`PAPERCLIP_CONFIG=${JSON.stringify(await fs.realpath(configPath))}`);
expect(repairedEnv).toContain(`PAPERCLIP_CONTEXT=${JSON.stringify(path.join(isolatedHome, "context.json"))}`);
expect(repairedEnv).toContain('PAPERCLIP_AGENT_JWT_SECRET="shared-secret"');
expect(process.env.PAPERCLIP_HOME).toBe(isolatedHome);
expect(process.env.PAPERCLIP_INSTANCE_ID).toBe("pap-884-ai-commits-component");
});
it("avoids sibling worktree ports when repairing legacy configs", async () => {
const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-worktree-repair-ports-"));
const worktreeRoot = path.join(tempRoot, "PAP-880-thumbs-capture-for-evals-feature");
const paperclipDir = path.join(worktreeRoot, ".paperclip");
const configPath = path.join(paperclipDir, "config.json");
const envPath = path.join(paperclipDir, ".env");
const sharedRoot = path.join(tempRoot, ".paperclip", "instances", "default");
const isolatedHome = path.join(tempRoot, ".paperclip-worktrees");
const siblingInstanceRoot = path.join(isolatedHome, "instances", "pap-878-create-a-mine-tab-in-inbox");
await fs.mkdir(paperclipDir, { recursive: true });
await fs.mkdir(siblingInstanceRoot, { recursive: true });
await fs.writeFile(configPath, JSON.stringify(buildLegacyConfig(sharedRoot), null, 2) + "\n", "utf8");
await fs.writeFile(
envPath,
[
"# Paperclip environment variables",
"PAPERCLIP_IN_WORKTREE=true",
"PAPERCLIP_WORKTREE_NAME=PAP-880-thumbs-capture-for-evals-feature",
"",
].join("\n"),
"utf8",
);
await fs.writeFile(
path.join(siblingInstanceRoot, "config.json"),
JSON.stringify(
{
...buildLegacyConfig(siblingInstanceRoot),
database: {
mode: "embedded-postgres",
embeddedPostgresDataDir: path.join(siblingInstanceRoot, "db"),
embeddedPostgresPort: 54330,
backup: {
enabled: true,
intervalMinutes: 60,
retentionDays: 30,
dir: path.join(siblingInstanceRoot, "data", "backups"),
},
},
server: {
deploymentMode: "local_trusted",
exposure: "private",
host: "127.0.0.1",
port: 3101,
allowedHostnames: [],
serveUi: true,
},
},
null,
2,
) + "\n",
"utf8",
);
process.chdir(worktreeRoot);
process.env.PAPERCLIP_IN_WORKTREE = "true";
process.env.PAPERCLIP_WORKTREE_NAME = "PAP-880-thumbs-capture-for-evals-feature";
process.env.PAPERCLIP_WORKTREES_DIR = isolatedHome;
const result = maybeRepairLegacyWorktreeConfigAndEnvFiles();
const repairedConfig = JSON.parse(await fs.readFile(configPath, "utf8"));
expect(result.repairedConfig).toBe(true);
expect(repairedConfig.server.port).toBe(3102);
expect(repairedConfig.database.embeddedPostgresPort).toBe(54331);
});
it("rebalances duplicate ports for already isolated worktree configs", async () => {
const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-worktree-rebalance-"));
const isolatedHome = path.join(tempRoot, ".paperclip-worktrees");
const repoWorktreesRoot = path.join(tempRoot, "repo", ".paperclip", "worktrees");
const siblingWorktreeRoot = path.join(repoWorktreesRoot, "PAP-878-create-a-mine-tab-in-inbox");
const siblingInstanceRoot = path.join(isolatedHome, "instances", "pap-878-create-a-mine-tab-in-inbox");
const currentWorktreeRoot = path.join(repoWorktreesRoot, "PAP-884-ai-commits-component");
const paperclipDir = path.join(currentWorktreeRoot, ".paperclip");
const configPath = path.join(paperclipDir, "config.json");
const envPath = path.join(paperclipDir, ".env");
const currentInstanceRoot = path.join(isolatedHome, "instances", "pap-884-ai-commits-component");
const siblingConfigPath = path.join(siblingWorktreeRoot, ".paperclip", "config.json");
await fs.mkdir(paperclipDir, { recursive: true });
await fs.mkdir(path.dirname(siblingConfigPath), { recursive: true });
await fs.writeFile(
configPath,
JSON.stringify(
{
...buildLegacyConfig(currentInstanceRoot),
database: {
mode: "embedded-postgres",
embeddedPostgresDataDir: path.join(currentInstanceRoot, "db"),
embeddedPostgresPort: 54330,
backup: {
enabled: true,
intervalMinutes: 60,
retentionDays: 30,
dir: path.join(currentInstanceRoot, "data", "backups"),
},
},
logging: {
mode: "file",
logDir: path.join(currentInstanceRoot, "logs"),
},
server: {
deploymentMode: "local_trusted",
exposure: "private",
host: "127.0.0.1",
port: 3101,
allowedHostnames: [],
serveUi: true,
},
storage: {
provider: "local_disk",
localDisk: {
baseDir: path.join(currentInstanceRoot, "data", "storage"),
},
s3: {
bucket: "paperclip",
region: "us-east-1",
prefix: "",
forcePathStyle: false,
},
},
secrets: {
provider: "local_encrypted",
strictMode: false,
localEncrypted: {
keyFilePath: path.join(currentInstanceRoot, "secrets", "master.key"),
},
},
},
null,
2,
) + "\n",
"utf8",
);
await fs.writeFile(
envPath,
[
"# Paperclip environment variables",
"PAPERCLIP_IN_WORKTREE=true",
"PAPERCLIP_WORKTREE_NAME=PAP-884-ai-commits-component",
"",
].join("\n"),
"utf8",
);
await fs.writeFile(
siblingConfigPath,
JSON.stringify(
{
...buildLegacyConfig(siblingInstanceRoot),
database: {
mode: "embedded-postgres",
embeddedPostgresDataDir: path.join(siblingInstanceRoot, "db"),
embeddedPostgresPort: 54330,
backup: {
enabled: true,
intervalMinutes: 60,
retentionDays: 30,
dir: path.join(siblingInstanceRoot, "data", "backups"),
},
},
server: {
deploymentMode: "local_trusted",
exposure: "private",
host: "127.0.0.1",
port: 3101,
allowedHostnames: [],
serveUi: true,
},
},
null,
2,
) + "\n",
"utf8",
);
process.chdir(currentWorktreeRoot);
process.env.PAPERCLIP_IN_WORKTREE = "true";
process.env.PAPERCLIP_WORKTREE_NAME = "PAP-884-ai-commits-component";
process.env.PAPERCLIP_WORKTREES_DIR = isolatedHome;
const result = maybeRepairLegacyWorktreeConfigAndEnvFiles();
const repairedConfig = JSON.parse(await fs.readFile(configPath, "utf8"));
expect(result.repairedConfig).toBe(true);
expect(repairedConfig.server.port).toBe(3102);
expect(repairedConfig.database.embeddedPostgresPort).toBe(54331);
});
it("persists runtime-selected worktree ports back into config", async () => {
const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-worktree-ports-"));
const worktreeRoot = path.join(tempRoot, "PAP-878-create-a-mine-tab-in-inbox");
const paperclipDir = path.join(worktreeRoot, ".paperclip");
const configPath = path.join(paperclipDir, "config.json");
const isolatedHome = path.join(tempRoot, ".paperclip-worktrees");
const instanceRoot = path.join(isolatedHome, "instances", "pap-878-create-a-mine-tab-in-inbox");
await fs.mkdir(paperclipDir, { recursive: true });
await fs.writeFile(
configPath,
JSON.stringify(
{
...buildLegacyConfig(instanceRoot),
database: {
mode: "embedded-postgres",
embeddedPostgresDataDir: path.join(instanceRoot, "db"),
embeddedPostgresPort: 54331,
backup: {
enabled: true,
intervalMinutes: 60,
retentionDays: 30,
dir: path.join(instanceRoot, "data", "backups"),
},
},
logging: {
mode: "file",
logDir: path.join(instanceRoot, "logs"),
},
server: {
deploymentMode: "local_trusted",
exposure: "private",
host: "127.0.0.1",
port: 3101,
allowedHostnames: [],
serveUi: true,
},
storage: {
provider: "local_disk",
localDisk: {
baseDir: path.join(instanceRoot, "data", "storage"),
},
s3: {
bucket: "paperclip",
region: "us-east-1",
prefix: "",
forcePathStyle: false,
},
},
secrets: {
provider: "local_encrypted",
strictMode: false,
localEncrypted: {
keyFilePath: path.join(instanceRoot, "secrets", "master.key"),
},
},
},
null,
2,
) + "\n",
"utf8",
);
process.chdir(worktreeRoot);
process.env.PAPERCLIP_IN_WORKTREE = "true";
process.env.PAPERCLIP_WORKTREE_NAME = "PAP-878-create-a-mine-tab-in-inbox";
process.env.PAPERCLIP_HOME = isolatedHome;
process.env.PAPERCLIP_INSTANCE_ID = "pap-878-create-a-mine-tab-in-inbox";
process.env.PAPERCLIP_CONFIG = configPath;
maybePersistWorktreeRuntimePorts({
serverPort: 3103,
databasePort: 54335,
});
const writtenConfig = JSON.parse(await fs.readFile(configPath, "utf8"));
expect(writtenConfig.server.port).toBe(3103);
expect(writtenConfig.database.embeddedPostgresPort).toBe(54335);
expect(writtenConfig.auth.publicBaseUrl).toBe("http://127.0.0.1:3103/");
});
it("can update the in-memory config without rewriting env-driven ports", () => {
const { config, changed } = applyRuntimePortSelectionToConfig(buildLegacyConfig("/tmp/shared"), {
serverPort: 3104,
databasePort: 54340,
allowServerPortWrite: false,
allowDatabasePortWrite: true,
});
expect(changed).toBe(true);
expect(config.server.port).toBe(3100);
expect(config.database.embeddedPostgresPort).toBe(54340);
expect(config.auth.publicBaseUrl).toBe("http://127.0.0.1:3104/");
});
});

View file

@ -3,6 +3,7 @@ import { existsSync, realpathSync } from "node:fs";
import { resolve } from "node:path";
import { config as loadDotenv } from "dotenv";
import { resolvePaperclipEnvPath } from "./paths.js";
import { maybeRepairLegacyWorktreeConfigAndEnvFiles } from "./worktree-config.js";
import {
AUTH_BASE_URL_MODES,
DEPLOYMENT_EXPOSURES,
@ -36,6 +37,8 @@ if (!isSameFile && existsSync(CWD_ENV_PATH)) {
loadDotenv({ path: CWD_ENV_PATH, override: false, quiet: true });
}
maybeRepairLegacyWorktreeConfigAndEnvFiles();
type DatabaseMode = "embedded-postgres" | "postgres";
export interface Config {

View file

@ -0,0 +1,21 @@
import fs from "node:fs";
import path from "node:path";
function addIgnorePath(target: Set<string>, candidate: string): void {
target.add(candidate);
try {
target.add(fs.realpathSync(candidate));
} catch {
// Ignore paths that do not exist in the current checkout.
}
}
export function resolveServerDevWatchIgnorePaths(serverRoot: string): string[] {
const ignorePaths = new Set<string>();
for (const relativePath of ["../ui/node_modules", "../ui/.vite", "../ui/dist"]) {
addIgnorePath(ignorePaths, path.resolve(serverRoot, relativePath));
}
return [...ignorePaths];
}

View file

@ -30,6 +30,7 @@ import { heartbeatService, reconcilePersistedRuntimeServicesOnStartup, routineSe
import { createStorageServiceFromConfig } from "./storage/index.js";
import { printStartupBanner } from "./startup-banner.js";
import { getBoardClaimWarningUrl, initializeBoardClaimChallenge } from "./board-claim.js";
import { maybePersistWorktreeRuntimePorts } from "./worktree-config.js";
type BetterAuthSessionUser = {
id: string;
@ -69,7 +70,7 @@ export interface StartedServer {
}
export async function startServer(): Promise<StartedServer> {
const config = loadConfig();
let config = loadConfig();
if (process.env.PAPERCLIP_SECRETS_PROVIDER === undefined) {
process.env.PAPERCLIP_SECRETS_PROVIDER = config.secretsProvider;
}
@ -167,6 +168,18 @@ export async function startServer(): Promise<StartedServer> {
const normalized = host.trim().toLowerCase();
return normalized === "127.0.0.1" || normalized === "localhost" || normalized === "::1";
}
function rewriteLocalUrlPort(rawUrl: string | undefined, port: number): string | undefined {
if (!rawUrl) return undefined;
try {
const parsed = new URL(rawUrl);
if (!isLoopbackHost(parsed.hostname)) return rawUrl;
parsed.port = String(port);
return parsed.toString();
} catch {
return rawUrl;
}
}
const LOCAL_BOARD_USER_ID = "local-board";
const LOCAL_BOARD_USER_EMAIL = "local@paperclip.local";
@ -233,6 +246,7 @@ export async function startServer(): Promise<StartedServer> {
let embeddedPostgresStartedByThisProcess = false;
let migrationSummary: MigrationSummary = "skipped";
let activeDatabaseConnectionString: string;
let resolvedEmbeddedPostgresPort: number | null = null;
let startupDbInfo:
| { mode: "external-postgres"; connectionString: string }
| { mode: "embedded-postgres"; dataDir: string; port: number };
@ -395,6 +409,7 @@ export async function startServer(): Promise<StartedServer> {
db = createDb(embeddedConnectionString);
logger.info("Embedded PostgreSQL ready");
activeDatabaseConnectionString = embeddedConnectionString;
resolvedEmbeddedPostgresPort = port;
startupDbInfo = { mode: "embedded-postgres", dataDir, port };
}
@ -476,6 +491,19 @@ export async function startServer(): Promise<StartedServer> {
}
const listenPort = await detectPort(config.port);
if (listenPort !== config.port) {
config.port = listenPort;
}
if (resolvedEmbeddedPostgresPort !== null && resolvedEmbeddedPostgresPort !== config.embeddedPostgresPort) {
config.embeddedPostgresPort = resolvedEmbeddedPostgresPort;
}
if (config.authBaseUrlMode === "explicit" && config.authPublicBaseUrl) {
config.authPublicBaseUrl = rewriteLocalUrlPort(config.authPublicBaseUrl, listenPort);
}
maybePersistWorktreeRuntimePorts({
serverPort: listenPort,
databasePort: resolvedEmbeddedPostgresPort,
});
const uiMode = config.uiDevMiddleware ? "vite-dev" : config.serveUi ? "static" : "none";
const storageService = createStorageServiceFromConfig(config);
const app = await createApp(db as any, {

View file

@ -0,0 +1,467 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import type { PaperclipConfig } from "@paperclipai/shared";
import { resolvePaperclipConfigPath, resolvePaperclipEnvPath } from "./paths.js";
function nonEmpty(value: string | null | undefined): string | null {
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}
function expandHomePrefix(value: string): string {
if (value === "~") return os.homedir();
if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2));
return value;
}
function resolveHomeAwarePath(value: string): string {
return path.resolve(expandHomePrefix(value));
}
function sanitizeWorktreeInstanceId(rawValue: string): string {
const trimmed = rawValue.trim().toLowerCase();
const normalized = trimmed
.replace(/[^a-z0-9_-]+/g, "-")
.replace(/-+/g, "-")
.replace(/^[-_]+|[-_]+$/g, "");
return normalized || "worktree";
}
function isLoopbackHost(hostname: string): boolean {
const value = hostname.trim().toLowerCase();
return value === "127.0.0.1" || value === "localhost" || value === "::1";
}
function rewriteLocalUrlPort(rawUrl: string | undefined, port: number): string | undefined {
if (!rawUrl) return undefined;
try {
const parsed = new URL(rawUrl);
if (!isLoopbackHost(parsed.hostname)) return rawUrl;
parsed.port = String(port);
return parsed.toString();
} catch {
return rawUrl;
}
}
function parseEnvFile(contents: string): Record<string, string> {
const entries: Record<string, string> = {};
for (const rawLine of contents.split(/\r?\n/)) {
const line = rawLine.trim();
if (!line || line.startsWith("#")) continue;
const match = rawLine.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$/);
if (!match) continue;
const [, key, rawValue] = match;
const value = rawValue.trim();
if (!value) {
entries[key] = "";
continue;
}
if (
(value.startsWith("\"") && value.endsWith("\"")) ||
(value.startsWith("'") && value.endsWith("'"))
) {
entries[key] = value.slice(1, -1);
continue;
}
entries[key] = value.replace(/\s+#.*$/, "").trim();
}
return entries;
}
function readEnvEntries(envPath: string): Record<string, string> {
if (!fs.existsSync(envPath)) return {};
return parseEnvFile(fs.readFileSync(envPath, "utf8"));
}
function formatEnvEntries(entries: Record<string, string>): string {
return [
"# Paperclip environment variables",
"# Generated by Paperclip worktree repair",
...Object.entries(entries).map(([key, value]) => `${key}=${JSON.stringify(value)}`),
"",
].join("\n");
}
function isPathInside(candidatePath: string, rootPath: string): boolean {
const candidate = path.resolve(candidatePath);
const root = path.resolve(rootPath);
return candidate === root || candidate.startsWith(`${root}${path.sep}`);
}
type WorktreeRuntimeContext = {
configPath: string;
envPath: string;
worktreeName: string;
instanceId: string;
homeDir: string;
instanceRoot: string;
contextPath: string;
embeddedPostgresDataDir: string;
backupDir: string;
logDir: string;
storageDir: string;
secretsKeyFilePath: string;
};
function resolveWorktreeRuntimeContext(
env: NodeJS.ProcessEnv,
overrideConfigPath?: string,
): WorktreeRuntimeContext | null {
if (env.PAPERCLIP_IN_WORKTREE !== "true") return null;
const configPath = resolvePaperclipConfigPath(overrideConfigPath);
const envPath = resolvePaperclipEnvPath(configPath);
const worktreeRoot = path.resolve(path.dirname(configPath), "..");
const worktreeName = nonEmpty(env.PAPERCLIP_WORKTREE_NAME) ?? path.basename(worktreeRoot);
const instanceId = nonEmpty(env.PAPERCLIP_INSTANCE_ID) ?? sanitizeWorktreeInstanceId(worktreeName);
const homeDir = resolveHomeAwarePath(
nonEmpty(env.PAPERCLIP_HOME) ??
nonEmpty(env.PAPERCLIP_WORKTREES_DIR) ??
"~/.paperclip-worktrees",
);
const instanceRoot = path.resolve(homeDir, "instances", instanceId);
return {
configPath,
envPath,
worktreeName,
instanceId,
homeDir,
instanceRoot,
contextPath: path.resolve(homeDir, "context.json"),
embeddedPostgresDataDir: path.resolve(instanceRoot, "db"),
backupDir: path.resolve(instanceRoot, "data", "backups"),
logDir: path.resolve(instanceRoot, "logs"),
storageDir: path.resolve(instanceRoot, "data", "storage"),
secretsKeyFilePath: path.resolve(instanceRoot, "secrets", "master.key"),
};
}
function writeConfigFile(configPath: string, config: PaperclipConfig): void {
fs.mkdirSync(path.dirname(configPath), { recursive: true });
fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + "\n", { mode: 0o600 });
}
function resolveRepoManagedWorktreesRoot(worktreeRoot: string): string | null {
const normalized = path.resolve(worktreeRoot);
const marker = `${path.sep}.paperclip${path.sep}worktrees${path.sep}`;
const index = normalized.indexOf(marker);
if (index === -1) return null;
const repoRoot = normalized.slice(0, index);
return path.resolve(repoRoot, ".paperclip", "worktrees");
}
function collectSiblingWorktreePorts(context: WorktreeRuntimeContext): {
serverPorts: Set<number>;
databasePorts: Set<number>;
} {
const serverPorts = new Set<number>();
const databasePorts = new Set<number>();
const siblingConfigPaths = new Set<string>();
const instancesDir = path.resolve(context.homeDir, "instances");
if (fs.existsSync(instancesDir)) {
for (const entry of fs.readdirSync(instancesDir, { withFileTypes: true })) {
if (!entry.isDirectory() || entry.name === context.instanceId) continue;
const siblingConfigPath = path.resolve(instancesDir, entry.name, "config.json");
if (fs.existsSync(siblingConfigPath)) {
siblingConfigPaths.add(siblingConfigPath);
}
}
}
const repoManagedWorktreesRoot = resolveRepoManagedWorktreesRoot(path.dirname(context.configPath));
if (repoManagedWorktreesRoot && fs.existsSync(repoManagedWorktreesRoot)) {
for (const entry of fs.readdirSync(repoManagedWorktreesRoot, { withFileTypes: true })) {
if (!entry.isDirectory()) continue;
const siblingConfigPath = path.resolve(repoManagedWorktreesRoot, entry.name, ".paperclip", "config.json");
if (path.resolve(siblingConfigPath) === path.resolve(context.configPath)) continue;
if (fs.existsSync(siblingConfigPath)) {
siblingConfigPaths.add(siblingConfigPath);
}
}
}
for (const siblingConfigPath of siblingConfigPaths) {
try {
const siblingConfig = JSON.parse(fs.readFileSync(siblingConfigPath, "utf8")) as PaperclipConfig;
if (Number.isInteger(siblingConfig.server.port) && siblingConfig.server.port > 0) {
serverPorts.add(siblingConfig.server.port);
}
if (
siblingConfig.database.mode === "embedded-postgres" &&
Number.isInteger(siblingConfig.database.embeddedPostgresPort) &&
siblingConfig.database.embeddedPostgresPort > 0
) {
databasePorts.add(siblingConfig.database.embeddedPostgresPort);
}
} catch {
// Ignore sibling configs that are missing or malformed.
}
}
return { serverPorts, databasePorts };
}
function findNextUnclaimedPort(preferredPort: number, claimedPorts: Set<number>): number {
let port = Math.max(1, Math.trunc(preferredPort));
while (claimedPorts.has(port)) {
port += 1;
}
return port;
}
function buildIsolatedWorktreeConfig(
config: PaperclipConfig,
context: WorktreeRuntimeContext,
portOverrides?: {
serverPort?: number;
databasePort?: number;
},
): PaperclipConfig {
const serverPort = portOverrides?.serverPort ?? config.server.port;
const databasePort =
config.database.mode === "embedded-postgres"
? portOverrides?.databasePort ?? config.database.embeddedPostgresPort
: undefined;
const nextConfig: PaperclipConfig = {
...config,
database: {
...config.database,
...(config.database.mode === "embedded-postgres"
? {
embeddedPostgresDataDir: context.embeddedPostgresDataDir,
embeddedPostgresPort: databasePort ?? config.database.embeddedPostgresPort,
backup: {
...config.database.backup,
dir: context.backupDir,
},
}
: {}),
},
server: {
...config.server,
port: serverPort,
},
logging: {
...config.logging,
logDir: context.logDir,
},
storage: {
...config.storage,
localDisk: {
...config.storage.localDisk,
baseDir: context.storageDir,
},
},
secrets: {
...config.secrets,
localEncrypted: {
...config.secrets.localEncrypted,
keyFilePath: context.secretsKeyFilePath,
},
},
};
if (config.auth.baseUrlMode === "explicit" && config.auth.publicBaseUrl) {
nextConfig.auth = {
...config.auth,
publicBaseUrl: rewriteLocalUrlPort(config.auth.publicBaseUrl, serverPort),
};
}
return nextConfig;
}
function needsWorktreeConfigRepair(
config: PaperclipConfig,
context: WorktreeRuntimeContext,
): boolean {
if (config.database.mode === "embedded-postgres") {
if (!isPathInside(config.database.embeddedPostgresDataDir, context.instanceRoot)) {
return true;
}
if (!isPathInside(config.database.backup.dir, context.instanceRoot)) {
return true;
}
}
if (!isPathInside(config.logging.logDir, context.instanceRoot)) {
return true;
}
if (!isPathInside(config.storage.localDisk.baseDir, context.instanceRoot)) {
return true;
}
if (!isPathInside(config.secrets.localEncrypted.keyFilePath, context.instanceRoot)) {
return true;
}
return false;
}
export function applyRuntimePortSelectionToConfig(
config: PaperclipConfig,
input: {
serverPort: number;
databasePort?: number | null;
allowServerPortWrite?: boolean;
allowDatabasePortWrite?: boolean;
},
): { config: PaperclipConfig; changed: boolean } {
let changed = false;
let nextConfig = config;
if (input.allowServerPortWrite !== false && config.server.port !== input.serverPort) {
nextConfig = {
...nextConfig,
server: {
...nextConfig.server,
port: input.serverPort,
},
};
changed = true;
}
if (
input.allowDatabasePortWrite !== false &&
nextConfig.database.mode === "embedded-postgres" &&
typeof input.databasePort === "number" &&
nextConfig.database.embeddedPostgresPort !== input.databasePort
) {
nextConfig = {
...nextConfig,
database: {
...nextConfig.database,
embeddedPostgresPort: input.databasePort,
},
};
changed = true;
}
if (nextConfig.auth.baseUrlMode === "explicit" && nextConfig.auth.publicBaseUrl) {
const rewritten = rewriteLocalUrlPort(nextConfig.auth.publicBaseUrl, input.serverPort);
if (rewritten && rewritten !== nextConfig.auth.publicBaseUrl) {
nextConfig = {
...nextConfig,
auth: {
...nextConfig.auth,
publicBaseUrl: rewritten,
},
};
changed = true;
}
}
return { config: nextConfig, changed };
}
export function maybeRepairLegacyWorktreeConfigAndEnvFiles(): {
repairedConfig: boolean;
repairedEnv: boolean;
} {
const context = resolveWorktreeRuntimeContext(process.env);
if (!context) {
return { repairedConfig: false, repairedEnv: false };
}
process.env.PAPERCLIP_HOME = context.homeDir;
process.env.PAPERCLIP_INSTANCE_ID = context.instanceId;
process.env.PAPERCLIP_CONFIG = context.configPath;
process.env.PAPERCLIP_CONTEXT = context.contextPath;
process.env.PAPERCLIP_WORKTREE_NAME = context.worktreeName;
let repairedConfig = false;
if (fs.existsSync(context.configPath)) {
try {
const parsed = JSON.parse(fs.readFileSync(context.configPath, "utf8")) as PaperclipConfig;
const siblingPorts = collectSiblingWorktreePorts(context);
const hasSiblingPortCollision =
siblingPorts.serverPorts.has(parsed.server.port) ||
(parsed.database.mode === "embedded-postgres" &&
siblingPorts.databasePorts.has(parsed.database.embeddedPostgresPort));
if (needsWorktreeConfigRepair(parsed, context) || hasSiblingPortCollision) {
const selectedServerPort = findNextUnclaimedPort(
parsed.server.port === 3100 ? 3101 : parsed.server.port,
siblingPorts.serverPorts,
);
const selectedDatabasePort =
parsed.database.mode === "embedded-postgres"
? findNextUnclaimedPort(
parsed.database.embeddedPostgresPort === 54329
? 54330
: parsed.database.embeddedPostgresPort,
new Set([...siblingPorts.databasePorts, selectedServerPort]),
)
: undefined;
writeConfigFile(
context.configPath,
buildIsolatedWorktreeConfig(parsed, context, {
serverPort: selectedServerPort,
databasePort: selectedDatabasePort,
}),
);
repairedConfig = true;
}
} catch {
// Leave invalid configs to the normal startup validation path.
}
}
const existingEnvEntries = readEnvEntries(context.envPath);
const desiredEnvEntries: Record<string, string> = {
...existingEnvEntries,
PAPERCLIP_HOME: context.homeDir,
PAPERCLIP_INSTANCE_ID: context.instanceId,
PAPERCLIP_CONFIG: context.configPath,
PAPERCLIP_CONTEXT: context.contextPath,
PAPERCLIP_IN_WORKTREE: "true",
PAPERCLIP_WORKTREE_NAME: context.worktreeName,
};
const repairedEnv = Object.entries(desiredEnvEntries).some(
([key, value]) => existingEnvEntries[key] !== value,
);
if (repairedEnv) {
fs.mkdirSync(path.dirname(context.envPath), { recursive: true });
fs.writeFileSync(context.envPath, formatEnvEntries(desiredEnvEntries), { mode: 0o600 });
}
return { repairedConfig, repairedEnv };
}
export function maybePersistWorktreeRuntimePorts(input: {
serverPort: number;
databasePort?: number | null;
}): void {
const context = resolveWorktreeRuntimeContext(process.env);
if (!context || !fs.existsSync(context.configPath)) return;
let fileConfig: PaperclipConfig;
try {
fileConfig = JSON.parse(fs.readFileSync(context.configPath, "utf8")) as PaperclipConfig;
} catch {
return;
}
const { config, changed } = applyRuntimePortSelectionToConfig(fileConfig, {
serverPort: input.serverPort,
databasePort: input.databasePort,
allowServerPortWrite: !nonEmpty(process.env.PORT),
allowDatabasePortWrite: !nonEmpty(process.env.DATABASE_URL),
});
if (changed) {
writeConfigFile(context.configPath, config);
}
}