Merge pull request #2657 from paperclipai/fix/inbox-last-activity-ordering
Add versioned telemetry events
This commit is contained in:
commit
f8452a4520
16 changed files with 613 additions and 0 deletions
|
|
@ -58,6 +58,7 @@ export class TelemetryClient {
|
||||||
app,
|
app,
|
||||||
schemaVersion,
|
schemaVersion,
|
||||||
installId: state.installId,
|
installId: state.installId,
|
||||||
|
version: this.version,
|
||||||
events,
|
events,
|
||||||
}),
|
}),
|
||||||
signal: controller.signal,
|
signal: controller.signal,
|
||||||
|
|
|
||||||
|
|
@ -23,6 +23,48 @@ export function trackCompanyImported(
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function trackProjectCreated(client: TelemetryClient): void {
|
||||||
|
client.track("project.created");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function trackRoutineCreated(client: TelemetryClient): void {
|
||||||
|
client.track("routine.created");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function trackRoutineRun(
|
||||||
|
client: TelemetryClient,
|
||||||
|
dims: { source: string; status: string },
|
||||||
|
): void {
|
||||||
|
client.track("routine.run", {
|
||||||
|
source: dims.source,
|
||||||
|
status: dims.status,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function trackGoalCreated(
|
||||||
|
client: TelemetryClient,
|
||||||
|
dims?: { goalLevel?: string | null },
|
||||||
|
): void {
|
||||||
|
client.track("goal.created", dims?.goalLevel ? { goal_level: dims.goalLevel } : undefined);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function trackAgentCreated(
|
||||||
|
client: TelemetryClient,
|
||||||
|
dims: { agentRole: string },
|
||||||
|
): void {
|
||||||
|
client.track("agent.created", { agent_role: dims.agentRole });
|
||||||
|
}
|
||||||
|
|
||||||
|
export function trackSkillImported(
|
||||||
|
client: TelemetryClient,
|
||||||
|
dims: { sourceType: string; skillRef?: string | null },
|
||||||
|
): void {
|
||||||
|
client.track("skill.imported", {
|
||||||
|
source_type: dims.sourceType,
|
||||||
|
...(dims.skillRef ? { skill_ref: dims.skillRef } : {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export function trackAgentFirstHeartbeat(
|
export function trackAgentFirstHeartbeat(
|
||||||
client: TelemetryClient,
|
client: TelemetryClient,
|
||||||
dims: { agentRole: string },
|
dims: { agentRole: string },
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,12 @@ export {
|
||||||
trackInstallStarted,
|
trackInstallStarted,
|
||||||
trackInstallCompleted,
|
trackInstallCompleted,
|
||||||
trackCompanyImported,
|
trackCompanyImported,
|
||||||
|
trackProjectCreated,
|
||||||
|
trackRoutineCreated,
|
||||||
|
trackRoutineRun,
|
||||||
|
trackGoalCreated,
|
||||||
|
trackAgentCreated,
|
||||||
|
trackSkillImported,
|
||||||
trackAgentFirstHeartbeat,
|
trackAgentFirstHeartbeat,
|
||||||
trackAgentTaskCompleted,
|
trackAgentTaskCompleted,
|
||||||
trackErrorHandlerCrash,
|
trackErrorHandlerCrash,
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,7 @@ export interface TelemetryEventEnvelope {
|
||||||
app: string;
|
app: string;
|
||||||
schemaVersion: string;
|
schemaVersion: string;
|
||||||
installId: string;
|
installId: string;
|
||||||
|
version: string;
|
||||||
events: TelemetryEvent[];
|
events: TelemetryEvent[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -31,6 +32,12 @@ export type TelemetryEventName =
|
||||||
| "install.started"
|
| "install.started"
|
||||||
| "install.completed"
|
| "install.completed"
|
||||||
| "company.imported"
|
| "company.imported"
|
||||||
|
| "project.created"
|
||||||
|
| "routine.created"
|
||||||
|
| "routine.run"
|
||||||
|
| "goal.created"
|
||||||
|
| "agent.created"
|
||||||
|
| "skill.imported"
|
||||||
| "agent.first_heartbeat"
|
| "agent.first_heartbeat"
|
||||||
| "agent.task_completed"
|
| "agent.task_completed"
|
||||||
| "error.handler_crash"
|
| "error.handler_crash"
|
||||||
|
|
|
||||||
|
|
@ -51,12 +51,22 @@ const mockSecretService = vi.hoisted(() => ({
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const mockLogActivity = vi.hoisted(() => vi.fn());
|
const mockLogActivity = vi.hoisted(() => vi.fn());
|
||||||
|
const mockTrackAgentCreated = vi.hoisted(() => vi.fn());
|
||||||
|
const mockGetTelemetryClient = vi.hoisted(() => vi.fn());
|
||||||
|
|
||||||
const mockAdapter = vi.hoisted(() => ({
|
const mockAdapter = vi.hoisted(() => ({
|
||||||
listSkills: vi.fn(),
|
listSkills: vi.fn(),
|
||||||
syncSkills: vi.fn(),
|
syncSkills: vi.fn(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
vi.mock("@paperclipai/shared/telemetry", () => ({
|
||||||
|
trackAgentCreated: mockTrackAgentCreated,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("../telemetry.js", () => ({
|
||||||
|
getTelemetryClient: mockGetTelemetryClient,
|
||||||
|
}));
|
||||||
|
|
||||||
vi.mock("../services/index.js", () => ({
|
vi.mock("../services/index.js", () => ({
|
||||||
agentService: () => mockAgentService,
|
agentService: () => mockAgentService,
|
||||||
agentInstructionsService: () => mockAgentInstructionsService,
|
agentInstructionsService: () => mockAgentInstructionsService,
|
||||||
|
|
@ -132,6 +142,7 @@ function makeAgent(adapterType: string) {
|
||||||
describe("agent skill routes", () => {
|
describe("agent skill routes", () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.resetAllMocks();
|
vi.resetAllMocks();
|
||||||
|
mockGetTelemetryClient.mockReturnValue({ track: vi.fn() });
|
||||||
mockAgentService.resolveByReference.mockResolvedValue({
|
mockAgentService.resolveByReference.mockResolvedValue({
|
||||||
ambiguous: false,
|
ambiguous: false,
|
||||||
agent: makeAgent("claude_local"),
|
agent: makeAgent("claude_local"),
|
||||||
|
|
@ -330,6 +341,9 @@ describe("agent skill routes", () => {
|
||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
expect(mockTrackAgentCreated).toHaveBeenCalledWith(expect.anything(), {
|
||||||
|
agentRole: "engineer",
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("materializes a managed AGENTS.md for directly created local agents", async () => {
|
it("materializes a managed AGENTS.md for directly created local agents", async () => {
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,22 @@ const mockCompanySkillService = vi.hoisted(() => ({
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const mockLogActivity = vi.hoisted(() => vi.fn());
|
const mockLogActivity = vi.hoisted(() => vi.fn());
|
||||||
|
const mockTrackSkillImported = vi.hoisted(() => vi.fn());
|
||||||
|
const mockGetTelemetryClient = vi.hoisted(() => vi.fn());
|
||||||
|
|
||||||
|
vi.mock("@paperclipai/shared/telemetry", async () => {
|
||||||
|
const actual = await vi.importActual<typeof import("@paperclipai/shared/telemetry")>(
|
||||||
|
"@paperclipai/shared/telemetry",
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
...actual,
|
||||||
|
trackSkillImported: mockTrackSkillImported,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.mock("../telemetry.js", () => ({
|
||||||
|
getTelemetryClient: mockGetTelemetryClient,
|
||||||
|
}));
|
||||||
|
|
||||||
vi.mock("../services/index.js", () => ({
|
vi.mock("../services/index.js", () => ({
|
||||||
accessService: () => mockAccessService,
|
accessService: () => mockAccessService,
|
||||||
|
|
@ -41,6 +57,7 @@ function createApp(actor: Record<string, unknown>) {
|
||||||
describe("company skill mutation permissions", () => {
|
describe("company skill mutation permissions", () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.clearAllMocks();
|
vi.clearAllMocks();
|
||||||
|
mockGetTelemetryClient.mockReturnValue({ track: vi.fn() });
|
||||||
mockCompanySkillService.importFromSource.mockResolvedValue({
|
mockCompanySkillService.importFromSource.mockResolvedValue({
|
||||||
imported: [],
|
imported: [],
|
||||||
warnings: [],
|
warnings: [],
|
||||||
|
|
@ -68,6 +85,140 @@ describe("company skill mutation permissions", () => {
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("tracks public GitHub skill imports with an explicit skill reference", async () => {
|
||||||
|
mockCompanySkillService.importFromSource.mockResolvedValue({
|
||||||
|
imported: [
|
||||||
|
{
|
||||||
|
id: "skill-1",
|
||||||
|
companyId: "company-1",
|
||||||
|
key: "vercel-labs/agent-browser/find-skills",
|
||||||
|
slug: "find-skills",
|
||||||
|
name: "Find Skills",
|
||||||
|
description: null,
|
||||||
|
markdown: "# Find Skills",
|
||||||
|
sourceType: "github",
|
||||||
|
sourceLocator: "https://github.com/vercel-labs/agent-browser",
|
||||||
|
sourceRef: null,
|
||||||
|
trustLevel: "markdown_only",
|
||||||
|
compatibility: "compatible",
|
||||||
|
fileInventory: [],
|
||||||
|
metadata: {
|
||||||
|
hostname: "github.com",
|
||||||
|
owner: "vercel-labs",
|
||||||
|
repo: "agent-browser",
|
||||||
|
},
|
||||||
|
createdAt: new Date(),
|
||||||
|
updatedAt: new Date(),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
warnings: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await request(createApp({
|
||||||
|
type: "board",
|
||||||
|
userId: "local-board",
|
||||||
|
companyIds: ["company-1"],
|
||||||
|
source: "local_implicit",
|
||||||
|
isInstanceAdmin: false,
|
||||||
|
}))
|
||||||
|
.post("/api/companies/company-1/skills/import")
|
||||||
|
.send({ source: "https://github.com/vercel-labs/agent-browser" });
|
||||||
|
|
||||||
|
expect(res.status, JSON.stringify(res.body)).toBe(201);
|
||||||
|
expect(mockTrackSkillImported).toHaveBeenCalledWith(expect.anything(), {
|
||||||
|
sourceType: "github",
|
||||||
|
skillRef: "vercel-labs/agent-browser/find-skills",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not expose a skill reference for non-public skill imports", async () => {
|
||||||
|
mockCompanySkillService.importFromSource.mockResolvedValue({
|
||||||
|
imported: [
|
||||||
|
{
|
||||||
|
id: "skill-1",
|
||||||
|
companyId: "company-1",
|
||||||
|
key: "private-skill",
|
||||||
|
slug: "private-skill",
|
||||||
|
name: "Private Skill",
|
||||||
|
description: null,
|
||||||
|
markdown: "# Private Skill",
|
||||||
|
sourceType: "github",
|
||||||
|
sourceLocator: "https://ghe.example.com/acme/private-skill",
|
||||||
|
sourceRef: null,
|
||||||
|
trustLevel: "markdown_only",
|
||||||
|
compatibility: "compatible",
|
||||||
|
fileInventory: [],
|
||||||
|
metadata: {
|
||||||
|
hostname: "ghe.example.com",
|
||||||
|
owner: "acme",
|
||||||
|
repo: "private-skill",
|
||||||
|
},
|
||||||
|
createdAt: new Date(),
|
||||||
|
updatedAt: new Date(),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
warnings: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await request(createApp({
|
||||||
|
type: "board",
|
||||||
|
userId: "local-board",
|
||||||
|
companyIds: ["company-1"],
|
||||||
|
source: "local_implicit",
|
||||||
|
isInstanceAdmin: false,
|
||||||
|
}))
|
||||||
|
.post("/api/companies/company-1/skills/import")
|
||||||
|
.send({ source: "https://ghe.example.com/acme/private-skill" });
|
||||||
|
|
||||||
|
expect(res.status, JSON.stringify(res.body)).toBe(201);
|
||||||
|
expect(mockTrackSkillImported).toHaveBeenCalledWith(expect.anything(), {
|
||||||
|
sourceType: "github",
|
||||||
|
skillRef: null,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not expose a skill reference when GitHub metadata is missing", async () => {
|
||||||
|
mockCompanySkillService.importFromSource.mockResolvedValue({
|
||||||
|
imported: [
|
||||||
|
{
|
||||||
|
id: "skill-1",
|
||||||
|
companyId: "company-1",
|
||||||
|
key: "unknown/private-skill",
|
||||||
|
slug: "private-skill",
|
||||||
|
name: "Private Skill",
|
||||||
|
description: null,
|
||||||
|
markdown: "# Private Skill",
|
||||||
|
sourceType: "github",
|
||||||
|
sourceLocator: "https://github.com/acme/private-skill",
|
||||||
|
sourceRef: null,
|
||||||
|
trustLevel: "markdown_only",
|
||||||
|
compatibility: "compatible",
|
||||||
|
fileInventory: [],
|
||||||
|
metadata: null,
|
||||||
|
createdAt: new Date(),
|
||||||
|
updatedAt: new Date(),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
warnings: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await request(createApp({
|
||||||
|
type: "board",
|
||||||
|
userId: "local-board",
|
||||||
|
companyIds: ["company-1"],
|
||||||
|
source: "local_implicit",
|
||||||
|
isInstanceAdmin: false,
|
||||||
|
}))
|
||||||
|
.post("/api/companies/company-1/skills/import")
|
||||||
|
.send({ source: "https://github.com/acme/private-skill" });
|
||||||
|
|
||||||
|
expect(res.status, JSON.stringify(res.body)).toBe(201);
|
||||||
|
expect(mockTrackSkillImported).toHaveBeenCalledWith(expect.anything(), {
|
||||||
|
sourceType: "github",
|
||||||
|
skillRef: null,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("blocks same-company agents without management permission from mutating company skills", async () => {
|
it("blocks same-company agents without management permission from mutating company skills", async () => {
|
||||||
mockAgentService.getById.mockResolvedValue({
|
mockAgentService.getById.mockResolvedValue({
|
||||||
id: "agent-1",
|
id: "agent-1",
|
||||||
|
|
|
||||||
115
server/src/__tests__/project-goal-telemetry-routes.test.ts
Normal file
115
server/src/__tests__/project-goal-telemetry-routes.test.ts
Normal file
|
|
@ -0,0 +1,115 @@
|
||||||
|
import express from "express";
|
||||||
|
import request from "supertest";
|
||||||
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import { projectRoutes } from "../routes/projects.js";
|
||||||
|
import { goalRoutes } from "../routes/goals.js";
|
||||||
|
import { errorHandler } from "../middleware/index.js";
|
||||||
|
|
||||||
|
const mockProjectService = vi.hoisted(() => ({
|
||||||
|
list: vi.fn(),
|
||||||
|
getById: vi.fn(),
|
||||||
|
create: vi.fn(),
|
||||||
|
createWorkspace: vi.fn(),
|
||||||
|
resolveByReference: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
const mockGoalService = vi.hoisted(() => ({
|
||||||
|
list: vi.fn(),
|
||||||
|
getById: vi.fn(),
|
||||||
|
create: vi.fn(),
|
||||||
|
update: vi.fn(),
|
||||||
|
remove: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
const mockWorkspaceOperationService = vi.hoisted(() => ({}));
|
||||||
|
const mockLogActivity = vi.hoisted(() => vi.fn());
|
||||||
|
const mockTrackProjectCreated = vi.hoisted(() => vi.fn());
|
||||||
|
const mockTrackGoalCreated = vi.hoisted(() => vi.fn());
|
||||||
|
const mockGetTelemetryClient = vi.hoisted(() => vi.fn());
|
||||||
|
|
||||||
|
vi.mock("@paperclipai/shared/telemetry", async () => {
|
||||||
|
const actual = await vi.importActual<typeof import("@paperclipai/shared/telemetry")>(
|
||||||
|
"@paperclipai/shared/telemetry",
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
...actual,
|
||||||
|
trackProjectCreated: mockTrackProjectCreated,
|
||||||
|
trackGoalCreated: mockTrackGoalCreated,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.mock("../telemetry.js", () => ({
|
||||||
|
getTelemetryClient: mockGetTelemetryClient,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("../services/index.js", () => ({
|
||||||
|
goalService: () => mockGoalService,
|
||||||
|
logActivity: mockLogActivity,
|
||||||
|
projectService: () => mockProjectService,
|
||||||
|
workspaceOperationService: () => mockWorkspaceOperationService,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("../services/workspace-runtime.js", () => ({
|
||||||
|
startRuntimeServicesForWorkspaceControl: vi.fn(),
|
||||||
|
stopRuntimeServicesForProjectWorkspace: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
function createApp(route: ReturnType<typeof projectRoutes> | ReturnType<typeof goalRoutes>) {
|
||||||
|
const app = express();
|
||||||
|
app.use(express.json());
|
||||||
|
app.use((req, _res, next) => {
|
||||||
|
(req as any).actor = {
|
||||||
|
type: "board",
|
||||||
|
userId: "board-user",
|
||||||
|
companyIds: ["company-1"],
|
||||||
|
source: "local_implicit",
|
||||||
|
isInstanceAdmin: false,
|
||||||
|
};
|
||||||
|
next();
|
||||||
|
});
|
||||||
|
app.use("/api", route);
|
||||||
|
app.use(errorHandler);
|
||||||
|
return app;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("project and goal telemetry routes", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
mockGetTelemetryClient.mockReturnValue({ track: vi.fn() });
|
||||||
|
mockProjectService.resolveByReference.mockResolvedValue({ ambiguous: false, project: null });
|
||||||
|
mockProjectService.create.mockResolvedValue({
|
||||||
|
id: "project-1",
|
||||||
|
companyId: "company-1",
|
||||||
|
name: "Telemetry project",
|
||||||
|
description: null,
|
||||||
|
status: "backlog",
|
||||||
|
});
|
||||||
|
mockGoalService.create.mockResolvedValue({
|
||||||
|
id: "goal-1",
|
||||||
|
companyId: "company-1",
|
||||||
|
title: "Telemetry goal",
|
||||||
|
description: null,
|
||||||
|
level: "team",
|
||||||
|
status: "planned",
|
||||||
|
});
|
||||||
|
mockLogActivity.mockResolvedValue(undefined);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("emits telemetry when a project is created", async () => {
|
||||||
|
const res = await request(createApp(projectRoutes({} as any)))
|
||||||
|
.post("/api/companies/company-1/projects")
|
||||||
|
.send({ name: "Telemetry project" });
|
||||||
|
|
||||||
|
expect(res.status, JSON.stringify(res.body)).toBe(201);
|
||||||
|
expect(mockTrackProjectCreated).toHaveBeenCalledWith(expect.anything());
|
||||||
|
});
|
||||||
|
|
||||||
|
it("emits telemetry when a goal is created", async () => {
|
||||||
|
const res = await request(createApp(goalRoutes({} as any)))
|
||||||
|
.post("/api/companies/company-1/goals")
|
||||||
|
.send({ title: "Telemetry goal", level: "team" });
|
||||||
|
|
||||||
|
expect(res.status, JSON.stringify(res.body)).toBe(201);
|
||||||
|
expect(mockTrackGoalCreated).toHaveBeenCalledWith(expect.anything(), { goalLevel: "team" });
|
||||||
|
});
|
||||||
|
});
|
||||||
163
server/src/__tests__/routine-run-telemetry.test.ts
Normal file
163
server/src/__tests__/routine-run-telemetry.test.ts
Normal file
|
|
@ -0,0 +1,163 @@
|
||||||
|
import { randomUUID } from "node:crypto";
|
||||||
|
import { eq } from "drizzle-orm";
|
||||||
|
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||||
|
import {
|
||||||
|
agents,
|
||||||
|
companies,
|
||||||
|
createDb,
|
||||||
|
executionWorkspaces,
|
||||||
|
heartbeatRuns,
|
||||||
|
issues,
|
||||||
|
projectWorkspaces,
|
||||||
|
projects,
|
||||||
|
routineRuns,
|
||||||
|
routines,
|
||||||
|
routineTriggers,
|
||||||
|
} from "@paperclipai/db";
|
||||||
|
import {
|
||||||
|
getEmbeddedPostgresTestSupport,
|
||||||
|
startEmbeddedPostgresTestDatabase,
|
||||||
|
} from "./helpers/embedded-postgres.js";
|
||||||
|
|
||||||
|
const mockTelemetryClient = vi.hoisted(() => ({ track: vi.fn() }));
|
||||||
|
const mockTrackRoutineRun = vi.hoisted(() => vi.fn());
|
||||||
|
|
||||||
|
vi.mock("../telemetry.js", () => ({
|
||||||
|
getTelemetryClient: () => mockTelemetryClient,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("@paperclipai/shared/telemetry", async () => {
|
||||||
|
const actual = await vi.importActual<typeof import("@paperclipai/shared/telemetry")>(
|
||||||
|
"@paperclipai/shared/telemetry",
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
...actual,
|
||||||
|
trackRoutineRun: mockTrackRoutineRun,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
import { routineService } from "../services/routines.ts";
|
||||||
|
|
||||||
|
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||||
|
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||||
|
|
||||||
|
describeEmbeddedPostgres("routine run telemetry", () => {
|
||||||
|
let db!: ReturnType<typeof createDb>;
|
||||||
|
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-routine-telemetry-");
|
||||||
|
db = createDb(tempDb.connectionString);
|
||||||
|
}, 20_000);
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
await db.delete(routineRuns);
|
||||||
|
await db.delete(routineTriggers);
|
||||||
|
await db.delete(routines);
|
||||||
|
await db.delete(heartbeatRuns);
|
||||||
|
await db.delete(issues);
|
||||||
|
await db.delete(executionWorkspaces);
|
||||||
|
await db.delete(projectWorkspaces);
|
||||||
|
await db.delete(projects);
|
||||||
|
await db.delete(agents);
|
||||||
|
await db.delete(companies);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await tempDb?.cleanup();
|
||||||
|
});
|
||||||
|
|
||||||
|
async function seedFixture() {
|
||||||
|
const companyId = randomUUID();
|
||||||
|
const agentId = randomUUID();
|
||||||
|
const projectId = randomUUID();
|
||||||
|
|
||||||
|
await db.insert(companies).values({
|
||||||
|
id: companyId,
|
||||||
|
name: "Paperclip",
|
||||||
|
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
|
||||||
|
requireBoardApprovalForNewAgents: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
await db.insert(agents).values({
|
||||||
|
id: agentId,
|
||||||
|
companyId,
|
||||||
|
name: "CodexCoder",
|
||||||
|
role: "engineer",
|
||||||
|
status: "active",
|
||||||
|
adapterType: "codex_local",
|
||||||
|
adapterConfig: {},
|
||||||
|
runtimeConfig: {},
|
||||||
|
permissions: {},
|
||||||
|
});
|
||||||
|
|
||||||
|
await db.insert(projects).values({
|
||||||
|
id: projectId,
|
||||||
|
companyId,
|
||||||
|
name: "Routines",
|
||||||
|
status: "in_progress",
|
||||||
|
});
|
||||||
|
|
||||||
|
const svc = routineService(db, {
|
||||||
|
heartbeat: {
|
||||||
|
wakeup: async (wakeupAgentId, wakeupOpts) => {
|
||||||
|
const issueId =
|
||||||
|
(typeof wakeupOpts.payload?.issueId === "string" && wakeupOpts.payload.issueId)
|
||||||
|
|| (typeof wakeupOpts.contextSnapshot?.issueId === "string" && wakeupOpts.contextSnapshot.issueId)
|
||||||
|
|| null;
|
||||||
|
if (!issueId) return null;
|
||||||
|
const queuedRunId = randomUUID();
|
||||||
|
await db.insert(heartbeatRuns).values({
|
||||||
|
id: queuedRunId,
|
||||||
|
companyId,
|
||||||
|
agentId: wakeupAgentId,
|
||||||
|
invocationSource: wakeupOpts.source ?? "assignment",
|
||||||
|
triggerDetail: wakeupOpts.triggerDetail ?? null,
|
||||||
|
status: "queued",
|
||||||
|
contextSnapshot: { ...(wakeupOpts.contextSnapshot ?? {}), issueId },
|
||||||
|
});
|
||||||
|
await db
|
||||||
|
.update(issues)
|
||||||
|
.set({
|
||||||
|
executionRunId: queuedRunId,
|
||||||
|
executionLockedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(eq(issues.id, issueId));
|
||||||
|
return { id: queuedRunId };
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const routine = await svc.create(
|
||||||
|
companyId,
|
||||||
|
{
|
||||||
|
projectId,
|
||||||
|
goalId: null,
|
||||||
|
parentIssueId: null,
|
||||||
|
title: "Run telemetry test",
|
||||||
|
description: "Routine body",
|
||||||
|
assigneeAgentId: agentId,
|
||||||
|
priority: "medium",
|
||||||
|
status: "active",
|
||||||
|
concurrencyPolicy: "coalesce_if_active",
|
||||||
|
catchUpPolicy: "skip_missed",
|
||||||
|
},
|
||||||
|
{},
|
||||||
|
);
|
||||||
|
|
||||||
|
return { routine, svc };
|
||||||
|
}
|
||||||
|
|
||||||
|
it("emits telemetry for routine runs from the service layer", async () => {
|
||||||
|
const { routine, svc } = await seedFixture();
|
||||||
|
|
||||||
|
const run = await svc.runRoutine(routine.id, { source: "manual" });
|
||||||
|
|
||||||
|
expect(run.status).toBe("issue_created");
|
||||||
|
expect(mockTrackRoutineRun).toHaveBeenCalledWith(mockTelemetryClient, {
|
||||||
|
source: "manual",
|
||||||
|
status: "issue_created",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -82,6 +82,22 @@ const mockAccessService = vi.hoisted(() => ({
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const mockLogActivity = vi.hoisted(() => vi.fn());
|
const mockLogActivity = vi.hoisted(() => vi.fn());
|
||||||
|
const mockTrackRoutineCreated = vi.hoisted(() => vi.fn());
|
||||||
|
const mockGetTelemetryClient = vi.hoisted(() => vi.fn());
|
||||||
|
|
||||||
|
vi.mock("@paperclipai/shared/telemetry", async () => {
|
||||||
|
const actual = await vi.importActual<typeof import("@paperclipai/shared/telemetry")>(
|
||||||
|
"@paperclipai/shared/telemetry",
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
...actual,
|
||||||
|
trackRoutineCreated: mockTrackRoutineCreated,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.mock("../telemetry.js", () => ({
|
||||||
|
getTelemetryClient: mockGetTelemetryClient,
|
||||||
|
}));
|
||||||
|
|
||||||
vi.mock("../services/index.js", () => ({
|
vi.mock("../services/index.js", () => ({
|
||||||
accessService: () => mockAccessService,
|
accessService: () => mockAccessService,
|
||||||
|
|
@ -104,6 +120,7 @@ function createApp(actor: Record<string, unknown>) {
|
||||||
describe("routine routes", () => {
|
describe("routine routes", () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.clearAllMocks();
|
vi.clearAllMocks();
|
||||||
|
mockGetTelemetryClient.mockReturnValue({ track: vi.fn() });
|
||||||
mockRoutineService.create.mockResolvedValue(routine);
|
mockRoutineService.create.mockResolvedValue(routine);
|
||||||
mockRoutineService.get.mockResolvedValue(routine);
|
mockRoutineService.get.mockResolvedValue(routine);
|
||||||
mockRoutineService.getTrigger.mockResolvedValue(trigger);
|
mockRoutineService.getTrigger.mockResolvedValue(trigger);
|
||||||
|
|
@ -267,5 +284,6 @@ describe("routine routes", () => {
|
||||||
agentId: null,
|
agentId: null,
|
||||||
userId: "board-user",
|
userId: "board-user",
|
||||||
});
|
});
|
||||||
|
expect(mockTrackRoutineCreated).toHaveBeenCalledWith(expect.anything());
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -33,6 +33,25 @@ describe("TelemetryClient periodic flush", () => {
|
||||||
|
|
||||||
await vi.advanceTimersByTimeAsync(1000);
|
await vi.advanceTimersByTimeAsync(1000);
|
||||||
expect(fetch).toHaveBeenCalledTimes(1);
|
expect(fetch).toHaveBeenCalledTimes(1);
|
||||||
|
const lastCall = vi.mocked(fetch).mock.calls.at(-1);
|
||||||
|
expect(lastCall?.[0]).toBe("http://localhost:9999/ingest");
|
||||||
|
const requestInit = lastCall?.[1] as RequestInit | undefined;
|
||||||
|
expect(requestInit?.method).toBe("POST");
|
||||||
|
expect(requestInit?.headers).toEqual({ "Content-Type": "application/json" });
|
||||||
|
const body = JSON.parse(String(requestInit?.body ?? "{}"));
|
||||||
|
expect(body).toMatchObject({
|
||||||
|
app: "paperclip",
|
||||||
|
schemaVersion: "1",
|
||||||
|
installId: "test-install",
|
||||||
|
version: "0.0.0-test",
|
||||||
|
events: [
|
||||||
|
{
|
||||||
|
name: "install.started",
|
||||||
|
dimensions: {},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
expect(body.events[0]?.occurredAt).toEqual(expect.any(String));
|
||||||
|
|
||||||
// Second tick with no new events — no additional call
|
// Second tick with no new events — no additional call
|
||||||
await vi.advanceTimersByTimeAsync(1000);
|
await vi.advanceTimersByTimeAsync(1000);
|
||||||
|
|
|
||||||
|
|
@ -27,6 +27,7 @@ import {
|
||||||
readPaperclipSkillSyncPreference,
|
readPaperclipSkillSyncPreference,
|
||||||
writePaperclipSkillSyncPreference,
|
writePaperclipSkillSyncPreference,
|
||||||
} from "@paperclipai/adapter-utils/server-utils";
|
} from "@paperclipai/adapter-utils/server-utils";
|
||||||
|
import { trackAgentCreated } from "@paperclipai/shared/telemetry";
|
||||||
import { validate } from "../middleware/validate.js";
|
import { validate } from "../middleware/validate.js";
|
||||||
import {
|
import {
|
||||||
agentService,
|
agentService,
|
||||||
|
|
@ -62,6 +63,7 @@ import {
|
||||||
loadDefaultAgentInstructionsBundle,
|
loadDefaultAgentInstructionsBundle,
|
||||||
resolveDefaultAgentInstructionsBundleRole,
|
resolveDefaultAgentInstructionsBundleRole,
|
||||||
} from "../services/default-agent-instructions.js";
|
} from "../services/default-agent-instructions.js";
|
||||||
|
import { getTelemetryClient } from "../telemetry.js";
|
||||||
|
|
||||||
export function agentRoutes(db: Db) {
|
export function agentRoutes(db: Db) {
|
||||||
const DEFAULT_INSTRUCTIONS_PATH_KEYS: Record<string, string> = {
|
const DEFAULT_INSTRUCTIONS_PATH_KEYS: Record<string, string> = {
|
||||||
|
|
@ -1387,6 +1389,10 @@ export function agentRoutes(db: Db) {
|
||||||
desiredSkills: desiredSkillAssignment.desiredSkills,
|
desiredSkills: desiredSkillAssignment.desiredSkills,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
const telemetryClient = getTelemetryClient();
|
||||||
|
if (telemetryClient) {
|
||||||
|
trackAgentCreated(telemetryClient, { agentRole: agent.role });
|
||||||
|
}
|
||||||
|
|
||||||
await applyDefaultAgentTaskAssignGrant(
|
await applyDefaultAgentTaskAssignGrant(
|
||||||
companyId,
|
companyId,
|
||||||
|
|
@ -1469,6 +1475,10 @@ export function agentRoutes(db: Db) {
|
||||||
desiredSkills: desiredSkillAssignment.desiredSkills,
|
desiredSkills: desiredSkillAssignment.desiredSkills,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
const telemetryClient = getTelemetryClient();
|
||||||
|
if (telemetryClient) {
|
||||||
|
trackAgentCreated(telemetryClient, { agentRole: agent.role });
|
||||||
|
}
|
||||||
|
|
||||||
await applyDefaultAgentTaskAssignGrant(
|
await applyDefaultAgentTaskAssignGrant(
|
||||||
companyId,
|
companyId,
|
||||||
|
|
|
||||||
|
|
@ -6,10 +6,20 @@ import {
|
||||||
companySkillImportSchema,
|
companySkillImportSchema,
|
||||||
companySkillProjectScanRequestSchema,
|
companySkillProjectScanRequestSchema,
|
||||||
} from "@paperclipai/shared";
|
} from "@paperclipai/shared";
|
||||||
|
import { trackSkillImported } from "@paperclipai/shared/telemetry";
|
||||||
import { validate } from "../middleware/validate.js";
|
import { validate } from "../middleware/validate.js";
|
||||||
import { accessService, agentService, companySkillService, logActivity } from "../services/index.js";
|
import { accessService, agentService, companySkillService, logActivity } from "../services/index.js";
|
||||||
import { forbidden } from "../errors.js";
|
import { forbidden } from "../errors.js";
|
||||||
import { assertCompanyAccess, getActorInfo } from "./authz.js";
|
import { assertCompanyAccess, getActorInfo } from "./authz.js";
|
||||||
|
import { getTelemetryClient } from "../telemetry.js";
|
||||||
|
|
||||||
|
type SkillTelemetryInput = {
|
||||||
|
key: string;
|
||||||
|
slug: string;
|
||||||
|
sourceType: string;
|
||||||
|
sourceLocator: string | null;
|
||||||
|
metadata: Record<string, unknown> | null;
|
||||||
|
};
|
||||||
|
|
||||||
export function companySkillRoutes(db: Db) {
|
export function companySkillRoutes(db: Db) {
|
||||||
const router = Router();
|
const router = Router();
|
||||||
|
|
@ -22,6 +32,26 @@ export function companySkillRoutes(db: Db) {
|
||||||
return Boolean((agent.permissions as Record<string, unknown>).canCreateAgents);
|
return Boolean((agent.permissions as Record<string, unknown>).canCreateAgents);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function asString(value: unknown): string | null {
|
||||||
|
if (typeof value !== "string") return null;
|
||||||
|
const trimmed = value.trim();
|
||||||
|
return trimmed.length > 0 ? trimmed : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function deriveTrackedSkillRef(skill: SkillTelemetryInput): string | null {
|
||||||
|
if (skill.sourceType === "skills_sh") {
|
||||||
|
return skill.key;
|
||||||
|
}
|
||||||
|
if (skill.sourceType !== "github") {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const hostname = asString(skill.metadata?.hostname);
|
||||||
|
if (hostname !== "github.com") {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return skill.key;
|
||||||
|
}
|
||||||
|
|
||||||
async function assertCanMutateCompanySkills(req: Request, companyId: string) {
|
async function assertCanMutateCompanySkills(req: Request, companyId: string) {
|
||||||
assertCompanyAccess(req, companyId);
|
assertCompanyAccess(req, companyId);
|
||||||
|
|
||||||
|
|
@ -183,6 +213,15 @@ export function companySkillRoutes(db: Db) {
|
||||||
warningCount: result.warnings.length,
|
warningCount: result.warnings.length,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
const telemetryClient = getTelemetryClient();
|
||||||
|
if (telemetryClient) {
|
||||||
|
for (const skill of result.imported) {
|
||||||
|
trackSkillImported(telemetryClient, {
|
||||||
|
sourceType: skill.sourceType,
|
||||||
|
skillRef: deriveTrackedSkillRef(skill),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
res.status(201).json(result);
|
res.status(201).json(result);
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,11 @@
|
||||||
import { Router } from "express";
|
import { Router } from "express";
|
||||||
import type { Db } from "@paperclipai/db";
|
import type { Db } from "@paperclipai/db";
|
||||||
import { createGoalSchema, updateGoalSchema } from "@paperclipai/shared";
|
import { createGoalSchema, updateGoalSchema } from "@paperclipai/shared";
|
||||||
|
import { trackGoalCreated } from "@paperclipai/shared/telemetry";
|
||||||
import { validate } from "../middleware/validate.js";
|
import { validate } from "../middleware/validate.js";
|
||||||
import { goalService, logActivity } from "../services/index.js";
|
import { goalService, logActivity } from "../services/index.js";
|
||||||
import { assertCompanyAccess, getActorInfo } from "./authz.js";
|
import { assertCompanyAccess, getActorInfo } from "./authz.js";
|
||||||
|
import { getTelemetryClient } from "../telemetry.js";
|
||||||
|
|
||||||
export function goalRoutes(db: Db) {
|
export function goalRoutes(db: Db) {
|
||||||
const router = Router();
|
const router = Router();
|
||||||
|
|
@ -42,6 +44,10 @@ export function goalRoutes(db: Db) {
|
||||||
entityId: goal.id,
|
entityId: goal.id,
|
||||||
details: { title: goal.title },
|
details: { title: goal.title },
|
||||||
});
|
});
|
||||||
|
const telemetryClient = getTelemetryClient();
|
||||||
|
if (telemetryClient) {
|
||||||
|
trackGoalCreated(telemetryClient, { goalLevel: goal.level });
|
||||||
|
}
|
||||||
res.status(201).json(goal);
|
res.status(201).json(goal);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,11 +7,13 @@ import {
|
||||||
updateProjectSchema,
|
updateProjectSchema,
|
||||||
updateProjectWorkspaceSchema,
|
updateProjectWorkspaceSchema,
|
||||||
} from "@paperclipai/shared";
|
} from "@paperclipai/shared";
|
||||||
|
import { trackProjectCreated } from "@paperclipai/shared/telemetry";
|
||||||
import { validate } from "../middleware/validate.js";
|
import { validate } from "../middleware/validate.js";
|
||||||
import { projectService, logActivity, workspaceOperationService } from "../services/index.js";
|
import { projectService, logActivity, workspaceOperationService } from "../services/index.js";
|
||||||
import { conflict } from "../errors.js";
|
import { conflict } from "../errors.js";
|
||||||
import { assertCompanyAccess, getActorInfo } from "./authz.js";
|
import { assertCompanyAccess, getActorInfo } from "./authz.js";
|
||||||
import { startRuntimeServicesForWorkspaceControl, stopRuntimeServicesForProjectWorkspace } from "../services/workspace-runtime.js";
|
import { startRuntimeServicesForWorkspaceControl, stopRuntimeServicesForProjectWorkspace } from "../services/workspace-runtime.js";
|
||||||
|
import { getTelemetryClient } from "../telemetry.js";
|
||||||
|
|
||||||
export function projectRoutes(db: Db) {
|
export function projectRoutes(db: Db) {
|
||||||
const router = Router();
|
const router = Router();
|
||||||
|
|
@ -107,6 +109,10 @@ export function projectRoutes(db: Db) {
|
||||||
workspaceId: createdWorkspaceId,
|
workspaceId: createdWorkspaceId,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
const telemetryClient = getTelemetryClient();
|
||||||
|
if (telemetryClient) {
|
||||||
|
trackProjectCreated(telemetryClient);
|
||||||
|
}
|
||||||
res.status(201).json(hydratedProject ?? project);
|
res.status(201).json(hydratedProject ?? project);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -8,10 +8,12 @@ import {
|
||||||
updateRoutineSchema,
|
updateRoutineSchema,
|
||||||
updateRoutineTriggerSchema,
|
updateRoutineTriggerSchema,
|
||||||
} from "@paperclipai/shared";
|
} from "@paperclipai/shared";
|
||||||
|
import { trackRoutineCreated } from "@paperclipai/shared/telemetry";
|
||||||
import { validate } from "../middleware/validate.js";
|
import { validate } from "../middleware/validate.js";
|
||||||
import { accessService, logActivity, routineService } from "../services/index.js";
|
import { accessService, logActivity, routineService } from "../services/index.js";
|
||||||
import { assertCompanyAccess, getActorInfo } from "./authz.js";
|
import { assertCompanyAccess, getActorInfo } from "./authz.js";
|
||||||
import { forbidden, unauthorized } from "../errors.js";
|
import { forbidden, unauthorized } from "../errors.js";
|
||||||
|
import { getTelemetryClient } from "../telemetry.js";
|
||||||
|
|
||||||
export function routineRoutes(db: Db) {
|
export function routineRoutes(db: Db) {
|
||||||
const router = Router();
|
const router = Router();
|
||||||
|
|
@ -76,6 +78,10 @@ export function routineRoutes(db: Db) {
|
||||||
entityId: created.id,
|
entityId: created.id,
|
||||||
details: { title: created.title, assigneeAgentId: created.assigneeAgentId },
|
details: { title: created.title, assigneeAgentId: created.assigneeAgentId },
|
||||||
});
|
});
|
||||||
|
const telemetryClient = getTelemetryClient();
|
||||||
|
if (telemetryClient) {
|
||||||
|
trackRoutineCreated(telemetryClient);
|
||||||
|
}
|
||||||
res.status(201).json(created);
|
res.status(201).json(created);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -31,8 +31,10 @@ import {
|
||||||
stringifyRoutineVariableValue,
|
stringifyRoutineVariableValue,
|
||||||
syncRoutineVariablesWithTemplate,
|
syncRoutineVariablesWithTemplate,
|
||||||
} from "@paperclipai/shared";
|
} from "@paperclipai/shared";
|
||||||
|
import { trackRoutineRun } from "@paperclipai/shared/telemetry";
|
||||||
import { conflict, forbidden, notFound, unauthorized, unprocessable } from "../errors.js";
|
import { conflict, forbidden, notFound, unauthorized, unprocessable } from "../errors.js";
|
||||||
import { logger } from "../middleware/logger.js";
|
import { logger } from "../middleware/logger.js";
|
||||||
|
import { getTelemetryClient } from "../telemetry.js";
|
||||||
import { issueService } from "./issues.js";
|
import { issueService } from "./issues.js";
|
||||||
import { secretService } from "./secrets.js";
|
import { secretService } from "./secrets.js";
|
||||||
import { parseCron, validateCron } from "./cron.js";
|
import { parseCron, validateCron } from "./cron.js";
|
||||||
|
|
@ -856,6 +858,14 @@ export function routineService(db: Db, deps: { heartbeat?: IssueAssignmentWakeup
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const telemetryClient = getTelemetryClient();
|
||||||
|
if (telemetryClient) {
|
||||||
|
trackRoutineRun(telemetryClient, {
|
||||||
|
source: run.source,
|
||||||
|
status: run.status,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
return run;
|
return run;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue