Speed up issue search

This commit is contained in:
dotta
2026-04-06 20:30:50 -05:00
parent 0edac73a68
commit 5136381d8f
15 changed files with 13127 additions and 27 deletions
+24
View File
@@ -45,6 +45,11 @@ type TableDefinition = {
tablename: string;
};
type ExtensionDefinition = {
extension_name: string;
schema_name: string;
};
const DRIZZLE_SCHEMA = "drizzle";
const DRIZZLE_MIGRATIONS_TABLE = "__drizzle_migrations";
const DEFAULT_BACKUP_WRITE_BUFFER_BYTES = 1024 * 1024;
@@ -340,6 +345,25 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise
emit("");
}
const extensions = await sql<ExtensionDefinition[]>`
SELECT
e.extname AS extension_name,
n.nspname AS schema_name
FROM pg_extension e
JOIN pg_namespace n ON n.oid = e.extnamespace
WHERE e.extname <> 'plpgsql'
ORDER BY e.extname
`;
if (extensions.length > 0) {
emit("-- Extensions");
for (const extension of extensions) {
emitStatement(
`CREATE EXTENSION IF NOT EXISTS ${quoteIdentifier(extension.extension_name)} WITH SCHEMA ${quoteIdentifier(extension.schema_name)};`,
);
}
emit("");
}
if (sequences.length > 0) {
emit("-- Sequences");
for (const seq of sequences) {
@@ -0,0 +1,5 @@
CREATE EXTENSION IF NOT EXISTS pg_trgm;--> statement-breakpoint
CREATE INDEX "issue_comments_body_search_idx" ON "issue_comments" USING gin ("body" gin_trgm_ops);--> statement-breakpoint
CREATE INDEX "issues_title_search_idx" ON "issues" USING gin ("title" gin_trgm_ops);--> statement-breakpoint
CREATE INDEX "issues_identifier_search_idx" ON "issues" USING gin ("identifier" gin_trgm_ops);--> statement-breakpoint
CREATE INDEX "issues_description_search_idx" ON "issues" USING gin ("description" gin_trgm_ops);
File diff suppressed because it is too large Load Diff
@@ -351,6 +351,13 @@
"when": 1775349863293,
"tag": "0049_flawless_abomination",
"breakpoints": true
},
{
"idx": 50,
"version": "7",
"when": 1775524651831,
"tag": "0051_young_korg",
"breakpoints": true
}
]
}
}
+1
View File
@@ -31,5 +31,6 @@ export const issueComments = pgTable(
table.issueId,
table.createdAt,
),
bodySearchIdx: index("issue_comments_body_search_idx").using("gin", table.body.op("gin_trgm_ops")),
}),
);
+3
View File
@@ -76,6 +76,9 @@ export const issues = pgTable(
projectWorkspaceIdx: index("issues_company_project_workspace_idx").on(table.companyId, table.projectWorkspaceId),
executionWorkspaceIdx: index("issues_company_execution_workspace_idx").on(table.companyId, table.executionWorkspaceId),
identifierIdx: uniqueIndex("issues_identifier_idx").on(table.identifier),
titleSearchIdx: index("issues_title_search_idx").using("gin", table.title.op("gin_trgm_ops")),
identifierSearchIdx: index("issues_identifier_search_idx").using("gin", table.identifier.op("gin_trgm_ops")),
descriptionSearchIdx: index("issues_description_search_idx").using("gin", table.description.op("gin_trgm_ops")),
openRoutineExecutionIdx: uniqueIndex("issues_open_routine_execution_uq")
.on(table.companyId, table.originKind, table.originId)
.where(
@@ -249,6 +249,55 @@ describeEmbeddedPostgres("issueService.list participantAgentId", () => {
expect(result.map((issue) => issue.id)).toEqual([matchedIssueId]);
});
it("applies result limits to issue search", async () => {
const companyId = randomUUID();
await db.insert(companies).values({
id: companyId,
name: "Paperclip",
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
requireBoardApprovalForNewAgents: false,
});
const exactIdentifierId = randomUUID();
const titleMatchId = randomUUID();
const descriptionMatchId = randomUUID();
await db.insert(issues).values([
{
id: exactIdentifierId,
companyId,
issueNumber: 42,
identifier: "PAP-42",
title: "Completely unrelated",
status: "todo",
priority: "medium",
},
{
id: titleMatchId,
companyId,
title: "Search ranking issue",
status: "todo",
priority: "medium",
},
{
id: descriptionMatchId,
companyId,
title: "Another item",
description: "Contains the search keyword",
status: "todo",
priority: "medium",
},
]);
const result = await svc.list(companyId, {
q: "search",
limit: 2,
});
expect(result.map((issue) => issue.id)).toEqual([titleMatchId, descriptionMatchId]);
});
it("accepts issue identifiers through getById", async () => {
const companyId = randomUUID();
const issueId = randomUUID();
@@ -3,6 +3,8 @@ import express from "express";
import request from "supertest";
import { privateHostnameGuard } from "../middleware/private-hostname-guard.js";
const unknownHostname = "blocked-host.invalid";
function createApp(opts: { enabled: boolean; allowedHostnames?: string[]; bindHost?: string }) {
const app = express();
app.use(
@@ -42,15 +44,15 @@ describe("privateHostnameGuard", () => {
it("blocks unknown hostnames with remediation command", async () => {
const app = createApp({ enabled: true, allowedHostnames: ["some-other-host"] });
const res = await request(app).get("/api/health").set("Host", "dotta-macbook-pro:3100");
const res = await request(app).get("/api/health").set("Host", `${unknownHostname}:3100`);
expect(res.status).toBe(403);
expect(res.body?.error).toContain("please run pnpm paperclipai allowed-hostname dotta-macbook-pro");
expect(res.body?.error).toContain(`please run pnpm paperclipai allowed-hostname ${unknownHostname}`);
});
it("blocks unknown hostnames on page routes with plain-text remediation command", async () => {
const app = createApp({ enabled: true, allowedHostnames: ["some-other-host"] });
const res = await request(app).get("/dashboard").set("Host", "dotta-macbook-pro:3100");
const res = await request(app).get("/dashboard").set("Host", `${unknownHostname}:3100`);
expect(res.status).toBe(403);
expect(res.text).toContain("please run pnpm paperclipai allowed-hostname dotta-macbook-pro");
expect(res.text).toContain(`please run pnpm paperclipai allowed-hostname ${unknownHostname}`);
}, 20_000);
});
+8
View File
@@ -346,6 +346,9 @@ export function issueRoutes(
unreadForUserFilterRaw === "me" && req.actor.type === "board"
? req.actor.userId
: unreadForUserFilterRaw;
const rawLimit = req.query.limit as string | undefined;
const parsedLimit = rawLimit ? Number.parseInt(rawLimit, 10) : null;
const limit = parsedLimit ?? undefined;
if (assigneeUserFilterRaw === "me" && (!assigneeUserId || req.actor.type !== "board")) {
res.status(403).json({ error: "assigneeUserId=me requires board authentication" });
@@ -363,6 +366,10 @@ export function issueRoutes(
res.status(403).json({ error: "unreadForUserId=me requires board authentication" });
return;
}
if (rawLimit !== undefined && (parsedLimit === null || !Number.isInteger(parsedLimit) || parsedLimit <= 0)) {
res.status(400).json({ error: "limit must be a positive integer" });
return;
}
const result = await svc.list(companyId, {
status: req.query.status as string | undefined,
@@ -381,6 +388,7 @@ export function issueRoutes(
includeRoutineExecutions:
req.query.includeRoutineExecutions === "true" || req.query.includeRoutineExecutions === "1",
q: req.query.q as string | undefined,
limit,
});
res.json(result);
});
+6 -1
View File
@@ -80,6 +80,7 @@ export interface IssueFilters {
originId?: string;
includeRoutineExecutions?: boolean;
q?: string;
limit?: number;
}
type IssueRow = typeof issues.$inferSelect;
@@ -911,6 +912,9 @@ export function issueService(db: Db) {
return {
list: async (companyId: string, filters?: IssueFilters) => {
const conditions = [eq(issues.companyId, companyId)];
const limit = typeof filters?.limit === "number" && Number.isFinite(filters.limit)
? Math.max(1, Math.floor(filters.limit))
: undefined;
const touchedByUserId = filters?.touchedByUserId?.trim() || undefined;
const inboxArchivedByUserId = filters?.inboxArchivedByUserId?.trim() || undefined;
const unreadForUserId = filters?.unreadForUserId?.trim() || undefined;
@@ -999,7 +1003,7 @@ export function issueService(db: Db) {
END
`;
const canonicalLastActivityAt = issueCanonicalLastActivityAtExpr(companyId);
const rows = await db
const baseQuery = db
.select()
.from(issues)
.where(and(...conditions))
@@ -1009,6 +1013,7 @@ export function issueService(db: Db) {
desc(canonicalLastActivityAt),
desc(issues.updatedAt),
);
const rows = limit === undefined ? await baseQuery : await baseQuery.limit(limit);
const withLabels = await withIssueLabels(db, rows);
const runMap = await activeRunMapForIssues(db, withLabels);
const withRuns = withActiveRuns(withLabels, runMap);
+2
View File
@@ -36,6 +36,7 @@ export const issuesApi = {
originId?: string;
includeRoutineExecutions?: boolean;
q?: string;
limit?: number;
},
) => {
const params = new URLSearchParams();
@@ -53,6 +54,7 @@ export const issuesApi = {
if (filters?.originId) params.set("originId", filters.originId);
if (filters?.includeRoutineExecutions) params.set("includeRoutineExecutions", "true");
if (filters?.q) params.set("q", filters.q);
if (filters?.limit) params.set("limit", String(filters.limit));
const qs = params.toString();
return api.get<Issue[]>(`/companies/${companyId}/issues${qs ? `?${qs}` : ""}`);
},
+3 -3
View File
@@ -60,12 +60,12 @@ export function CommandPalette() {
const { data: issues = [] } = useQuery({
queryKey: queryKeys.issues.list(selectedCompanyId!),
queryFn: () => issuesApi.list(selectedCompanyId!),
enabled: !!selectedCompanyId && open,
enabled: !!selectedCompanyId && open && searchQuery.length === 0,
});
const { data: searchedIssues = [] } = useQuery({
queryKey: queryKeys.issues.search(selectedCompanyId!, searchQuery),
queryFn: () => issuesApi.list(selectedCompanyId!, { q: searchQuery }),
queryKey: queryKeys.issues.search(selectedCompanyId!, searchQuery, undefined, 10),
queryFn: () => issuesApi.list(selectedCompanyId!, { q: searchQuery, limit: 10 }),
enabled: !!selectedCompanyId && open && searchQuery.length > 0,
});
+172
View File
@@ -0,0 +1,172 @@
// @vitest-environment jsdom
import { act } from "react";
import { createRoot } from "react-dom/client";
import type { ReactNode } from "react";
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import type { Issue } from "@paperclipai/shared";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { IssuesList } from "./IssuesList";
const companyState = vi.hoisted(() => ({
selectedCompanyId: "company-1",
}));
const dialogState = vi.hoisted(() => ({
openNewIssue: vi.fn(),
}));
const mockIssuesApi = vi.hoisted(() => ({
list: vi.fn(),
listLabels: vi.fn(),
}));
const mockAuthApi = vi.hoisted(() => ({
getSession: vi.fn(),
}));
vi.mock("../context/CompanyContext", () => ({
useCompany: () => companyState,
}));
vi.mock("../context/DialogContext", () => ({
useDialog: () => dialogState,
}));
vi.mock("../api/issues", () => ({
issuesApi: mockIssuesApi,
}));
vi.mock("../api/auth", () => ({
authApi: mockAuthApi,
}));
vi.mock("./IssueRow", () => ({
IssueRow: ({ issue }: { issue: Issue }) => <div data-testid="issue-row">{issue.title}</div>,
}));
vi.mock("./KanbanBoard", () => ({
KanbanBoard: () => null,
}));
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(globalThis as any).IS_REACT_ACT_ENVIRONMENT = true;
function createIssue(overrides: Partial<Issue> = {}): Issue {
return {
id: "issue-1",
identifier: "PAP-1",
companyId: "company-1",
projectId: null,
projectWorkspaceId: null,
goalId: null,
parentId: null,
title: "Issue title",
description: null,
status: "todo",
priority: "medium",
assigneeAgentId: null,
assigneeUserId: null,
createdByAgentId: null,
createdByUserId: null,
issueNumber: 1,
requestDepth: 0,
billingCode: null,
assigneeAdapterOverrides: null,
executionWorkspaceId: null,
executionWorkspacePreference: null,
executionWorkspaceSettings: null,
checkoutRunId: null,
executionRunId: null,
executionAgentNameKey: null,
executionLockedAt: null,
startedAt: null,
completedAt: null,
cancelledAt: null,
hiddenAt: null,
createdAt: new Date("2026-04-07T00:00:00.000Z"),
updatedAt: new Date("2026-04-07T00:00:00.000Z"),
labels: [],
labelIds: [],
myLastTouchAt: null,
lastExternalCommentAt: null,
isUnreadForMe: false,
...overrides,
};
}
async function flush() {
await act(async () => {
await Promise.resolve();
});
}
function renderWithQueryClient(node: ReactNode, container: HTMLDivElement) {
const root = createRoot(container);
const queryClient = new QueryClient({
defaultOptions: {
queries: {
retry: false,
},
},
});
act(() => {
root.render(
<QueryClientProvider client={queryClient}>
{node}
</QueryClientProvider>,
);
});
return { root, queryClient };
}
describe("IssuesList", () => {
let container: HTMLDivElement;
beforeEach(() => {
container = document.createElement("div");
document.body.appendChild(container);
dialogState.openNewIssue.mockReset();
mockIssuesApi.list.mockReset();
mockIssuesApi.listLabels.mockReset();
mockAuthApi.getSession.mockReset();
mockIssuesApi.listLabels.mockResolvedValue([]);
mockAuthApi.getSession.mockResolvedValue({ user: null, session: null });
});
afterEach(() => {
container.remove();
});
it("renders server search results instead of filtering the full issue list locally", async () => {
const localIssue = createIssue({ id: "issue-local", identifier: "PAP-1", title: "Local issue" });
const serverIssue = createIssue({ id: "issue-server", identifier: "PAP-2", title: "Server result" });
mockIssuesApi.list.mockResolvedValue([serverIssue]);
const { root } = renderWithQueryClient(
<IssuesList
issues={[localIssue]}
agents={[]}
projects={[]}
viewStateKey="paperclip:test-issues"
initialSearch="server"
onUpdateIssue={() => undefined}
/>,
container,
);
await flush();
await flush();
expect(mockIssuesApi.list).toHaveBeenCalledWith("company-1", { q: "server", projectId: undefined });
expect(container.textContent).toContain("Server result");
expect(container.textContent).not.toContain("Local issue");
act(() => {
root.unmount();
});
});
});
+2 -16
View File
@@ -145,18 +145,6 @@ function countActiveFilters(state: IssueViewState): number {
return count;
}
function matchesIssueSearch(issue: Issue, normalizedSearch: string): boolean {
if (!normalizedSearch) return true;
return [
issue.identifier,
issue.title,
issue.description,
]
.filter((value): value is string => Boolean(value))
.some((value) => value.toLowerCase().includes(normalizedSearch));
}
/* ── Component ── */
interface Agent {
@@ -278,12 +266,10 @@ export function IssuesList({
}, [agents]);
const filtered = useMemo(() => {
const sourceIssues = normalizedIssueSearch.length > 0
? issues.filter((issue) => matchesIssueSearch(issue, normalizedIssueSearch))
: issues;
const sourceIssues = normalizedIssueSearch.length > 0 ? searchedIssues : issues;
const filteredByControls = applyFilters(sourceIssues, viewState, currentUserId);
return sortIssues(filteredByControls, viewState);
}, [issues, viewState, normalizedIssueSearch, currentUserId]);
}, [issues, searchedIssues, viewState, normalizedIssueSearch, currentUserId]);
const { data: labels } = useQuery({
queryKey: queryKeys.issues.labels(selectedCompanyId!),
+2 -2
View File
@@ -30,8 +30,8 @@ export const queryKeys = {
},
issues: {
list: (companyId: string) => ["issues", companyId] as const,
search: (companyId: string, q: string, projectId?: string) =>
["issues", companyId, "search", q, projectId ?? "__all-projects__"] as const,
search: (companyId: string, q: string, projectId?: string, limit?: number) =>
["issues", companyId, "search", q, projectId ?? "__all-projects__", limit ?? "__no-limit__"] as const,
listAssignedToMe: (companyId: string) => ["issues", companyId, "assigned-to-me"] as const,
listMineByMe: (companyId: string) => ["issues", companyId, "mine-by-me"] as const,
listTouchedByMe: (companyId: string) => ["issues", companyId, "touched-by-me"] as const,