From fcbae62baf1e821e2b65f64545381defccda3230 Mon Sep 17 00:00:00 2001 From: Aron Prins Date: Tue, 7 Apr 2026 09:54:39 +0200 Subject: [PATCH] feat(backups): tiered daily/weekly/monthly retention with UI controls Replace single retentionDays with a three-tier BackupRetentionPolicy: - Daily: keep all backups (presets: 3, 7, 14 days; default 7) - Weekly: keep one per calendar week (presets: 1, 2, 4 weeks; default 4) - Monthly: keep one per calendar month (presets: 1, 3, 6 months; default 1) Pruning sorts backups newest-first and applies each tier's cutoff, keeping only the newest entry per ISO week/month bucket. The Instance Settings General page now shows three preset selectors (no icon, matches existing page design). Remove Database icon import. --- cli/src/commands/db-backup.ts | 2 +- cli/src/commands/worktree.ts | 2 +- packages/db/src/backup-lib.test.ts | 2 +- packages/db/src/backup-lib.ts | 97 ++++++++++++-- packages/db/src/backup.ts | 4 +- packages/db/src/index.ts | 1 + packages/shared/src/index.ts | 8 +- packages/shared/src/types/index.ts | 4 +- packages/shared/src/types/instance.ts | 20 ++- packages/shared/src/validators/instance.ts | 26 +++- server/src/index.ts | 6 +- server/src/services/instance-settings.ts | 6 +- ui/src/pages/InstanceGeneralSettings.tsx | 144 +++++++++++++++------ 13 files changed, 243 insertions(+), 79 deletions(-) diff --git a/cli/src/commands/db-backup.ts b/cli/src/commands/db-backup.ts index bdbf739f..7e1adce5 100644 --- a/cli/src/commands/db-backup.ts +++ b/cli/src/commands/db-backup.ts @@ -73,7 +73,7 @@ export async function dbBackupCommand(opts: DbBackupOptions): Promise { const result = await runDatabaseBackup({ connectionString: connection.value, backupDir, - retentionDays, + retention: { dailyDays: retentionDays, weeklyWeeks: 4, monthlyMonths: 1 }, filenamePrefix, }); spinner.stop(`Backup saved: ${formatDatabaseBackupResult(result)}`); diff --git a/cli/src/commands/worktree.ts b/cli/src/commands/worktree.ts index 3025e955..d67bd154 100644 --- a/cli/src/commands/worktree.ts +++ b/cli/src/commands/worktree.ts @@ -903,7 +903,7 @@ async function seedWorktreeDatabase(input: { const backup = await runDatabaseBackup({ connectionString: sourceConnectionString, backupDir: path.resolve(input.targetPaths.backupDir, "seed"), - retentionDays: 7, + retention: { dailyDays: 7, weeklyWeeks: 4, monthlyMonths: 1 }, filenamePrefix: `${input.instanceId}-seed`, includeMigrationJournal: true, excludeTables: seedPlan.excludedTables, diff --git a/packages/db/src/backup-lib.test.ts b/packages/db/src/backup-lib.test.ts index 2367d26d..2ea9b070 100644 --- a/packages/db/src/backup-lib.test.ts +++ b/packages/db/src/backup-lib.test.ts @@ -125,7 +125,7 @@ describeEmbeddedPostgres("runDatabaseBackup", () => { const result = await runDatabaseBackup({ connectionString: sourceConnectionString, backupDir, - retentionDays: 7, + retention: { dailyDays: 7, weeklyWeeks: 4, monthlyMonths: 1 }, filenamePrefix: "paperclip-test", }); diff --git a/packages/db/src/backup-lib.ts b/packages/db/src/backup-lib.ts index 36b5ee98..0fe70f13 100644 --- a/packages/db/src/backup-lib.ts +++ b/packages/db/src/backup-lib.ts @@ -5,10 +5,16 @@ import { pipeline } from "node:stream/promises"; import { createGunzip, createGzip } from "node:zlib"; import postgres from "postgres"; +export type BackupRetentionPolicy = { + dailyDays: number; + weeklyWeeks: number; + monthlyMonths: number; +}; + export type RunDatabaseBackupOptions = { connectionString: string; backupDir: string; - retentionDays: number; + retention: BackupRetentionPolicy; filenamePrefix?: string; connectTimeoutSeconds?: number; includeMigrationJournal?: boolean; @@ -77,24 +83,91 @@ function timestamp(date: Date = new Date()): string { return `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}-${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`; } -function pruneOldBackups(backupDir: string, retentionDays: number, filenamePrefix: string): number { +/** + * ISO week key for grouping backups by calendar week (ISO 8601). + */ +function isoWeekKey(date: Date): string { + const d = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate())); + d.setUTCDate(d.getUTCDate() + 4 - (d.getUTCDay() || 7)); + const yearStart = new Date(Date.UTC(d.getUTCFullYear(), 0, 1)); + const weekNo = Math.ceil(((d.getTime() - yearStart.getTime()) / 86400000 + 1) / 7); + return `${d.getUTCFullYear()}-W${String(weekNo).padStart(2, "0")}`; +} + +function monthKey(date: Date): string { + return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, "0")}`; +} + +/** + * Tiered backup pruning: + * - Daily tier: keep ALL backups from the last `dailyDays` days + * - Weekly tier: keep the NEWEST backup per calendar week for `weeklyWeeks` weeks + * - Monthly tier: keep the NEWEST backup per calendar month for `monthlyMonths` months + * - Everything else is deleted + */ +function pruneOldBackups(backupDir: string, retention: BackupRetentionPolicy, filenamePrefix: string): number { if (!existsSync(backupDir)) return 0; - const safeRetention = Math.max(1, Math.trunc(retentionDays)); - const cutoff = Date.now() - safeRetention * 24 * 60 * 60 * 1000; - let pruned = 0; + + const now = Date.now(); + const dailyCutoff = now - Math.max(1, retention.dailyDays) * 24 * 60 * 60 * 1000; + const weeklyCutoff = now - Math.max(1, retention.weeklyWeeks) * 7 * 24 * 60 * 60 * 1000; + const monthlyCutoff = now - Math.max(1, retention.monthlyMonths) * 30 * 24 * 60 * 60 * 1000; + + type BackupEntry = { name: string; fullPath: string; mtimeMs: number }; + const entries: BackupEntry[] = []; for (const name of readdirSync(backupDir)) { if (!name.startsWith(`${filenamePrefix}-`)) continue; if (!name.endsWith(".sql") && !name.endsWith(".sql.gz")) continue; const fullPath = resolve(backupDir, name); const stat = statSync(fullPath); - if (stat.mtimeMs < cutoff) { - unlinkSync(fullPath); - pruned++; - } + entries.push({ name, fullPath, mtimeMs: stat.mtimeMs }); } - return pruned; + // Sort newest first so the first entry per week/month bucket is the one we keep + entries.sort((a, b) => b.mtimeMs - a.mtimeMs); + + const keepWeekBuckets = new Set(); + const keepMonthBuckets = new Set(); + const toDelete: string[] = []; + + for (const entry of entries) { + // Daily tier — keep everything within dailyDays + if (entry.mtimeMs >= dailyCutoff) continue; + + const date = new Date(entry.mtimeMs); + const week = isoWeekKey(date); + const month = monthKey(date); + + // Weekly tier — keep newest per calendar week + if (entry.mtimeMs >= weeklyCutoff) { + if (keepWeekBuckets.has(week)) { + toDelete.push(entry.fullPath); + } else { + keepWeekBuckets.add(week); + } + continue; + } + + // Monthly tier — keep newest per calendar month + if (entry.mtimeMs >= monthlyCutoff) { + if (keepMonthBuckets.has(month)) { + toDelete.push(entry.fullPath); + } else { + keepMonthBuckets.add(month); + } + continue; + } + + // Beyond all retention tiers — delete + toDelete.push(entry.fullPath); + } + + for (const filePath of toDelete) { + unlinkSync(filePath); + } + + return toDelete.length; } function formatBackupSize(sizeBytes: number): string { @@ -287,7 +360,7 @@ export function createBufferedTextFileWriter(filePath: string, maxBufferedBytes export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise { const filenamePrefix = opts.filenamePrefix ?? "paperclip"; - const retentionDays = Math.max(1, Math.trunc(opts.retentionDays)); + const retention = opts.retention; const connectTimeout = Math.max(1, Math.trunc(opts.connectTimeoutSeconds ?? 5)); const includeMigrationJournal = opts.includeMigrationJournal === true; const excludedTableNames = normalizeTableNameSet(opts.excludeTables); @@ -678,7 +751,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise unlinkSync(sqlFile); const sizeBytes = statSync(backupFile).size; - const prunedCount = pruneOldBackups(opts.backupDir, retentionDays, filenamePrefix); + const prunedCount = pruneOldBackups(opts.backupDir, retention, filenamePrefix); return { backupFile, diff --git a/packages/db/src/backup.ts b/packages/db/src/backup.ts index f07dc646..c11822c9 100644 --- a/packages/db/src/backup.ts +++ b/packages/db/src/backup.ts @@ -85,7 +85,7 @@ function resolveBackupDir(config: PartialConfig | null): string { } function resolveRetentionDays(config: PartialConfig | null): number { - return asPositiveInt(config?.database?.backup?.retentionDays) ?? 30; + return asPositiveInt(config?.database?.backup?.retentionDays) ?? 7; } async function main() { @@ -103,7 +103,7 @@ async function main() { const result = await runDatabaseBackup({ connectionString, backupDir, - retentionDays, + retention: { dailyDays: retentionDays, weeklyWeeks: 4, monthlyMonths: 1 }, filenamePrefix: "paperclip", }); diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index cf4a2633..200e2981 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -21,6 +21,7 @@ export { runDatabaseBackup, runDatabaseRestore, formatDatabaseBackupResult, + type BackupRetentionPolicy, type RunDatabaseBackupOptions, type RunDatabaseBackupResult, type RunDatabaseRestoreOptions, diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index ef9ce5ba..e9075356 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -189,7 +189,7 @@ export type { InstanceExperimentalSettings, InstanceGeneralSettings, InstanceSettings, - BackupRetentionDays, + BackupRetentionPolicy, Agent, AgentAccessState, AgentChainOfCommandEntry, @@ -371,8 +371,10 @@ export { } from "./types/feedback.js"; export { - BACKUP_RETENTION_PRESETS, - DEFAULT_BACKUP_RETENTION_DAYS, + DAILY_RETENTION_PRESETS, + WEEKLY_RETENTION_PRESETS, + MONTHLY_RETENTION_PRESETS, + DEFAULT_BACKUP_RETENTION, } from "./types/instance.js"; export { diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index 20d38a7a..1f0ec2c7 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -11,8 +11,8 @@ export type { FeedbackTraceBundleFile, FeedbackTraceBundle, } from "./feedback.js"; -export type { InstanceExperimentalSettings, InstanceGeneralSettings, InstanceSettings, BackupRetentionDays } from "./instance.js"; -export { BACKUP_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION_DAYS } from "./instance.js"; +export type { InstanceExperimentalSettings, InstanceGeneralSettings, InstanceSettings, BackupRetentionPolicy } from "./instance.js"; +export { DAILY_RETENTION_PRESETS, WEEKLY_RETENTION_PRESETS, MONTHLY_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION } from "./instance.js"; export type { CompanySkillSourceType, CompanySkillTrustLevel, diff --git a/packages/shared/src/types/instance.ts b/packages/shared/src/types/instance.ts index 30f07f7d..4e83c925 100644 --- a/packages/shared/src/types/instance.ts +++ b/packages/shared/src/types/instance.ts @@ -1,14 +1,26 @@ import type { FeedbackDataSharingPreference } from "./feedback.js"; -export const BACKUP_RETENTION_PRESETS = [7, 14, 30] as const; -export type BackupRetentionDays = (typeof BACKUP_RETENTION_PRESETS)[number]; -export const DEFAULT_BACKUP_RETENTION_DAYS: BackupRetentionDays = 7; +export const DAILY_RETENTION_PRESETS = [3, 7, 14] as const; +export const WEEKLY_RETENTION_PRESETS = [1, 2, 4] as const; +export const MONTHLY_RETENTION_PRESETS = [1, 3, 6] as const; + +export interface BackupRetentionPolicy { + dailyDays: (typeof DAILY_RETENTION_PRESETS)[number]; + weeklyWeeks: (typeof WEEKLY_RETENTION_PRESETS)[number]; + monthlyMonths: (typeof MONTHLY_RETENTION_PRESETS)[number]; +} + +export const DEFAULT_BACKUP_RETENTION: BackupRetentionPolicy = { + dailyDays: 7, + weeklyWeeks: 4, + monthlyMonths: 1, +}; export interface InstanceGeneralSettings { censorUsernameInLogs: boolean; keyboardShortcuts: boolean; feedbackDataSharingPreference: FeedbackDataSharingPreference; - backupRetentionDays: BackupRetentionDays; + backupRetention: BackupRetentionPolicy; } export interface InstanceExperimentalSettings { diff --git a/packages/shared/src/validators/instance.ts b/packages/shared/src/validators/instance.ts index a62f1996..930e183f 100644 --- a/packages/shared/src/validators/instance.ts +++ b/packages/shared/src/validators/instance.ts @@ -1,13 +1,25 @@ import { z } from "zod"; import { DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE } from "../types/feedback.js"; -import { BACKUP_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION_DAYS } from "../types/instance.js"; +import { + DAILY_RETENTION_PRESETS, + WEEKLY_RETENTION_PRESETS, + MONTHLY_RETENTION_PRESETS, + DEFAULT_BACKUP_RETENTION, +} from "../types/instance.js"; import { feedbackDataSharingPreferenceSchema } from "./feedback.js"; -export const backupRetentionDaysSchema = z.number().refine( - (v): v is (typeof BACKUP_RETENTION_PRESETS)[number] => - (BACKUP_RETENTION_PRESETS as readonly number[]).includes(v), - { message: `Must be one of: ${BACKUP_RETENTION_PRESETS.join(", ")}` }, -); +function presetSchema(presets: T, label: string) { + return z.number().refine( + (v): v is T[number] => (presets as readonly number[]).includes(v), + { message: `${label} must be one of: ${presets.join(", ")}` }, + ); +} + +export const backupRetentionPolicySchema = z.object({ + dailyDays: presetSchema(DAILY_RETENTION_PRESETS, "dailyDays").default(DEFAULT_BACKUP_RETENTION.dailyDays), + weeklyWeeks: presetSchema(WEEKLY_RETENTION_PRESETS, "weeklyWeeks").default(DEFAULT_BACKUP_RETENTION.weeklyWeeks), + monthlyMonths: presetSchema(MONTHLY_RETENTION_PRESETS, "monthlyMonths").default(DEFAULT_BACKUP_RETENTION.monthlyMonths), +}); export const instanceGeneralSettingsSchema = z.object({ censorUsernameInLogs: z.boolean().default(false), @@ -15,7 +27,7 @@ export const instanceGeneralSettingsSchema = z.object({ feedbackDataSharingPreference: feedbackDataSharingPreferenceSchema.default( DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE, ), - backupRetentionDays: backupRetentionDaysSchema.default(DEFAULT_BACKUP_RETENTION_DAYS), + backupRetention: backupRetentionPolicySchema.default(DEFAULT_BACKUP_RETENTION), }).strict(); export const patchInstanceGeneralSettingsSchema = instanceGeneralSettingsSchema.partial(); diff --git a/server/src/index.ts b/server/src/index.ts index 768d1fd6..b63789a2 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -642,12 +642,12 @@ export async function startServer(): Promise { try { // Read retention from Instance Settings (DB) so changes take effect without restart const generalSettings = await settingsSvc.getGeneral(); - const retentionDays = generalSettings.backupRetentionDays; + const retention = generalSettings.backupRetention; const result = await runDatabaseBackup({ connectionString: activeDatabaseConnectionString, backupDir: config.databaseBackupDir, - retentionDays, + retention, filenamePrefix: "paperclip", }); logger.info( @@ -656,7 +656,7 @@ export async function startServer(): Promise { sizeBytes: result.sizeBytes, prunedCount: result.prunedCount, backupDir: config.databaseBackupDir, - retentionDays, + retention, }, `Automatic database backup complete: ${formatDatabaseBackupResult(result)}`, ); diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index 5c7db61d..65a12632 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -2,7 +2,7 @@ import type { Db } from "@paperclipai/db"; import { companies, instanceSettings } from "@paperclipai/db"; import { DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE, - DEFAULT_BACKUP_RETENTION_DAYS, + DEFAULT_BACKUP_RETENTION, instanceGeneralSettingsSchema, type InstanceGeneralSettings, instanceExperimentalSettingsSchema, @@ -23,14 +23,14 @@ function normalizeGeneralSettings(raw: unknown): InstanceGeneralSettings { keyboardShortcuts: parsed.data.keyboardShortcuts ?? false, feedbackDataSharingPreference: parsed.data.feedbackDataSharingPreference ?? DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE, - backupRetentionDays: parsed.data.backupRetentionDays ?? DEFAULT_BACKUP_RETENTION_DAYS, + backupRetention: parsed.data.backupRetention ?? DEFAULT_BACKUP_RETENTION, }; } return { censorUsernameInLogs: false, keyboardShortcuts: false, feedbackDataSharingPreference: DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE, - backupRetentionDays: DEFAULT_BACKUP_RETENTION_DAYS, + backupRetention: DEFAULT_BACKUP_RETENTION, }; } diff --git a/ui/src/pages/InstanceGeneralSettings.tsx b/ui/src/pages/InstanceGeneralSettings.tsx index 9004e418..04c3638d 100644 --- a/ui/src/pages/InstanceGeneralSettings.tsx +++ b/ui/src/pages/InstanceGeneralSettings.tsx @@ -1,8 +1,13 @@ import { useEffect, useState } from "react"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import type { PatchInstanceGeneralSettings, BackupRetentionDays } from "@paperclipai/shared"; -import { BACKUP_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION_DAYS } from "@paperclipai/shared"; -import { Database, LogOut, SlidersHorizontal } from "lucide-react"; +import type { PatchInstanceGeneralSettings, BackupRetentionPolicy } from "@paperclipai/shared"; +import { + DAILY_RETENTION_PRESETS, + WEEKLY_RETENTION_PRESETS, + MONTHLY_RETENTION_PRESETS, + DEFAULT_BACKUP_RETENTION, +} from "@paperclipai/shared"; +import { LogOut, SlidersHorizontal } from "lucide-react"; import { authApi } from "@/api/auth"; import { instanceSettingsApi } from "@/api/instanceSettings"; import { Button } from "../components/ui/button"; @@ -68,7 +73,7 @@ export function InstanceGeneralSettings() { const censorUsernameInLogs = generalQuery.data?.censorUsernameInLogs === true; const keyboardShortcuts = generalQuery.data?.keyboardShortcuts === true; const feedbackDataSharingPreference = generalQuery.data?.feedbackDataSharingPreference ?? "prompt"; - const backupRetentionDays: BackupRetentionDays = generalQuery.data?.backupRetentionDays ?? DEFAULT_BACKUP_RETENTION_DAYS; + const backupRetention: BackupRetentionPolicy = generalQuery.data?.backupRetention ?? DEFAULT_BACKUP_RETENTION; return (
@@ -126,44 +131,103 @@ export function InstanceGeneralSettings() {
-
-
- -
-

Backup retention

-

- How long to keep automatic database backups before pruning. Backups are compressed - with gzip to minimize disk usage. -

+
+
+

Backup retention

+

+ Configure how long to keep automatic database backups at each tier. Daily backups + are kept in full, then thinned to one per week and one per month. Backups are + compressed with gzip. +

+
+ +
+

Daily

+
+ {DAILY_RETENTION_PRESETS.map((days) => { + const active = backupRetention.dailyDays === days; + return ( + + ); + })}
-
- {BACKUP_RETENTION_PRESETS.map((days) => { - const active = backupRetentionDays === days; - const label = - days === 7 ? "7 days" : days === 14 ? "2 weeks" : "1 month"; - return ( - - ); - })} + +
+

Weekly

+
+ {WEEKLY_RETENTION_PRESETS.map((weeks) => { + const active = backupRetention.weeklyWeeks === weeks; + const label = weeks === 1 ? "1 week" : `${weeks} weeks`; + return ( + + ); + })} +
+
+ +
+

Monthly

+
+ {MONTHLY_RETENTION_PRESETS.map((months) => { + const active = backupRetention.monthlyMonths === months; + const label = months === 1 ? "1 month" : `${months} months`; + return ( + + ); + })} +