fix(api): improves the backup system to support per-database backup scheduling and retention

apps/api/src/api/backups/index.ts

@@ -1,44 +1,36 @@
 import { Elysia } from "elysia";
 import { BackupOrchestrator } from "../../backup/orchestrator";
-import type { BackupConfig, BackupTarget } from "../../backup/types";
+import { S3_BACKUP_PREFIX } from "../../backup/types";
+import type { BackupTarget, StorageConfig } from "../../backup/types";
 import { deleteBackupRecord, getBackupRecord, listBackupRecords } from "../../db/repo/backups";
 import { getDatabaseById } from "../../db/repo/databases";
 import { getBackupStorageSettings } from "../../db/repo/settings";
 import { created, fail, ok } from "../response";
 
-async function getBackupConfig(targetId?: string): Promise<BackupConfig> {
-	const storageSettings = await getBackupStorageSettings();
-	let retentionCount = parseInt(process.env.BACKUP_RETENTION || "7", 10);
-	let enabled = process.env.BACKUP_ENABLED === "true";
-	let scheduleCron = process.env.BACKUP_SCHEDULE || "0 */6 * * *";
+const DEFAULT_RETENTION = 7;
 
+async function loadStorageConfig(): Promise<StorageConfig> {
+	const settings = await getBackupStorageSettings();
+	if (settings.type === "s3") {
+		return {
+			type: "s3",
+			endpoint: settings.s3Endpoint,
+			accessKeyId: settings.s3AccessKeyId,
+			secretAccessKey: settings.s3SecretAccessKey,
+			bucket: settings.s3Bucket,
+			region: settings.s3Region,
+			prefix: S3_BACKUP_PREFIX,
+		};
+	}
+	return { type: "local", path: settings.path };
+}
+
+async function getRetentionCount(targetId?: string): Promise<number> {
 	if (targetId && targetId !== "internal") {
 		const db = await getDatabaseById(targetId);
-		if (db) {
-			retentionCount = db.backupRetention ?? retentionCount;
-			enabled = db.backupEnabled ?? enabled;
-			scheduleCron = db.backupSchedule ?? scheduleCron;
-		}
+		if (db) return db.backupRetention ?? DEFAULT_RETENTION;
 	}
-
-	return {
-		enabled,
-		scheduleCron,
-		retentionCount,
-		storage: {
-			type: storageSettings.type,
-			path: storageSettings.path || "/data/backups",
-			...(storageSettings.type === "s3"
-				? {
-						endpoint: storageSettings.s3Endpoint || "",
-						accessKeyId: storageSettings.s3AccessKeyId || "",
-						secretAccessKey: storageSettings.s3SecretAccessKey || "",
-						bucket: storageSettings.s3Bucket || "",
-						region: storageSettings.s3Region || "auto",
-					}
-				: {}),
-		},
-	};
+	return DEFAULT_RETENTION;
 }
 
 async function resolveTarget(targetId: string): Promise<BackupTarget> {
@@ -47,7 +39,7 @@ async function resolveTarget(targetId: string): Promise<BackupTarget> {
 			id: "internal",
 			type: "internal",
 			engine: "postgresql",
-			containerName: "postgres",
+			containerName: process.env.POSTGRES_CONTAINER || "dequel-postgres-1",
 			databaseName: process.env.POSTGRES_DB || "dequel",
 			serverId: null,
 			credentials: {
@@ -81,13 +73,14 @@ export const backupRoutes = new Elysia({ prefix: "/backups" })
 		return ok(records);
 	})
 	.get("/config", async () => {
-		return ok(await getBackupConfig());
+		const settings = await getBackupStorageSettings();
+		return ok(settings);
 	})
 	.post("/", async ({ set }) => {
 		try {
-			const config = await getBackupConfig("internal");
-			const orchestrator = new BackupOrchestrator(config);
-			const job = await orchestrator.backup("internal", resolveTarget);
+			const storage = await loadStorageConfig();
+			const orchestrator = new BackupOrchestrator(storage);
+			const job = await orchestrator.backup("internal", DEFAULT_RETENTION, resolveTarget);
 			return created(job);
 		} catch (error) {
 			set.status = 500;
@@ -96,9 +89,10 @@ export const backupRoutes = new Elysia({ prefix: "/backups" })
 	})
 	.post("/:id/trigger", async ({ params, set }) => {
 		try {
-			const config = await getBackupConfig(params.id);
-			const orchestrator = new BackupOrchestrator(config);
-			const job = await orchestrator.backup(params.id, resolveTarget);
+			const storage = await loadStorageConfig();
+			const retention = await getRetentionCount(params.id);
+			const orchestrator = new BackupOrchestrator(storage);
+			const job = await orchestrator.backup(params.id, retention, resolveTarget);
 			return created(job);
 		} catch (error) {
 			set.status = 500;
@@ -124,9 +118,8 @@ export const backupRoutes = new Elysia({ prefix: "/backups" })
 	})
 	.post("/:id/restore", async ({ params, set }) => {
 		try {
-			const record = await getBackupRecord(params.id);
-			const config = await getBackupConfig(record?.targetId);
-			const orchestrator = new BackupOrchestrator(config);
+			const storage = await loadStorageConfig();
+			const orchestrator = new BackupOrchestrator(storage);
 			await orchestrator.restore(params.id, resolveTarget);
 			return ok({ restored: true });
 		} catch (error) {

apps/api/src/backup/orchestrator.ts

@@ -1,6 +1,5 @@
 import { randomUUID } from "node:crypto";
-import { pipeline } from "node:stream/promises";
-import { createGunzip, createGzip } from "node:zlib";
+import { createGzip } from "node:zlib";
 import { getServerById } from "../db/repo";
 import {
 	createBackupRecord,
@@ -13,7 +12,7 @@ import type { BackupContext } from "./adapter";
 import { getAdapter } from "./adapters";
 import type { BackupStorage } from "./storage";
 import { createStorage } from "./storage/index";
-import type { BackupConfig, BackupJob, BackupTarget } from "./types";
+import type { BackupJob, BackupTarget, StorageConfig } from "./types";
 
 async function buildContext(target: BackupTarget): Promise<BackupContext> {
 	const server = target.serverId ? await getServerById(target.serverId) : null;
@@ -22,14 +21,18 @@ async function buildContext(target: BackupTarget): Promise<BackupContext> {
 
 export class BackupOrchestrator {
 	private storage: BackupStorage;
-	private config: BackupConfig;
+	private storageType: "local" | "s3";
 
-	constructor(config: BackupConfig) {
-		this.config = config;
-		this.storage = createStorage(config.storage);
+	constructor(storage: StorageConfig) {
+		this.storage = createStorage(storage);
+		this.storageType = storage.type;
 	}
 
-	async backup(targetId: string, resolveTarget: (id: string) => Promise<BackupTarget>): Promise<BackupJob> {
+	async backup(
+		targetId: string,
+		retentionCount: number,
+		resolveTarget: (id: string) => Promise<BackupTarget>,
+	): Promise<BackupJob> {
 		const target = await resolveTarget(targetId);
 		const ctx = await buildContext(target);
 		const adapter = getAdapter(target.engine);
@@ -44,16 +47,17 @@ export class BackupOrchestrator {
 
 			await this.updateJob(job.id, { status: "uploading" });
 			const filename = `${target.id}-${new Date().toISOString().replace(/[:.]/g, "-")}.sql.gz`;
-			const storagePath = await this.storage.upload(filename, compressed);
+			const { path: storagePath, size: sizeBytes } = await this.storage.upload(filename, compressed);
 
 			const completed = await this.updateJob(job.id, {
 				status: "completed",
 				filename,
 				storagePath,
+				sizeBytes,
 				completedAt: new Date(),
 			});
 
-			await this.enforceRetention(targetId);
+			await this.enforceRetention(targetId, retentionCount);
 
 			return completed;
 		} catch (error) {
@@ -69,13 +73,14 @@ export class BackupOrchestrator {
 		const adapter = getAdapter(target.engine);
 
 		const compressed = await this.storage.download(job.storagePath!);
+		const { createGunzip } = await import("node:zlib");
 		const dump = compressed.pipe(createGunzip());
 		await adapter.restore(ctx, dump);
 	}
 
-	private async enforceRetention(targetId: string): Promise<void> {
+	private async enforceRetention(targetId: string, retentionCount: number): Promise<void> {
 		const completed = await listCompletedBackupsForTarget(targetId);
-		const toDelete = completed.slice(this.config.retentionCount);
+		const toDelete = completed.slice(retentionCount);
 
 		for (const job of toDelete) {
 			if (job.storagePath) {
@@ -92,7 +97,7 @@ export class BackupOrchestrator {
 			targetType: target.type,
 			engine: target.engine,
 			filename: null,
-			storageType: this.config.storage.type,
+			storageType: this.storageType,
 			storagePath: null,
 			sizeBytes: null,
 			status: "pending",

apps/api/src/backup/scheduler.ts

@@ -1,25 +1,42 @@
 import { listAllDatabases } from "../db/repo/databases";
+import { getBackupStorageSettings } from "../db/repo/settings";
 import { BackupOrchestrator } from "./orchestrator";
-import type { BackupConfig, BackupTarget } from "./types";
+import { S3_BACKUP_PREFIX } from "./types";
+import type { BackupTarget, StorageConfig } from "./types";
 
-export function startBackupScheduler(config: BackupConfig): void {
-	if (!config.enabled) return;
+const lastFiredMinute = new Map<string, number>();
 
-	const orchestrator = new BackupOrchestrator(config);
-	const interval = cronToMs(config.scheduleCron);
+const DEFAULT_SCHEDULE = "0 */6 * * *";
+const DEFAULT_RETENTION = 7;
 
-	console.log(`[Backup] Scheduler started (interval: ${interval}ms)`);
+export function startBackupScheduler(): void {
+	console.log("[Backup] Scheduler started (checking every 60s)");
 
 	setInterval(async () => {
 		try {
-			const targets = await getAllBackupTargets();
+			const settings = await getBackupStorageSettings();
+			const storage = toStorageConfig(settings);
+			const orchestrator = new BackupOrchestrator(storage);
+			const targets = await getAllBackupTargets(settings.systemBackupSchedule, settings.systemBackupRetention);
+			const now = new Date();
+
 			for (const target of targets) {
 				try {
-					console.log(`[Backup] Backing up ${target.id}...`);
-					await orchestrator.backup(target.id, async (id) => {
+					const minuteKey = `${now.getFullYear()}-${now.getMonth()}-${now.getDate()}-${now.getHours()}-${now.getMinutes()}`;
+					const lastKey = lastFiredMinute.get(target.id);
+
+					if (lastKey === minuteKey) continue;
+
+					if (!matchesCron(target.scheduleCron, now)) continue;
+
+					console.log(`[Backup] Backing up ${target.id} (${target.engine})...`);
+					lastFiredMinute.set(target.id, minuteKey);
+
+					await orchestrator.backup(target.id, target.retentionCount, async (id) => {
 						if (id === "internal") return getInternalTarget();
 						return getManagedTarget(id);
 					});
+
 					console.log(`[Backup] Completed ${target.id}`);
 				} catch (error) {
 					console.error(`[Backup] Failed for ${target.id}:`, error);
@@ -28,28 +45,90 @@ export function startBackupScheduler(config: BackupConfig): void {
 		} catch (error) {
 			console.error("[Backup] Scheduler error:", error);
 		}
-	}, interval);
+	}, 60_000);
+}
+
+function toStorageConfig(settings: {
+	type: string;
+	path?: string;
+	s3Endpoint?: string;
+	s3AccessKeyId?: string;
+	s3SecretAccessKey?: string;
+	s3Bucket?: string;
+	s3Region?: string;
+}): StorageConfig {
+	if (settings.type === "s3") {
+		return {
+			type: "s3",
+			endpoint: settings.s3Endpoint || "",
+			accessKeyId: settings.s3AccessKeyId || "",
+			secretAccessKey: settings.s3SecretAccessKey || "",
+			bucket: settings.s3Bucket || "",
+			region: settings.s3Region || "auto",
+			prefix: S3_BACKUP_PREFIX,
+		};
+	}
+	return { type: "local", path: settings.path || "/data/backups" };
+}
+
+function matchesCron(cron: string, date: Date): boolean {
+	const parts = cron.trim().split(/\s+/);
+	if (parts.length !== 5) return false;
+
+	const [minExpr, hourExpr, dayExpr, monthExpr, dowExpr] = parts;
+
+	if (!matchField(minExpr, date.getMinutes())) return false;
+	if (!matchField(hourExpr, date.getHours())) return false;
+	if (!matchField(dayExpr, date.getDate())) return false;
+	if (!matchField(monthExpr, date.getMonth() + 1)) return false;
+	if (!matchField(dowExpr, date.getDay())) return false;
+
+	return true;
 }
 
-function cronToMs(cron: string): number {
-	const parts = cron.split(" ");
-	if (parts.length !== 5) return 3600000;
+function matchField(expr: string, value: number): boolean {
+	if (expr === "*") return true;
 
-	const [, hourPart] = parts;
-	if (hourPart.includes("*/")) {
-		const hours = parseInt(hourPart.replace("*/", ""), 10);
-		return hours * 3600000;
+	for (const part of expr.split(",")) {
+		if (part.includes("-")) {
+			const [start, end] = part.split("-").map(Number);
+			if (value >= start && value <= end) return true;
+		} else if (part.includes("/")) {
+			const [range, step] = part.split("/");
+			const stepNum = parseInt(step, 10);
+			if (range === "*") {
+				if (value % stepNum === 0) return true;
+			} else {
+				const start = parseInt(range, 10);
+				if (value >= start && value % stepNum === 0) return true;
+			}
+		} else {
+			if (parseInt(part, 10) === value) return true;
+		}
 	}
 
-	return 3600000;
+	return false;
 }
 
-async function getAllBackupTargets(): Promise<BackupTarget[]> {
-	const targets: BackupTarget[] = [getInternalTarget()];
+interface SchedulableTarget extends BackupTarget {
+	scheduleCron: string;
+	retentionCount: number;
+}
+
+async function getAllBackupTargets(systemSchedule: string, systemRetention: number): Promise<SchedulableTarget[]> {
+	const targets: SchedulableTarget[] = [
+		{
+			...getInternalTarget(),
+			scheduleCron: systemSchedule || DEFAULT_SCHEDULE,
+			retentionCount: systemRetention ?? DEFAULT_RETENTION,
+		},
+	];
 
 	const databases = await listAllDatabases();
 	for (const db of databases) {
 		if (db.status !== "running" || !db.containerName) continue;
+		if (!db.backupEnabled) continue;
+
 		targets.push({
 			id: db.id,
 			type: "managed",
@@ -61,6 +140,8 @@ async function getAllBackupTargets(): Promise<BackupTarget[]> {
 				username: db.username,
 				password: db.password,
 			},
+			scheduleCron: db.backupSchedule || DEFAULT_SCHEDULE,
+			retentionCount: db.backupRetention ?? DEFAULT_RETENTION,
 		});
 	}
 
@@ -72,7 +153,7 @@ function getInternalTarget(): BackupTarget {
 		id: "internal",
 		type: "internal",
 		engine: "postgresql",
-		containerName: "postgres",
+		containerName: process.env.POSTGRES_CONTAINER || "dequel-postgres-1",
 		databaseName: process.env.POSTGRES_DB || "dequel",
 		serverId: null,
 		credentials: {

apps/api/src/backup/storage.ts

@@ -1,7 +1,7 @@
 import type { Readable } from "node:stream";
 
 export interface BackupStorage {
-	upload(key: string, data: Readable): Promise<string>;
+	upload(key: string, data: Readable): Promise<{ path: string; size: number }>;
 
 	download(key: string): Promise<Readable>;