diff --git a/apps/api/src/api/backups/index.ts b/apps/api/src/api/backups/index.ts index a2082d5..5bee6d6 100644 --- a/apps/api/src/api/backups/index.ts +++ b/apps/api/src/api/backups/index.ts @@ -1,44 +1,36 @@ import { Elysia } from "elysia"; import { BackupOrchestrator } from "../../backup/orchestrator"; -import type { BackupConfig, BackupTarget } from "../../backup/types"; +import { S3_BACKUP_PREFIX } from "../../backup/types"; +import type { BackupTarget, StorageConfig } from "../../backup/types"; import { deleteBackupRecord, getBackupRecord, listBackupRecords } from "../../db/repo/backups"; import { getDatabaseById } from "../../db/repo/databases"; import { getBackupStorageSettings } from "../../db/repo/settings"; import { created, fail, ok } from "../response"; -async function getBackupConfig(targetId?: string): Promise { - const storageSettings = await getBackupStorageSettings(); - let retentionCount = parseInt(process.env.BACKUP_RETENTION || "7", 10); - let enabled = process.env.BACKUP_ENABLED === "true"; - let scheduleCron = process.env.BACKUP_SCHEDULE || "0 */6 * * *"; +const DEFAULT_RETENTION = 7; +async function loadStorageConfig(): Promise { + const settings = await getBackupStorageSettings(); + if (settings.type === "s3") { + return { + type: "s3", + endpoint: settings.s3Endpoint, + accessKeyId: settings.s3AccessKeyId, + secretAccessKey: settings.s3SecretAccessKey, + bucket: settings.s3Bucket, + region: settings.s3Region, + prefix: S3_BACKUP_PREFIX, + }; + } + return { type: "local", path: settings.path }; +} + +async function getRetentionCount(targetId?: string): Promise { if (targetId && targetId !== "internal") { const db = await getDatabaseById(targetId); - if (db) { - retentionCount = db.backupRetention ?? retentionCount; - enabled = db.backupEnabled ?? enabled; - scheduleCron = db.backupSchedule ?? scheduleCron; - } + if (db) return db.backupRetention ?? DEFAULT_RETENTION; } - - return { - enabled, - scheduleCron, - retentionCount, - storage: { - type: storageSettings.type, - path: storageSettings.path || "/data/backups", - ...(storageSettings.type === "s3" - ? { - endpoint: storageSettings.s3Endpoint || "", - accessKeyId: storageSettings.s3AccessKeyId || "", - secretAccessKey: storageSettings.s3SecretAccessKey || "", - bucket: storageSettings.s3Bucket || "", - region: storageSettings.s3Region || "auto", - } - : {}), - }, - }; + return DEFAULT_RETENTION; } async function resolveTarget(targetId: string): Promise { @@ -47,7 +39,7 @@ async function resolveTarget(targetId: string): Promise { id: "internal", type: "internal", engine: "postgresql", - containerName: "postgres", + containerName: process.env.POSTGRES_CONTAINER || "dequel-postgres-1", databaseName: process.env.POSTGRES_DB || "dequel", serverId: null, credentials: { @@ -81,13 +73,14 @@ export const backupRoutes = new Elysia({ prefix: "/backups" }) return ok(records); }) .get("/config", async () => { - return ok(await getBackupConfig()); + const settings = await getBackupStorageSettings(); + return ok(settings); }) .post("/", async ({ set }) => { try { - const config = await getBackupConfig("internal"); - const orchestrator = new BackupOrchestrator(config); - const job = await orchestrator.backup("internal", resolveTarget); + const storage = await loadStorageConfig(); + const orchestrator = new BackupOrchestrator(storage); + const job = await orchestrator.backup("internal", DEFAULT_RETENTION, resolveTarget); return created(job); } catch (error) { set.status = 500; @@ -96,9 +89,10 @@ export const backupRoutes = new Elysia({ prefix: "/backups" }) }) .post("/:id/trigger", async ({ params, set }) => { try { - const config = await getBackupConfig(params.id); - const orchestrator = new BackupOrchestrator(config); - const job = await orchestrator.backup(params.id, resolveTarget); + const storage = await loadStorageConfig(); + const retention = await getRetentionCount(params.id); + const orchestrator = new BackupOrchestrator(storage); + const job = await orchestrator.backup(params.id, retention, resolveTarget); return created(job); } catch (error) { set.status = 500; @@ -124,9 +118,8 @@ export const backupRoutes = new Elysia({ prefix: "/backups" }) }) .post("/:id/restore", async ({ params, set }) => { try { - const record = await getBackupRecord(params.id); - const config = await getBackupConfig(record?.targetId); - const orchestrator = new BackupOrchestrator(config); + const storage = await loadStorageConfig(); + const orchestrator = new BackupOrchestrator(storage); await orchestrator.restore(params.id, resolveTarget); return ok({ restored: true }); } catch (error) { diff --git a/apps/api/src/backup/orchestrator.ts b/apps/api/src/backup/orchestrator.ts index e663211..3d2b1e3 100644 --- a/apps/api/src/backup/orchestrator.ts +++ b/apps/api/src/backup/orchestrator.ts @@ -1,6 +1,5 @@ import { randomUUID } from "node:crypto"; -import { pipeline } from "node:stream/promises"; -import { createGunzip, createGzip } from "node:zlib"; +import { createGzip } from "node:zlib"; import { getServerById } from "../db/repo"; import { createBackupRecord, @@ -13,7 +12,7 @@ import type { BackupContext } from "./adapter"; import { getAdapter } from "./adapters"; import type { BackupStorage } from "./storage"; import { createStorage } from "./storage/index"; -import type { BackupConfig, BackupJob, BackupTarget } from "./types"; +import type { BackupJob, BackupTarget, StorageConfig } from "./types"; async function buildContext(target: BackupTarget): Promise { const server = target.serverId ? await getServerById(target.serverId) : null; @@ -22,14 +21,18 @@ async function buildContext(target: BackupTarget): Promise { export class BackupOrchestrator { private storage: BackupStorage; - private config: BackupConfig; + private storageType: "local" | "s3"; - constructor(config: BackupConfig) { - this.config = config; - this.storage = createStorage(config.storage); + constructor(storage: StorageConfig) { + this.storage = createStorage(storage); + this.storageType = storage.type; } - async backup(targetId: string, resolveTarget: (id: string) => Promise): Promise { + async backup( + targetId: string, + retentionCount: number, + resolveTarget: (id: string) => Promise, + ): Promise { const target = await resolveTarget(targetId); const ctx = await buildContext(target); const adapter = getAdapter(target.engine); @@ -44,16 +47,17 @@ export class BackupOrchestrator { await this.updateJob(job.id, { status: "uploading" }); const filename = `${target.id}-${new Date().toISOString().replace(/[:.]/g, "-")}.sql.gz`; - const storagePath = await this.storage.upload(filename, compressed); + const { path: storagePath, size: sizeBytes } = await this.storage.upload(filename, compressed); const completed = await this.updateJob(job.id, { status: "completed", filename, storagePath, + sizeBytes, completedAt: new Date(), }); - await this.enforceRetention(targetId); + await this.enforceRetention(targetId, retentionCount); return completed; } catch (error) { @@ -69,13 +73,14 @@ export class BackupOrchestrator { const adapter = getAdapter(target.engine); const compressed = await this.storage.download(job.storagePath!); + const { createGunzip } = await import("node:zlib"); const dump = compressed.pipe(createGunzip()); await adapter.restore(ctx, dump); } - private async enforceRetention(targetId: string): Promise { + private async enforceRetention(targetId: string, retentionCount: number): Promise { const completed = await listCompletedBackupsForTarget(targetId); - const toDelete = completed.slice(this.config.retentionCount); + const toDelete = completed.slice(retentionCount); for (const job of toDelete) { if (job.storagePath) { @@ -92,7 +97,7 @@ export class BackupOrchestrator { targetType: target.type, engine: target.engine, filename: null, - storageType: this.config.storage.type, + storageType: this.storageType, storagePath: null, sizeBytes: null, status: "pending", diff --git a/apps/api/src/backup/scheduler.ts b/apps/api/src/backup/scheduler.ts index 2dbad0b..4b7f340 100644 --- a/apps/api/src/backup/scheduler.ts +++ b/apps/api/src/backup/scheduler.ts @@ -1,25 +1,42 @@ import { listAllDatabases } from "../db/repo/databases"; +import { getBackupStorageSettings } from "../db/repo/settings"; import { BackupOrchestrator } from "./orchestrator"; -import type { BackupConfig, BackupTarget } from "./types"; +import { S3_BACKUP_PREFIX } from "./types"; +import type { BackupTarget, StorageConfig } from "./types"; -export function startBackupScheduler(config: BackupConfig): void { - if (!config.enabled) return; +const lastFiredMinute = new Map(); - const orchestrator = new BackupOrchestrator(config); - const interval = cronToMs(config.scheduleCron); +const DEFAULT_SCHEDULE = "0 */6 * * *"; +const DEFAULT_RETENTION = 7; - console.log(`[Backup] Scheduler started (interval: ${interval}ms)`); +export function startBackupScheduler(): void { + console.log("[Backup] Scheduler started (checking every 60s)"); setInterval(async () => { try { - const targets = await getAllBackupTargets(); + const settings = await getBackupStorageSettings(); + const storage = toStorageConfig(settings); + const orchestrator = new BackupOrchestrator(storage); + const targets = await getAllBackupTargets(settings.systemBackupSchedule, settings.systemBackupRetention); + const now = new Date(); + for (const target of targets) { try { - console.log(`[Backup] Backing up ${target.id}...`); - await orchestrator.backup(target.id, async (id) => { + const minuteKey = `${now.getFullYear()}-${now.getMonth()}-${now.getDate()}-${now.getHours()}-${now.getMinutes()}`; + const lastKey = lastFiredMinute.get(target.id); + + if (lastKey === minuteKey) continue; + + if (!matchesCron(target.scheduleCron, now)) continue; + + console.log(`[Backup] Backing up ${target.id} (${target.engine})...`); + lastFiredMinute.set(target.id, minuteKey); + + await orchestrator.backup(target.id, target.retentionCount, async (id) => { if (id === "internal") return getInternalTarget(); return getManagedTarget(id); }); + console.log(`[Backup] Completed ${target.id}`); } catch (error) { console.error(`[Backup] Failed for ${target.id}:`, error); @@ -28,28 +45,90 @@ export function startBackupScheduler(config: BackupConfig): void { } catch (error) { console.error("[Backup] Scheduler error:", error); } - }, interval); + }, 60_000); +} + +function toStorageConfig(settings: { + type: string; + path?: string; + s3Endpoint?: string; + s3AccessKeyId?: string; + s3SecretAccessKey?: string; + s3Bucket?: string; + s3Region?: string; +}): StorageConfig { + if (settings.type === "s3") { + return { + type: "s3", + endpoint: settings.s3Endpoint || "", + accessKeyId: settings.s3AccessKeyId || "", + secretAccessKey: settings.s3SecretAccessKey || "", + bucket: settings.s3Bucket || "", + region: settings.s3Region || "auto", + prefix: S3_BACKUP_PREFIX, + }; + } + return { type: "local", path: settings.path || "/data/backups" }; +} + +function matchesCron(cron: string, date: Date): boolean { + const parts = cron.trim().split(/\s+/); + if (parts.length !== 5) return false; + + const [minExpr, hourExpr, dayExpr, monthExpr, dowExpr] = parts; + + if (!matchField(minExpr, date.getMinutes())) return false; + if (!matchField(hourExpr, date.getHours())) return false; + if (!matchField(dayExpr, date.getDate())) return false; + if (!matchField(monthExpr, date.getMonth() + 1)) return false; + if (!matchField(dowExpr, date.getDay())) return false; + + return true; } -function cronToMs(cron: string): number { - const parts = cron.split(" "); - if (parts.length !== 5) return 3600000; +function matchField(expr: string, value: number): boolean { + if (expr === "*") return true; - const [, hourPart] = parts; - if (hourPart.includes("*/")) { - const hours = parseInt(hourPart.replace("*/", ""), 10); - return hours * 3600000; + for (const part of expr.split(",")) { + if (part.includes("-")) { + const [start, end] = part.split("-").map(Number); + if (value >= start && value <= end) return true; + } else if (part.includes("/")) { + const [range, step] = part.split("/"); + const stepNum = parseInt(step, 10); + if (range === "*") { + if (value % stepNum === 0) return true; + } else { + const start = parseInt(range, 10); + if (value >= start && value % stepNum === 0) return true; + } + } else { + if (parseInt(part, 10) === value) return true; + } } - return 3600000; + return false; } -async function getAllBackupTargets(): Promise { - const targets: BackupTarget[] = [getInternalTarget()]; +interface SchedulableTarget extends BackupTarget { + scheduleCron: string; + retentionCount: number; +} + +async function getAllBackupTargets(systemSchedule: string, systemRetention: number): Promise { + const targets: SchedulableTarget[] = [ + { + ...getInternalTarget(), + scheduleCron: systemSchedule || DEFAULT_SCHEDULE, + retentionCount: systemRetention ?? DEFAULT_RETENTION, + }, + ]; const databases = await listAllDatabases(); for (const db of databases) { if (db.status !== "running" || !db.containerName) continue; + if (!db.backupEnabled) continue; + targets.push({ id: db.id, type: "managed", @@ -61,6 +140,8 @@ async function getAllBackupTargets(): Promise { username: db.username, password: db.password, }, + scheduleCron: db.backupSchedule || DEFAULT_SCHEDULE, + retentionCount: db.backupRetention ?? DEFAULT_RETENTION, }); } @@ -72,7 +153,7 @@ function getInternalTarget(): BackupTarget { id: "internal", type: "internal", engine: "postgresql", - containerName: "postgres", + containerName: process.env.POSTGRES_CONTAINER || "dequel-postgres-1", databaseName: process.env.POSTGRES_DB || "dequel", serverId: null, credentials: { diff --git a/apps/api/src/backup/storage.ts b/apps/api/src/backup/storage.ts index f61b0ee..0b09e66 100644 --- a/apps/api/src/backup/storage.ts +++ b/apps/api/src/backup/storage.ts @@ -1,7 +1,7 @@ import type { Readable } from "node:stream"; export interface BackupStorage { - upload(key: string, data: Readable): Promise; + upload(key: string, data: Readable): Promise<{ path: string; size: number }>; download(key: string): Promise; diff --git a/apps/api/src/backup/storage/local.ts b/apps/api/src/backup/storage/local.ts index cfd8568..a84ddad 100644 --- a/apps/api/src/backup/storage/local.ts +++ b/apps/api/src/backup/storage/local.ts @@ -13,12 +13,13 @@ export class LocalStorage implements BackupStorage { this.basePath = basePath; } - async upload(key: string, data: Readable): Promise { + async upload(key: string, data: Readable): Promise<{ path: string; size: number }> { await mkdir(this.basePath, { recursive: true }); const filePath = join(this.basePath, key); const writeStream = createWriteStream(filePath); await pipeline(data, writeStream); - return filePath; + const info = await stat(filePath); + return { path: filePath, size: info.size }; } async download(key: string): Promise { diff --git a/apps/api/src/backup/storage/s3.ts b/apps/api/src/backup/storage/s3.ts index af15784..1a036e8 100644 --- a/apps/api/src/backup/storage/s3.ts +++ b/apps/api/src/backup/storage/s3.ts @@ -13,35 +13,43 @@ export class S3Storage implements BackupStorage { readonly type = "s3" as const; private client: S3Client; private bucket: string; + private prefix: string; constructor(config: S3StorageConfig) { this.client = new S3Client({ endpoint: config.endpoint, region: config.region, + forcePathStyle: true, credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey, }, }); this.bucket = config.bucket; + this.prefix = config.prefix ? config.prefix.replace(/\/+$/, "") + "/" : ""; } - async upload(key: string, data: Readable): Promise { + private keyPath(key: string): string { + return this.prefix + key; + } + + async upload(key: string, data: Readable): Promise<{ path: string; size: number }> { const chunks: Buffer[] = []; for await (const chunk of data) { chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); } const body = Buffer.concat(chunks); + const fullKey = this.keyPath(key); await this.client.send( new PutObjectCommand({ Bucket: this.bucket, - Key: key, + Key: fullKey, Body: body, }), ); - return key; + return { path: fullKey, size: body.length }; } async download(key: string): Promise { @@ -59,6 +67,7 @@ export class S3Storage implements BackupStorage { const response = await this.client.send( new ListObjectsV2Command({ Bucket: this.bucket, + Prefix: this.prefix || undefined, }), ); diff --git a/apps/api/src/backup/types.ts b/apps/api/src/backup/types.ts index 6bbac63..65d824e 100644 --- a/apps/api/src/backup/types.ts +++ b/apps/api/src/backup/types.ts @@ -1,3 +1,5 @@ +export const S3_BACKUP_PREFIX = "dequel-db-backup"; + export type DatabaseType = "postgresql" | "mysql" | "redis" | "mongodb"; export interface BackupTarget { @@ -30,13 +32,6 @@ export interface BackupJob { completedAt: Date | null; } -export interface BackupConfig { - enabled: boolean; - scheduleCron: string; - retentionCount: number; - storage: StorageConfig; -} - export type StorageConfig = LocalStorageConfig | S3StorageConfig; export interface LocalStorageConfig { @@ -51,4 +46,5 @@ export interface S3StorageConfig { secretAccessKey: string; bucket: string; region: string; + prefix: string; } diff --git a/apps/api/src/db/migrations/0033_add_system_backup_settings.sql b/apps/api/src/db/migrations/0033_add_system_backup_settings.sql new file mode 100644 index 0000000..ce89205 --- /dev/null +++ b/apps/api/src/db/migrations/0033_add_system_backup_settings.sql @@ -0,0 +1,2 @@ +ALTER TABLE backup_storage_settings ADD COLUMN system_backup_schedule text DEFAULT '0 */6 * * *'; +ALTER TABLE backup_storage_settings ADD COLUMN system_backup_retention integer DEFAULT 7; diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index eba63ef..6497c94 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -64,6 +64,13 @@ "when": 1788500000000, "tag": "0032_add_backup_storage_and_db_schedule", "breakpoints": true + }, + { + "idx": 9, + "version": "7", + "when": 1788600000000, + "tag": "0033_add_system_backup_settings", + "breakpoints": true } ] } diff --git a/apps/api/src/db/repo/settings.ts b/apps/api/src/db/repo/settings.ts index 05ff59c..2673ef6 100644 --- a/apps/api/src/db/repo/settings.ts +++ b/apps/api/src/db/repo/settings.ts @@ -84,13 +84,15 @@ export const getBackupStorageSettings = async (): Promise { startReconciliation(); startStaleAgentCleanup(); startAbandonedJobCleanup(); - startBackupScheduler({ - enabled: process.env.BACKUP_ENABLED === "true", - scheduleCron: process.env.BACKUP_SCHEDULE || "0 */6 * * *", - retentionCount: parseInt(process.env.BACKUP_RETENTION || "7", 10), - storage: { - type: (process.env.BACKUP_STORAGE_TYPE as "local" | "s3") || "local", - path: process.env.BACKUP_STORAGE_PATH || "/data/backups", - ...(process.env.BACKUP_STORAGE_TYPE === "s3" - ? { - endpoint: process.env.BACKUP_S3_ENDPOINT || "", - accessKeyId: process.env.BACKUP_S3_ACCESS_KEY_ID || "", - secretAccessKey: process.env.BACKUP_S3_SECRET_ACCESS_KEY || "", - bucket: process.env.BACKUP_S3_BUCKET || "", - region: process.env.BACKUP_S3_REGION || "auto", - } - : {}), - }, - }); + startBackupScheduler(); setInterval(() => { cleanupExpiredTokens().catch(() => {}); }, 60_000); diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts index 38236b8..4f1ef2b 100644 --- a/apps/api/src/types.ts +++ b/apps/api/src/types.ts @@ -167,6 +167,8 @@ export interface BackupStorageSettingsData { s3SecretAccessKey?: string; s3Bucket?: string; s3Region?: string; + systemBackupSchedule?: string; + systemBackupRetention?: number; } export interface Domain { diff --git a/apps/web/src/components/databases/DatabaseBackupManager.tsx b/apps/web/src/components/databases/DatabaseBackupManager.tsx index 9a47503..624c05a 100644 --- a/apps/web/src/components/databases/DatabaseBackupManager.tsx +++ b/apps/web/src/components/databases/DatabaseBackupManager.tsx @@ -7,6 +7,7 @@ import { Badge } from "../ui/badge"; import { Button } from "../ui/button"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "../ui/card"; import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "../ui/dialog"; +import { Pagination } from "../ui/pagination"; interface DatabaseBackupManagerProps { database: Database; @@ -20,6 +21,8 @@ export function DatabaseBackupManager({ database }: DatabaseBackupManagerProps) const [deletingBackup, setDeletingBackup] = useState(null); const [isDeleting, setIsDeleting] = useState(false); const [error, setError] = useState(null); + const [page, setPage] = useState(1); + const pageSize = 5; const { data: allBackups = [], refetch } = useQuery({ queryKey: ["backups"], @@ -28,6 +31,7 @@ export function DatabaseBackupManager({ database }: DatabaseBackupManagerProps) }); const instanceBackups = allBackups.filter((b) => b.targetId === database.id); + const paginatedBackups = instanceBackups.slice((page - 1) * pageSize, page * pageSize); const handleTriggerBackup = async () => { setIsTriggering(true); @@ -127,7 +131,7 @@ export function DatabaseBackupManager({ database }: DatabaseBackupManagerProps) ) : (
- {instanceBackups.map((job) => ( + {paginatedBackups.map((job) => (
@@ -176,6 +180,9 @@ export function DatabaseBackupManager({ database }: DatabaseBackupManagerProps) ))}
)} + {instanceBackups.length > 0 && ( + + )} diff --git a/apps/web/src/components/databases/QueryEditor.tsx b/apps/web/src/components/databases/QueryEditor.tsx index ce1882a..7df178b 100644 --- a/apps/web/src/components/databases/QueryEditor.tsx +++ b/apps/web/src/components/databases/QueryEditor.tsx @@ -177,8 +177,8 @@ export function QueryEditor({ database }: QueryEditorProps) { value={query} onChange={(e) => setQuery(e.target.value)} placeholder="Enter SQL query or command..." - rows={4} - className="w-full bg-black/40 border border-border/60 rounded-2xl p-4 font-mono text-xs text-foreground focus:outline-none focus:ring-1 focus:ring-orange-500/50 resize-y" + rows={12} + className="w-full bg-black/50 border border-border/60 rounded-2xl p-4 font-mono text-xs text-foreground focus:outline-none focus:ring-1 focus:ring-orange-500/50 min-h-[260px] resize-y" />
diff --git a/apps/web/src/components/databases/studio/SqlQueryView.tsx b/apps/web/src/components/databases/studio/SqlQueryView.tsx index a14f3e8..e1af95c 100644 --- a/apps/web/src/components/databases/studio/SqlQueryView.tsx +++ b/apps/web/src/components/databases/studio/SqlQueryView.tsx @@ -40,8 +40,9 @@ export function SqlQueryView({