Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ ERPNext is not required.
one place, restricted to System Managers.
- **Retention:** local backup sets and off-site snapshots are pruned automatically; the
repository is checked weekly.
- **Alerts:** failure emails, optional success emails and Uptime Kuma push monitors.
- **Alerts:** every System Manager is told about failures in the notification bell and by
email; optional success emails and Uptime Kuma push monitors.
- **Safe deployments and restores for containers:** back up before every migration,
return to that backup if the migration fails, and restore a site from a snapshot into
new or existing volumes. See [Deployment](docs/deployment.md).
Expand Down Expand Up @@ -67,7 +68,8 @@ Then open **Restic Backup Settings**:
- **Enable Scheduled Backups** and add schedule rows.
- **Local Backup Sets** (default 4) are kept on the server after a successful upload.
- **Remote Retention (Days)** (default 14) keeps snapshots in that window, and always the newest.
- **Notification Recipients** receive failures; enable **Email on Success** to also get successes.
- **Email on Success** also emails System Managers about successful runs. They are always
told about failures, in the notification bell and by email.

Leave Frappe's own backup encryption disabled: Restic encrypts the off-site copy, and
encrypted Frappe archives cannot be validated before upload.
Expand Down
5 changes: 3 additions & 2 deletions docs/deployment.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,9 @@ absolute path; it runs before the migration with `RESTIC_RECOVERY_FINALIZING=1`.
## Alerts while services are stopped

Deployment upload failures and failed or rolled-back restores happen while the site is
down, so they are recorded on the sites volume. The running site emails them within
15 minutes and logs restore failures to Error Log. A site that stays in maintenance or
down, so they are recorded on the sites volume. The running site emails them and shows
them in every System Manager's notification bell within 15 minutes, and logs restore
failures to Error Log. A site that stays in maintenance or
blocked cannot send anything: watch the job's exit status and the site from outside.

## Custom orchestration
Expand Down
39 changes: 34 additions & 5 deletions frappe_restic/restic_backup/backup_control.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,16 @@

import frappe
from frappe import _
from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification
from frappe.utils import cint, get_time, now_datetime
from frappe.utils.background_jobs import enqueue
from frappe.utils.backups import BackupGenerator, get_backup_path
from frappe.utils.user import get_users_with_role
from redis.exceptions import LockError

from frappe_restic.config import namespace
from frappe_restic.restic_backup.doctype.restic_backup_settings.restic_backup_settings import (
parse_backup_schedule,
parse_recipients,
)
from frappe_restic.restic_backup.recovery import image_commit, read_json, snapshot_manifest, write_json
from frappe_restic.restic_backup.restic_transport import (
Expand Down Expand Up @@ -464,14 +465,31 @@ def _notify_run(run_name: str, *, succeeded: bool) -> None:
f"Snapshot: {run.restic_snapshot_id or 'none'}<br>"
f"Details: {run.error_summary or 'none'}",
succeeded=succeeded,
run_name=run.name,
)
_set_run(run_name, notification_status=status)


def _send_alert(subject: str, message: str, *, succeeded: bool = False) -> str:
settings = frappe.get_single("Restic Backup Settings")
recipients = parse_recipients(settings.notification_recipients)
if not recipients or (succeeded and not cint(settings.email_on_success)):
def _send_alert(subject: str, message: str, *, succeeded: bool = False, run_name: str | None = None) -> str:
"""Tell every System Manager: failures in the desk and by email, successes by email on request."""
if succeeded and not _settings_int("email_on_success", 0):
return "Not requested"
try:
managers = frappe.get_all(
"User",
filters={"name": ["in", get_users_with_role(SYSTEM_MANAGER_ROLE)]},
fields=["name", "email"],
)
except Exception: # notification failure must not replace the operation outcome
frappe.logger("restic_backup").exception("Could not find the System Managers to notify")
return "Failed"
if not succeeded:
_notify_desk([manager.email for manager in managers], subject, message, run_name)
# Administrator's address is a placeholder unless someone signs in with it as their own user.
recipients = [
manager.email for manager in managers if manager.email and manager.name not in frappe.STANDARD_USERS
]
if not recipients:
return "Not requested"
try:
frappe.sendmail(recipients=recipients, subject=subject, message=message, delayed=False)
Expand All @@ -481,6 +499,17 @@ def _send_alert(subject: str, message: str, *, succeeded: bool = False) -> str:
return "Sent"


def _notify_desk(recipients: list[str], subject: str, message: str, run_name: str | None) -> None:
"""Show a failure in the notification bell. An Alert never sends an email of its own."""
notification = {"type": "Alert", "subject": subject, "email_content": message}
if run_name:
notification.update(document_type="Restic Backup Run", document_name=run_name)
try:
enqueue_create_notification(recipients, notification)
except Exception: # notification failure must not replace the operation outcome
frappe.logger("restic_backup").exception("Could not create backup notification")


def _push_heartbeat(*, succeeded: bool, message: str) -> None:
url = os.environ.get("RESTIC_BACKUP_UPTIME_KUMA_URL", "").strip()
if not url:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@
"retention_column",
"remote_keep_days",
"notification_section",
"notification_recipients",
"notification_column",
"email_on_success"
],
"fields": [
Expand Down Expand Up @@ -66,26 +64,17 @@
"fieldtype": "Section Break",
"label": "Notifications"
},
{
"description": "Comma-separated email addresses. Failures are always sent; successes are optional.",
"fieldname": "notification_recipients",
"fieldtype": "Data",
"label": "Notification Recipients"
},
{
"fieldname": "notification_column",
"fieldtype": "Column Break"
},
{
"default": "0",
"description": "System Managers are always told about failures, in the notification bell and by email.",
"fieldname": "email_on_success",
"fieldtype": "Check",
"label": "Email on Success"
}
],
"issingle": 1,
"links": [],
"modified": "2026-09-16 15:50:00.000000",
"modified": "2026-10-07 19:00:00.000000",
"modified_by": "Administrator",
"module": "Restic Backup",
"name": "Restic Backup Settings",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@

from __future__ import annotations

import re

import frappe
from frappe import _
from frappe.model.document import Document
Expand All @@ -18,7 +16,6 @@
"Saturday",
"Sunday",
)
EMAIL_PATTERN = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")


class ResticBackupSettings(Document):
Expand Down Expand Up @@ -52,10 +49,6 @@ def validate(self) -> None:
if not 1 <= cint(self.remote_keep_days) <= 3650:
frappe.throw(_("Remote Retention must be between 1 and 3650 days."))

for email in parse_recipients(self.notification_recipients):
if not EMAIL_PATTERN.match(email):
frappe.throw(_("Invalid notification email address: {0}").format(email))

frappe.db.set_single_value("System Settings", "backup_limit", cint(self.local_backup_limit))


Expand Down Expand Up @@ -92,8 +85,3 @@ def parse_backup_schedule(rows) -> list[tuple[list[str], str]]:
if not schedule:
frappe.throw(_("At least one backup schedule row is required."))
return sorted(schedule, key=lambda row: row[1])


def parse_recipients(value: str | None) -> list[str]:
"""Parse a comma/newline-separated recipient list."""
return list(dict.fromkeys(part.strip() for part in re.split(r"[,\n]", value or "") if part.strip()))
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ def test_system_manager_can_save_settings_from_backup_control(self) -> None:
],
"local_backup_limit": 5,
"remote_keep_days": 21,
"notification_recipients": "ops@example.com",
"email_on_success": 1,
}

Expand All @@ -124,7 +123,6 @@ def test_system_manager_can_save_settings_from_backup_control(self) -> None:
)
self.assertEqual(result["local_backup_limit"], 5)
self.assertEqual(result["remote_keep_days"], 21)
self.assertEqual(result["notification_recipients"], "ops@example.com")
self.assertEqual(result["email_on_success"], 1)

def test_schedule_rejects_duplicate_day_and_time(self) -> None:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -174,20 +174,14 @@ class ResticBackupControl {
description: __("The newest off-site snapshot is always retained."),
},
{ fieldtype: "Section Break", label: __("Email notifications") },
{
fieldtype: "Data",
fieldname: "notification_recipients",
label: __("Recipients"),
default: settings.notification_recipients,
description: __("Separate multiple email addresses with commas."),
},
{ fieldtype: "Column Break" },
{
fieldtype: "Check",
fieldname: "email_on_success",
label: __("Email successful runs"),
default: settings.email_on_success,
description: __("Failure notifications are always sent."),
description: __(
"System Managers are always told about failures, in the notification bell and by email."
),
},
],
primary_action_label: __("Save"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
"backup_schedule",
"local_backup_limit",
"remote_keep_days",
"notification_recipients",
"email_on_success",
}

Expand Down Expand Up @@ -210,7 +209,6 @@ def _settings_payload(settings) -> dict[str, Any]:
],
"local_backup_limit": cint(settings.local_backup_limit) or 4,
"remote_keep_days": cint(settings.remote_keep_days) or 14,
"notification_recipients": settings.notification_recipients or "",
"email_on_success": cint(settings.email_on_success),
}

Expand Down
55 changes: 55 additions & 0 deletions frappe_restic/restic_backup/test_backup_control.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,61 @@ def test_failures_recorded_while_stopped_are_alerted_once(self) -> None:
self.assertEqual(read_json(receipts / "rolled-back.json")["notification_status"], "Sent")
self.assertNotIn("notification_status", read_json(receipts / "completed.json"))

def _alert(self, *, succeeded: bool, email_on_success: int = 0, desk=None, email=None):
managers = [
frappe._dict(name="Administrator", email="admin@example.com"),
frappe._dict(name="ops@example.com", email="ops@example.com"),
]
with (
patch.object(control, "_settings_int", return_value=email_on_success),
patch.object(control, "get_users_with_role", return_value=["Administrator", "ops@example.com"]),
patch("frappe.get_all", return_value=managers),
patch("frappe.logger"),
patch.object(control, "enqueue_create_notification", side_effect=desk) as notify,
patch("frappe.sendmail", side_effect=email) as sendmail,
):
status = control._send_alert(
"FAILED: backup", "disk full", succeeded=succeeded, run_name="run123"
)
return status, notify, sendmail

def test_failure_reaches_every_system_manager_in_the_desk_and_by_email(self) -> None:
status, notify, sendmail = self._alert(succeeded=False)
self.assertEqual(status, "Sent")
recipients, notification = notify.call_args.args
self.assertEqual(recipients, ["admin@example.com", "ops@example.com"])
self.assertEqual(
notification,
{
"type": "Alert",
"subject": "FAILED: backup",
"email_content": "disk full",
"document_type": "Restic Backup Run",
"document_name": "run123",
},
)
self.assertEqual(sendmail.call_args.kwargs["recipients"], ["ops@example.com"])

def test_success_is_emailed_only_on_request_and_never_shown_in_the_desk(self) -> None:
status, notify, sendmail = self._alert(succeeded=True)
self.assertEqual(status, "Not requested")
notify.assert_not_called()
sendmail.assert_not_called()

status, notify, sendmail = self._alert(succeeded=True, email_on_success=1)
self.assertEqual(status, "Sent")
notify.assert_not_called()
self.assertEqual(sendmail.call_args.kwargs["recipients"], ["ops@example.com"])

def test_one_failing_channel_does_not_block_the_other(self) -> None:
status, _notify, sendmail = self._alert(succeeded=False, desk=RuntimeError("queue down"))
self.assertEqual(status, "Sent")
sendmail.assert_called_once()

status, notify, _sendmail = self._alert(succeeded=False, email=RuntimeError("no mailbox"))
self.assertEqual(status, "Failed")
notify.assert_called_once()

def test_schedule_checkbox_does_not_disable_retention_for_manual_remote_backups(self) -> None:
with (
patch.dict(os.environ, {"RESTIC_OFFSITE_BACKUP_ENABLED": "1"}),
Expand Down
Loading