From b5f6e58de3131b17837c0c8b673dd512a36cb829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Kocsis?= Date: Mon, 13 Apr 2026 19:34:37 +0200 Subject: [PATCH 01/11] pyproject.toml: Removed an unnecessary project classifier --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5aee45c..fee8e3d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,6 @@ keywords = [ "automation"] classifiers = [ "Programming Language :: Python :: 3.14", - "Private :: Do Not Upload" ] [project.optional-dependencies] From 48f3fbe66fc19bf986673fdf4e79fa2b7877049f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Kocsis?= Date: Mon, 13 Apr 2026 19:34:48 +0200 Subject: [PATCH 02/11] pyproject.toml: Minor formatting fix --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fee8e3d..8eb442b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,8 @@ dynamic = ["version"] keywords = [ "restic", "backup", - "automation"] + "automation" +] classifiers = [ "Programming Language :: Python :: 3.14", ] From d3c3eba32b041a82f01b0014cb97fe67a2ace214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Kocsis?= Date: Mon, 13 Apr 2026 19:35:27 +0200 Subject: [PATCH 03/11] README: Minor formatting fix --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 80c28a8..18864ea 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ These represent their matching restic commands: [restic backup][restic-docs-back #### Backup With the backup step a path can be backed up to a repository. + The `backup` step needs to have the following fields: - `repository`: This is a reference to a repository via the repository id - `source_path`: The path that needs to be backed up to the repository @@ -100,6 +101,7 @@ Optionally the `tags` field can be provided with a list of tags that needs to be #### Copy With the copy step snapshots can be copied between repositories. + The `copy` step needs to have the following fields: - `source_repository`: Reference to the soruce repository via the repository id - `target_repository`: Reference to the target repository via the repository id From f33608a4fccba857212c86234c056b7a4646af11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Kocsis?= Date: Mon, 13 Apr 2026 22:45:43 +0200 Subject: [PATCH 04/11] ResticBackupStep parsing: Added checks for missing keys --- .../restic/restic_playbook_step_parser.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/source/backup_automation/restic/restic_playbook_step_parser.py b/source/backup_automation/restic/restic_playbook_step_parser.py index e56f084..98953ab 100644 --- a/source/backup_automation/restic/restic_playbook_step_parser.py +++ b/source/backup_automation/restic/restic_playbook_step_parser.py @@ -37,10 +37,17 @@ def parse(self, step_json: JsonDict) -> ResticPlaybookStep: raise ResticPlaybookException(f"Unexpected command in step: {step_json}") def __parse_backup_step(self, step_json: JsonDict) -> ResticPlaybookBackupStep: - repository_id = step_json[self.__format.STEPS_BACKUP_REPOSITORY_KEY] + repository_id = step_json.get(self.__format.STEPS_BACKUP_REPOSITORY_KEY, None) + if repository_id is None: + raise ResticPlaybookException(f"Missing required key for backup step: {self.__format.STEPS_BACKUP_REPOSITORY_KEY}") + repository = self.__repository_lookup(repository_id) - source_path = pathlib.Path(step_json[self.__format.STEPS_BACKUP_SOURCE_PATH_KEY]) + raw_source_path = step_json.get(self.__format.STEPS_BACKUP_SOURCE_PATH_KEY, None) + if raw_source_path is None: + raise ResticPlaybookException(f"Missing required key for backup step: {self.__format.STEPS_BACKUP_SOURCE_PATH_KEY}") + + source_path = pathlib.Path(raw_source_path) if not source_path.is_dir(): raise ResticPlaybookException(f"Source path is not a valid directory: {source_path}") From 8e4291773179c63003017333e59ddbd9560eecd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Kocsis?= Date: Mon, 13 Apr 2026 19:37:25 +0200 Subject: [PATCH 05/11] Added the feature that enables defining a working directory for restic backup steps --- README.md | 7 +++++- .../restic/restic_playbook_format.py | 1 + .../restic/restic_playbook_step_parser.py | 25 ++++++++++++++++--- .../restic/restic_playbook_steps.py | 12 +++++++-- 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 18864ea..8394642 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,12 @@ The `backup` step needs to have the following fields: - `repository`: This is a reference to a repository via the repository id - `source_path`: The path that needs to be backed up to the repository -Optionally the `tags` field can be provided with a list of tags that needs to be applied to the new snapshot. +The following fields are optional: +- `tags`: list of tags that needs to be applied to the new snapshot +- `working_dir`: The restic command will be executed in this directory. + The working directory will be changed only temporarily while executing the step + and after execution the previous value will be restored. + If undefined, no change will be done. #### Copy diff --git a/source/backup_automation/restic/restic_playbook_format.py b/source/backup_automation/restic/restic_playbook_format.py index 426c2ba..0ac9baa 100644 --- a/source/backup_automation/restic/restic_playbook_format.py +++ b/source/backup_automation/restic/restic_playbook_format.py @@ -22,5 +22,6 @@ class ResticPlaybookFormat(PlaybookFormat): STEPS_BACKUP_REPOSITORY_KEY = "repository" STEPS_BACKUP_SOURCE_PATH_KEY = "source_path" STEPS_BACKUP_TAGS_KEY = "tags" + STEPS_BACKUP_WORKING_DIR_KEY = "working_dir" STEPS_COPY_SOURCE_REPOSITORY_KEY = "source_repository" STEPS_COPY_TARGET_REPOSITORY_KEY = "target_repository" diff --git a/source/backup_automation/restic/restic_playbook_step_parser.py b/source/backup_automation/restic/restic_playbook_step_parser.py index 98953ab..b784adc 100644 --- a/source/backup_automation/restic/restic_playbook_step_parser.py +++ b/source/backup_automation/restic/restic_playbook_step_parser.py @@ -48,14 +48,33 @@ def __parse_backup_step(self, step_json: JsonDict) -> ResticPlaybookBackupStep: raise ResticPlaybookException(f"Missing required key for backup step: {self.__format.STEPS_BACKUP_SOURCE_PATH_KEY}") source_path = pathlib.Path(raw_source_path) - if not source_path.is_dir(): - raise ResticPlaybookException(f"Source path is not a valid directory: {source_path}") tags = step_json.get(self.__format.STEPS_BACKUP_TAGS_KEY, []) if not isinstance(tags, list): raise ResticPlaybookException(f"Tags is not a valid JSON array: {tags}") - return ResticPlaybookBackupStep(self.__backend, repository, source_path, tuple(tags)) + raw_working_dir = step_json.get(self.__format.STEPS_BACKUP_WORKING_DIR_KEY, None) + working_dir = pathlib.Path(raw_working_dir) if raw_working_dir else None + + if working_dir: + if source_path.is_absolute(): + raise ResticPlaybookException(f"If working_dir is defined, source_path must be a relative path!" + f" working_dir: {working_dir} source_path: {source_path}") + + path_to_backup = working_dir / source_path + if not path_to_backup.is_dir(): + raise ResticPlaybookException(f"The path to backup (combination of working_dir and source_path)" + f" is not a valid directory: {path_to_backup}") + else: + if not source_path.is_absolute(): + raise ResticPlaybookException(f"If working_dir is not defined, source_path must be an absolute path! " + f"source_path: {source_path}") + + if not source_path.is_dir(): + raise ResticPlaybookException(f"The path to backup (source_path)" + f" is not a valid directory: {source_path}") + + return ResticPlaybookBackupStep(self.__backend, repository, source_path, tuple(tags), working_dir) def __parse_copy_step(self, step_json: JsonDict) -> ResticPlaybookCopyStep: source_repository_id = step_json[self.__format.STEPS_COPY_SOURCE_REPOSITORY_KEY] diff --git a/source/backup_automation/restic/restic_playbook_steps.py b/source/backup_automation/restic/restic_playbook_steps.py index 77840fa..c4422b9 100644 --- a/source/backup_automation/restic/restic_playbook_steps.py +++ b/source/backup_automation/restic/restic_playbook_steps.py @@ -1,3 +1,4 @@ +import contextlib import pathlib from abc import ABC @@ -22,19 +23,26 @@ class ResticPlaybookBackupStep(ResticPlaybookStep): """ Class to represent the restic specific backup playbook step. """ + # pylint: disable=too-many-arguments + # pylint: disable=too-many-positional-arguments + # This class needs to represent a backup step with all its fields. def __init__(self, backend: ResticBackend, repository: ResticRepository, source_path: pathlib.Path, - tags: tuple[str, ...]): + tags: tuple[str, ...], + working_dir: pathlib.Path | None): super().__init__() self.__backend = backend self.__repository = repository self.__source_path = source_path self.__tags = tags + self.__working_dir = working_dir def execute(self) -> None: - self.__backend.backup(self.__repository, self.__source_path, self.__tags) + working_dir_context = contextlib.chdir(self.__working_dir) if self.__working_dir else contextlib.nullcontext() + with working_dir_context: + self.__backend.backup(self.__repository, self.__source_path, self.__tags) class ResticPlaybookCopyStep(ResticPlaybookStep): From b00fc121c9bec7cb5705a00e6f58073c896bdf73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Kocsis?= Date: Mon, 13 Apr 2026 22:51:28 +0200 Subject: [PATCH 06/11] ResticCopyStep parsing: Added checks for missing keys --- .../restic/restic_playbook_step_parser.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/source/backup_automation/restic/restic_playbook_step_parser.py b/source/backup_automation/restic/restic_playbook_step_parser.py index b784adc..a67638f 100644 --- a/source/backup_automation/restic/restic_playbook_step_parser.py +++ b/source/backup_automation/restic/restic_playbook_step_parser.py @@ -77,11 +77,18 @@ def __parse_backup_step(self, step_json: JsonDict) -> ResticPlaybookBackupStep: return ResticPlaybookBackupStep(self.__backend, repository, source_path, tuple(tags), working_dir) def __parse_copy_step(self, step_json: JsonDict) -> ResticPlaybookCopyStep: - source_repository_id = step_json[self.__format.STEPS_COPY_SOURCE_REPOSITORY_KEY] + source_repository_id = step_json.get(self.__format.STEPS_COPY_SOURCE_REPOSITORY_KEY, None) + if source_repository_id is None: + raise ResticPlaybookException(f"Missing required key for copy step: {self.__format.STEPS_COPY_SOURCE_REPOSITORY_KEY}") + source_repository = self.__repository_lookup(source_repository_id) - target_repository_id = step_json[self.__format.STEPS_COPY_TARGET_REPOSITORY_KEY] + target_repository_id = step_json.get(self.__format.STEPS_COPY_TARGET_REPOSITORY_KEY, None) + if target_repository_id is None: + raise ResticPlaybookException(f"Missing required key for copy step: {self.__format.STEPS_COPY_TARGET_REPOSITORY_KEY}") + target_repository = self.__repository_lookup(target_repository_id) + if source_repository == target_repository: raise ResticPlaybookException("The source and target repositories cannot be the same!") From 5c1e30eb7cf2c1516e67c6656ea80e8a6a40b129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Kocsis?= Date: Tue, 14 Apr 2026 19:56:12 +0200 Subject: [PATCH 07/11] Added json_config.py --- source/backup_automation/json_config.py | 42 +++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 source/backup_automation/json_config.py diff --git a/source/backup_automation/json_config.py b/source/backup_automation/json_config.py new file mode 100644 index 0000000..6ab7f8e --- /dev/null +++ b/source/backup_automation/json_config.py @@ -0,0 +1,42 @@ +from typing import Type, TypeVar + +from backup_automation.typehints import JsonDict, JsonDictKey + + +T = TypeVar("T") + + +class JsonConfigException(Exception): + """ + Exception to raise when restic playbook related errors happen. + """ + + +def __get_config_value(configuration: JsonDict, key: JsonDictKey, expected_type: Type[T]) -> T: + value = configuration[key] + if not isinstance(value, expected_type): + raise JsonConfigException(f"Expected type {expected_type} for key {key} but got {type(value)} instead.") + + return value + + +def get_optional_config_value(configuration: JsonDict, key: JsonDictKey, expected_type: Type[T]) -> T | None: + """ + Returns value from configuration if key is present, returns None otherwise. + If the key is present but the value is not of the expected type, raises JsonConfigException. + """ + if key not in configuration: + return None + + return __get_config_value(configuration, key, expected_type) + + +def get_required_config_value(configuration: JsonDict, key: JsonDictKey, expected_type: Type[T]) -> T: + """ + Returns value from configuration if key is present, raises JsonConfigException otherwise. + If the key is present but the value is not of the expected type, raises JsonConfigException. + """ + if key not in configuration: + raise JsonConfigException(f"Missing required key in configuration: {key}") + + return __get_config_value(configuration, key, expected_type) From f7d28cb62dd8ccd154f88f2f51484f4dff4fa599 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Kocsis?= Date: Tue, 14 Apr 2026 19:56:26 +0200 Subject: [PATCH 08/11] Extended typehints.py --- source/backup_automation/typehints.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/backup_automation/typehints.py b/source/backup_automation/typehints.py index f754658..be7c1f8 100644 --- a/source/backup_automation/typehints.py +++ b/source/backup_automation/typehints.py @@ -1,5 +1,8 @@ from typing import Any -JsonDict = dict[str, Any] -JsonList = list[Any] +JsonDictKey = str +JsonDictValue = Any +JsonDict = dict[JsonDictKey, JsonDictValue] +JsonListElement = Any +JsonList = list[JsonListElement] From a3c81db06e58855c0677c93fecb02df93bd6a9b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Kocsis?= Date: Tue, 14 Apr 2026 19:59:04 +0200 Subject: [PATCH 09/11] restic_playbook_step_parser: Simplified parsing via json_config.py Added extra configuration checks as well: missing keys, type errors. --- README.md | 12 ++++++---- .../restic/restic_playbook_step_parser.py | 23 +++++-------------- 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 8394642..3148b68 100644 --- a/README.md +++ b/README.md @@ -93,23 +93,25 @@ These represent their matching restic commands: [restic backup][restic-docs-back With the backup step a path can be backed up to a repository. The `backup` step needs to have the following fields: -- `repository`: This is a reference to a repository via the repository id -- `source_path`: The path that needs to be backed up to the repository +- `repository`: This is a reference to a repository via the repository id. Must be a string. +- `source_path`: The path that needs to be backed up to the repository. + If a `working_dir` is defined, then it must be a relative path, + otherwise it must be an absolute path. Must be a string. The following fields are optional: - `tags`: list of tags that needs to be applied to the new snapshot - `working_dir`: The restic command will be executed in this directory. The working directory will be changed only temporarily while executing the step and after execution the previous value will be restored. - If undefined, no change will be done. + If undefined, no change will be done. Must be a string. #### Copy With the copy step snapshots can be copied between repositories. The `copy` step needs to have the following fields: -- `source_repository`: Reference to the soruce repository via the repository id -- `target_repository`: Reference to the target repository via the repository id +- `source_repository`: Reference to the source repository via the repository id. Must be a string. +- `target_repository`: Reference to the target repository via the repository id. Must be a string. ### Simple example diff --git a/source/backup_automation/restic/restic_playbook_step_parser.py b/source/backup_automation/restic/restic_playbook_step_parser.py index a67638f..a54a52e 100644 --- a/source/backup_automation/restic/restic_playbook_step_parser.py +++ b/source/backup_automation/restic/restic_playbook_step_parser.py @@ -1,6 +1,7 @@ import pathlib from typing import Callable +from backup_automation.json_config import JsonConfigException, get_optional_config_value, get_required_config_value from backup_automation.restic.restic_backend import ResticBackend from backup_automation.restic.restic_playbook_exception import ResticPlaybookException from backup_automation.restic.restic_playbook_format import ResticPlaybookFormat @@ -37,23 +38,17 @@ def parse(self, step_json: JsonDict) -> ResticPlaybookStep: raise ResticPlaybookException(f"Unexpected command in step: {step_json}") def __parse_backup_step(self, step_json: JsonDict) -> ResticPlaybookBackupStep: - repository_id = step_json.get(self.__format.STEPS_BACKUP_REPOSITORY_KEY, None) - if repository_id is None: - raise ResticPlaybookException(f"Missing required key for backup step: {self.__format.STEPS_BACKUP_REPOSITORY_KEY}") - + repository_id = get_required_config_value(step_json, self.__format.STEPS_BACKUP_REPOSITORY_KEY, str) repository = self.__repository_lookup(repository_id) - raw_source_path = step_json.get(self.__format.STEPS_BACKUP_SOURCE_PATH_KEY, None) - if raw_source_path is None: - raise ResticPlaybookException(f"Missing required key for backup step: {self.__format.STEPS_BACKUP_SOURCE_PATH_KEY}") - + raw_source_path = get_required_config_value(step_json, self.__format.STEPS_BACKUP_SOURCE_PATH_KEY, str) source_path = pathlib.Path(raw_source_path) tags = step_json.get(self.__format.STEPS_BACKUP_TAGS_KEY, []) if not isinstance(tags, list): raise ResticPlaybookException(f"Tags is not a valid JSON array: {tags}") - raw_working_dir = step_json.get(self.__format.STEPS_BACKUP_WORKING_DIR_KEY, None) + raw_working_dir = get_optional_config_value(step_json, self.__format.STEPS_BACKUP_WORKING_DIR_KEY, str) working_dir = pathlib.Path(raw_working_dir) if raw_working_dir else None if working_dir: @@ -77,16 +72,10 @@ def __parse_backup_step(self, step_json: JsonDict) -> ResticPlaybookBackupStep: return ResticPlaybookBackupStep(self.__backend, repository, source_path, tuple(tags), working_dir) def __parse_copy_step(self, step_json: JsonDict) -> ResticPlaybookCopyStep: - source_repository_id = step_json.get(self.__format.STEPS_COPY_SOURCE_REPOSITORY_KEY, None) - if source_repository_id is None: - raise ResticPlaybookException(f"Missing required key for copy step: {self.__format.STEPS_COPY_SOURCE_REPOSITORY_KEY}") - + source_repository_id = get_required_config_value(step_json, self.__format.STEPS_COPY_SOURCE_REPOSITORY_KEY, str) source_repository = self.__repository_lookup(source_repository_id) - target_repository_id = step_json.get(self.__format.STEPS_COPY_TARGET_REPOSITORY_KEY, None) - if target_repository_id is None: - raise ResticPlaybookException(f"Missing required key for copy step: {self.__format.STEPS_COPY_TARGET_REPOSITORY_KEY}") - + target_repository_id = get_required_config_value(step_json, self.__format.STEPS_COPY_TARGET_REPOSITORY_KEY, str) target_repository = self.__repository_lookup(target_repository_id) if source_repository == target_repository: From 996f19b27c1dcb6e2f26dfc190242790320007bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Kocsis?= Date: Tue, 14 Apr 2026 19:59:23 +0200 Subject: [PATCH 10/11] restic_playbook_step_parser: Improved reporting of parsing errors --- .../restic/restic_playbook_step_parser.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/source/backup_automation/restic/restic_playbook_step_parser.py b/source/backup_automation/restic/restic_playbook_step_parser.py index a54a52e..c7db764 100644 --- a/source/backup_automation/restic/restic_playbook_step_parser.py +++ b/source/backup_automation/restic/restic_playbook_step_parser.py @@ -28,14 +28,17 @@ def parse(self, step_json: JsonDict) -> ResticPlaybookStep: """ Parses a playbook step from received JSON object into a ResticPlaybookStep object. """ - step_command = step_json[self.__format.STEPS_COMMAND_KEY] - match step_command: - case self.__format.STEPS_COMMAND_VALUE_BACKUP: - return self.__parse_backup_step(step_json) - case self.__format.STEPS_COMMAND_VALUE_COPY: - return self.__parse_copy_step(step_json) - case _: - raise ResticPlaybookException(f"Unexpected command in step: {step_json}") + try: + step_command = step_json[self.__format.STEPS_COMMAND_KEY] + match step_command: + case self.__format.STEPS_COMMAND_VALUE_BACKUP: + return self.__parse_backup_step(step_json) + case self.__format.STEPS_COMMAND_VALUE_COPY: + return self.__parse_copy_step(step_json) + case _: + raise ResticPlaybookException(f"Unexpected command in step: {step_json}") + except (ResticPlaybookException, JsonConfigException) as e: + raise ResticPlaybookException(f"Could not parse playbook step: {step_json}") from e def __parse_backup_step(self, step_json: JsonDict) -> ResticPlaybookBackupStep: repository_id = get_required_config_value(step_json, self.__format.STEPS_BACKUP_REPOSITORY_KEY, str) From a28ed905bfb2d5016c35fd194de00ec6d2ba5e90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Kocsis?= Date: Tue, 14 Apr 2026 14:15:11 +0200 Subject: [PATCH 11/11] backup_automation: Bumped version number --- source/backup_automation/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/backup_automation/__init__.py b/source/backup_automation/__init__.py index 493f741..6a9beea 100644 --- a/source/backup_automation/__init__.py +++ b/source/backup_automation/__init__.py @@ -1 +1 @@ -__version__ = "0.3.0" +__version__ = "0.4.0"