diff --git a/bin/ecbundle b/bin/ecbundle index 642e9c2..32558ad 100755 --- a/bin/ecbundle +++ b/bin/ecbundle @@ -37,6 +37,9 @@ elif [[ "create" == "$1"* ]]; then elif [[ "populate" == "$1"* ]]; then shift ${SCRIPT_DIR}/ecbundle-populate "$@" +elif [[ "merge" == "$1"* ]]; then + shift + ${SCRIPT_DIR}/ecbundle-merge "$@" else echo "ERROR: Expected 'build' or 'create' or 'populate' as first argument" usage diff --git a/bin/ecbundle-merge b/bin/ecbundle-merge new file mode 100755 index 0000000..86deec0 --- /dev/null +++ b/bin/ecbundle-merge @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +# (C) Copyright 2020- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +""" +Script to merge and update bundle files +""" + +import os +import sys +from argparse import SUPPRESS, ArgumentParser, RawTextHelpFormatter + +sys.path.insert(0, os.path.realpath(os.path.dirname(os.path.realpath(__file__))+'/..')) +from ecbundle import BundleMerger +from ecbundle.logging import DEBUG, colors, error, logger, success + + +def main(): + + # Parse arguments + parser = ArgumentParser(description=__doc__, + formatter_class=RawTextHelpFormatter) + + # -------------------------------------------------------------------------- + # Parse common subcommands + # -------------------------------------------------------------------------- + parser.add_argument('--no-colour', '--no-color', + help='Disable color output', + action='store_true') + + parser.add_argument('--verbose', '-v', + help='Verbose output', + action='store_true') + + parser.add_argument('bundles', + help='Bundle files: the first is the original bundle, ' + 'any following are update bundles applied in order', + nargs='+') + + + parser.add_argument('-o', + help='output file', default="merged-bundle.yml") + + # -------------------------------------------------------------------------- + + # Close parser and populate variable args + args = parser.parse_args() + + if len(args.bundles) < 2: + parser.error('at least one update bundle is required in addition to the original') + + # Explicitly disable coloured logs + if args.no_colour: + colors.disable() + + # Log everything, including commands executed + if args.verbose: + logger.setLevel(DEBUG) + + + errcode = 0 + + if BundleMerger(**vars(args)).merge() != 0: + errcode = 1 # error + + if errcode == 1: + error("\n!!! Errors occured !!!") + + return errcode + +if __name__ == '__main__': + sys.exit(main()) diff --git a/ecbundle/__init__.py b/ecbundle/__init__.py index 1e2dde7..998c03f 100644 --- a/ecbundle/__init__.py +++ b/ecbundle/__init__.py @@ -12,6 +12,7 @@ from ecbundle.download import * # noqa from ecbundle.git import * # noqa from ecbundle.logging import * # noqa +from ecbundle.merge import * # noqa from ecbundle.option import * # noqa from ecbundle.populate import * # noqa from ecbundle.project import * # noqa diff --git a/ecbundle/merge.py b/ecbundle/merge.py new file mode 100644 index 0000000..8369926 --- /dev/null +++ b/ecbundle/merge.py @@ -0,0 +1,118 @@ +# (C) Copyright 2020- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +import copy +import os + +from .bundle import Bundle +from .logging import error, header, info, success +from .util import fullpath + +__all__ = ["BundleMerger"] + + +class BundleMerger(object): + def __init__(self, **kwargs): + self.config = kwargs + + def get(self, key, default=None): + return self.config[key] if self.config.get(key) is not None else default + + def deep_merge(self, original, updates): + """Recursively merge `updates` into `original`. + + Rules: + - Dictionaries are merged recursively. + - Lists and scalar values are replaced entirely. + - Keys missing from `updates` remain unchanged. + """ + if isinstance(original, dict) and isinstance(updates, dict): + merged = copy.deepcopy(original) + for key, value in updates.items(): + if key in merged: + if isinstance(merged[key], dict) and isinstance(value, dict): + merged[key] = self.deep_merge(merged[key], value) + else: + merged[key] = copy.deepcopy(value) + else: + merged[key] = copy.deepcopy(value) + return merged + + return copy.deepcopy(updates) + + def _load_bundle(self, path, label): + """Load a bundle file from `path`, or return None with an error.""" + bundle_path = fullpath(path) + if bundle_path and os.path.isfile(bundle_path): + return Bundle(bundle_path, env=True) + + error(f"ERROR: {label} '{path}' is not a valid bundle file path") + return None + + def _merge_named_list(self, base_bundle, key, base_items, update_items): + """Merge a named-item list (projects/options) from update into base.""" + base_dict = { + item.config["name"]: {k: v for k, v in item.config.items() if k != "name"} + for item in base_items + } + update_dict = { + item.config["name"]: {k: v for k, v in item.config.items() if k != "name"} + for item in update_items + } + + merged = self.deep_merge(base_dict, update_dict) + base_bundle.config[key] = [{name: value} for name, value in merged.items()] + + def _apply_update(self, bundle, bundle_update): + """Fold a single update bundle into `bundle` in place.""" + header("\nMerging bundle") + info(f" {bundle_update.file()}") + + self._merge_named_list( + bundle, "projects", bundle.projects(), bundle_update.projects() + ) + self._merge_named_list( + bundle, "options", bundle.options(), bundle_update.options() + ) + + for key in bundle_update.config.keys(): + if key not in ("projects", "options"): + bundle.config[key] = bundle_update.get(key) + success("Bundle succesfully merged") + + def merge(self): + bundles = self.get("bundles", []) + if not bundles or len(bundles) < 2: + error("ERROR: need at least one original bundle and one update bundle") + return 1 + + header("\nMerging bundles:") + for bundle in bundles: + info(f" - {bundle}") + + original_path, *update_paths = bundles + + bundle = self._load_bundle(original_path, "original bundle") + if bundle is None: + return 1 + + for path in update_paths: + bundle_update = self._load_bundle(path, "update bundle") + if bundle_update is None: + return 1 + self._apply_update(bundle, bundle_update) + + output_path = self.get("output", "merged-bundle.yml") + + header("\nWriting merge result into:") + info(f" - {output_path}") + + with open(output_path, "w", encoding="utf-8") as f: + f.write(bundle.yaml()) + success("Bundles succesfully merged\n") + return 0 diff --git a/ecbundle/project.py b/ecbundle/project.py index dfe8fe5..a55b28e 100644 --- a/ecbundle/project.py +++ b/ecbundle/project.py @@ -74,6 +74,9 @@ def require(self): else: return None + def get_dict(self): + return self.config + def optional(self): return self.get("optional", False) diff --git a/tests/bundle_merge/bundle-merge-base.yml b/tests/bundle_merge/bundle-merge-base.yml new file mode 100644 index 0000000..2b406af --- /dev/null +++ b/tests/bundle_merge/bundle-merge-base.yml @@ -0,0 +1,22 @@ +name : merge-test-full +cmake : CMAKE_BUILD_TYPE=Release + +options : + + - without-mpi : + help : Disable MPI + cmake : ENABLE_MPI=OFF + + - with-gpu : + help : Enable GPU support + cmake : ENABLE_GPU=ON + +projects : + + - project1 : + git : https://github.com/example/project1 + version : main + + - project2 : + git : https://github.com/example/project2 + version : main \ No newline at end of file diff --git a/tests/bundle_merge/bundle-merge-update-options.yml b/tests/bundle_merge/bundle-merge-update-options.yml new file mode 100644 index 0000000..80b822b --- /dev/null +++ b/tests/bundle_merge/bundle-merge-update-options.yml @@ -0,0 +1,9 @@ +options : + + - without-mpi : + help : MPI disabled (updated) + cmake : ENABLE_MPI=OFF + + - with-openmp : + help : Enable OpenMP + cmake : ENABLE_OMP=ON \ No newline at end of file diff --git a/tests/bundle_merge/bundle-merge-update-options2.yml b/tests/bundle_merge/bundle-merge-update-options2.yml new file mode 100644 index 0000000..5305f8c --- /dev/null +++ b/tests/bundle_merge/bundle-merge-update-options2.yml @@ -0,0 +1,9 @@ +options : + + - with-openmp : + help : OpenMP (final) + cmake : ENABLE_OMP=ON + + - with-gpu : + help : GPU (final) + cmake : ENABLE_GPU=ON \ No newline at end of file diff --git a/tests/bundle_merge/bundle-merge-update-toplevel.yml b/tests/bundle_merge/bundle-merge-update-toplevel.yml new file mode 100644 index 0000000..ac49adc --- /dev/null +++ b/tests/bundle_merge/bundle-merge-update-toplevel.yml @@ -0,0 +1,2 @@ +name : merge-test-renamed +cmake : CMAKE_BUILD_TYPE=Debug \ No newline at end of file diff --git a/tests/bundle_merge/bundle-merge-update.yml b/tests/bundle_merge/bundle-merge-update.yml new file mode 100644 index 0000000..94f80ab --- /dev/null +++ b/tests/bundle_merge/bundle-merge-update.yml @@ -0,0 +1,4 @@ +projects : + + - project1 : + version : updated-branch \ No newline at end of file diff --git a/tests/bundle_merge/bundle-merge-update2.yml b/tests/bundle_merge/bundle-merge-update2.yml new file mode 100644 index 0000000..58c73b3 --- /dev/null +++ b/tests/bundle_merge/bundle-merge-update2.yml @@ -0,0 +1,6 @@ +projects : + + - project1 : + version : final-branch + + \ No newline at end of file diff --git a/tests/bundle_merge/test_merge.py b/tests/bundle_merge/test_merge.py new file mode 100644 index 0000000..3596077 --- /dev/null +++ b/tests/bundle_merge/test_merge.py @@ -0,0 +1,252 @@ +# (C) Copyright 2020- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + + +import shutil +from pathlib import Path + +import pytest + +from ecbundle import BundleMerger + + +@pytest.fixture +def here(): + return Path(__file__).parent.resolve() + + +@pytest.fixture +def out_dir(here): + d = here / "merge-output" + if d.exists(): + shutil.rmtree(d) + d.mkdir() + yield d + if d.exists(): + shutil.rmtree(d) + + +def _args(bundles, output): + return { + "no_colour": True, + "verbose": False, + "bundles": [str(b) for b in bundles], + "output": str(output), + } + + +def test_merge_single_update(here, out_dir): + """Original bundle merged with a single update file.""" + base = here / "bundle-merge-base.yml" + upd = here / "bundle-merge-update.yml" + output = out_dir / "merged.yml" + + rc = BundleMerger(**_args([base, upd], output)).merge() + + assert rc == 0 + assert output.exists() + content = output.read_text() + assert "project1" in content + assert "updated-branch" in content + + +def test_merge_multiple_updates_applied_in_order(here, out_dir): + """With two updates, the later one wins on conflicting fields.""" + base = here / "bundle-merge-base.yml" + upd1 = here / "bundle-merge-update.yml" + upd2 = here / "bundle-merge-update2.yml" + output = out_dir / "merged.yml" + + rc = BundleMerger(**_args([base, upd1, upd2], output)).merge() + + assert rc == 0 + assert output.exists() + content = output.read_text() + assert "final-branch" in content + assert "updated-branch" not in content + + +def test_merge_preserves_untouched_project(here, out_dir): + """A project not mentioned in the update should be preserved unchanged.""" + base = here / "bundle-merge-base.yml" + upd = here / "bundle-merge-update.yml" + output = out_dir / "merged.yml" + + rc = BundleMerger(**_args([base, upd], output)).merge() + + assert rc == 0 + content = output.read_text() + # project2 is only in the base, must survive the merge + assert "project2" in content + + +def test_merge_missing_original_fails(here, out_dir): + """A non-existent original bundle should cause merge to return non-zero.""" + base = here / "does-not-exist.yml" + upd = here / "bundle-merge-update.yml" + output = out_dir / "merged.yml" + + rc = BundleMerger(**_args([base, upd], output)).merge() + + assert rc != 0 + + +def test_merge_missing_update_fails(here, out_dir): + """A non-existent update bundle should cause merge to return non-zero.""" + base = here / "bundle-merge-base.yml" + upd = here / "does-not-exist-update.yml" + output = out_dir / "merged.yml" + + rc = BundleMerger(**_args([base, upd], output)).merge() + + assert rc != 0 + + +def test_merge_requires_at_least_one_update(here, out_dir): + """Passing only the original bundle should be rejected by merge().""" + base = here / "bundle-merge-base.yml" + output = out_dir / "merged.yml" + + rc = BundleMerger(**_args([base], output)).merge() + + assert rc != 0 + + +# --------------------------------------------------------------------------- +# Options section +# --------------------------------------------------------------------------- + + +def test_merge_updates_existing_option(here, out_dir): + """An option present in both base and update should take the update's values.""" + base = here / "bundle-merge-base.yml" + upd = here / "bundle-merge-update-options.yml" + output = out_dir / "merged.yml" + + rc = BundleMerger(**_args([base, upd], output)).merge() + + assert rc == 0 + content = output.read_text() + assert "without-mpi" in content + assert "MPI disabled (updated)" in content + # Original help text must be gone + assert "Disable MPI" not in content + + +def test_merge_adds_new_option(here, out_dir): + """An option only in the update should be added to the merged bundle.""" + base = here / "bundle-merge-base.yml" + upd = here / "bundle-merge-update-options.yml" + output = out_dir / "merged.yml" + + rc = BundleMerger(**_args([base, upd], output)).merge() + + assert rc == 0 + content = output.read_text() + assert "with-openmp" in content + assert "ENABLE_OMP=ON" in content + + +def test_merge_preserves_untouched_option(here, out_dir): + """An option not mentioned in the update should remain in the merged bundle.""" + base = here / "bundle-merge-base.yml" + upd = here / "bundle-merge-update-options.yml" + output = out_dir / "merged.yml" + + rc = BundleMerger(**_args([base, upd], output)).merge() + + assert rc == 0 + content = output.read_text() + # with-gpu is only in the base, must survive + assert "with-gpu" in content + assert "ENABLE_GPU=ON" in content + + +def test_merge_multiple_option_updates_apply_in_order(here, out_dir): + """With two option updates, later values override earlier ones.""" + base = here / "bundle-merge-base.yml" + upd1 = here / "bundle-merge-update-options.yml" + upd2 = here / "bundle-merge-update-options2.yml" + output = out_dir / "merged.yml" + + rc = BundleMerger(**_args([base, upd1, upd2], output)).merge() + + assert rc == 0 + content = output.read_text() + # Values from the second update must win + assert "OpenMP (final)" in content + assert "GPU (final)" in content + # Values overridden by the second update must not remain + assert "Enable OpenMP" not in content + assert "Enable GPU support" not in content + + +# --------------------------------------------------------------------------- +# Top-level scalar keys +# --------------------------------------------------------------------------- + + +def test_merge_overrides_toplevel_scalars(here, out_dir): + """Top-level scalar keys like `name` and `cmake` must be overridden by the update.""" + base = here / "bundle-merge-base.yml" + upd = here / "bundle-merge-update-toplevel.yml" + output = out_dir / "merged.yml" + + rc = BundleMerger(**_args([base, upd], output)).merge() + + assert rc == 0 + content = output.read_text() + assert "merge-test-renamed" in content + assert "CMAKE_BUILD_TYPE=Debug" in content + # Original scalar values must be gone + assert "merge-test-full" not in content + assert "CMAKE_BUILD_TYPE=Release" not in content + + +def test_merge_toplevel_update_preserves_projects_and_options(here, out_dir): + """An update that only touches top-level keys must leave projects/options intact.""" + base = here / "bundle-merge-base.yml" + upd = here / "bundle-merge-update-toplevel.yml" + output = out_dir / "merged.yml" + + rc = BundleMerger(**_args([base, upd], output)).merge() + + assert rc == 0 + content = output.read_text() + assert "project1" in content + assert "project2" in content + assert "without-mpi" in content + assert "with-gpu" in content + + +def test_merge_mixed_updates_across_sections(here, out_dir): + """Chained updates touching different sections should all be reflected.""" + base = here / "bundle-merge-base.yml" + upd_projects = here / "bundle-merge-update.yml" # touches projects + upd_options = here / "bundle-merge-update-options.yml" # touches options + upd_toplevel = here / "bundle-merge-update-toplevel.yml" # touches scalars + output = out_dir / "merged.yml" + + rc = BundleMerger( + **_args([base, upd_projects, upd_options, upd_toplevel], output) + ).merge() + + assert rc == 0 + content = output.read_text() + + # Projects update + assert "updated-branch" in content + # Options update + assert "with-openmp" in content + assert "MPI disabled (updated)" in content + # Top-level update + assert "merge-test-renamed" in content + assert "CMAKE_BUILD_TYPE=Debug" in content + # Untouched items still present + assert "project2" in content + assert "with-gpu" in content