adds bundle merge functionality

bin/ecbundle

@@ -37,6 +37,9 @@ elif [[ "create" == "$1"* ]]; then
 elif [[ "populate" == "$1"* ]]; then
   shift
   ${SCRIPT_DIR}/ecbundle-populate "$@"
+elif [[ "merge" == "$1"* ]]; then
+  shift
+  ${SCRIPT_DIR}/ecbundle-merge "$@"
 else
   echo "ERROR: Expected 'build' or 'create' or 'populate' as first argument"
   usage

bin/ecbundle-merge

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+
+# (C) Copyright 2020- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation nor
+# does it submit to any jurisdiction.
+
+"""
+Script to merge and update bundle files
+"""
+
+import os
+import sys
+from argparse import SUPPRESS, ArgumentParser, RawTextHelpFormatter
+
+sys.path.insert(0, os.path.realpath(os.path.dirname(os.path.realpath(__file__))+'/..'))
+from ecbundle import BundleMerger
+from ecbundle.logging import DEBUG, colors, error, logger, success
+
+
+def main():
+
+    # Parse arguments
+    parser = ArgumentParser(description=__doc__,
+                            formatter_class=RawTextHelpFormatter)
+
+    # --------------------------------------------------------------------------
+    # Parse common subcommands
+    # --------------------------------------------------------------------------
+    parser.add_argument('--no-colour', '--no-color',
+                        help='Disable color output',
+                        action='store_true')
+
+    parser.add_argument('--verbose', '-v',
+                        help='Verbose output',
+                        action='store_true')
+
+    parser.add_argument('bundles',
+                        help='Bundle files: the first is the original bundle, '
+                             'any following are update bundles applied in order',
+                        nargs='+')
+
+
+    parser.add_argument('-o',
+                        help='output file', default="merged-bundle.yml")
+
+    # --------------------------------------------------------------------------
+
+    # Close parser and populate variable args
+    args = parser.parse_args()
+
+    if len(args.bundles) < 2:
+            parser.error('at least one update bundle is required in addition to the original')
+
+    # Explicitly disable coloured logs
+    if args.no_colour:
+        colors.disable()
+
+    # Log everything, including commands executed
+    if args.verbose:
+        logger.setLevel(DEBUG)
+
+
+    errcode = 0
+
+    if BundleMerger(**vars(args)).merge() != 0:
+        errcode = 1 # error
+
+    if errcode == 1:
+        error("\n!!! Errors occured !!!")
+
+    return errcode
+
+if __name__ == '__main__':
+    sys.exit(main())

ecbundle/__init__.py

@@ -12,6 +12,7 @@
 from ecbundle.download import *  # noqa
 from ecbundle.git import *  # noqa
 from ecbundle.logging import *  # noqa
+from ecbundle.merge import *  # noqa
 from ecbundle.option import *  # noqa
 from ecbundle.populate import *  # noqa
 from ecbundle.project import *  # noqa

ecbundle/merge.py

@@ -0,0 +1,118 @@
+# (C) Copyright 2020- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation nor
+# does it submit to any jurisdiction.
+
+import copy
+import os
+
+from .bundle import Bundle
+from .logging import error, header, info, success
+from .util import fullpath
+
+__all__ = ["BundleMerger"]
+
+
+class BundleMerger(object):
+    def __init__(self, **kwargs):
+        self.config = kwargs
+
+    def get(self, key, default=None):
+        return self.config[key] if self.config.get(key) is not None else default
+
+    def deep_merge(self, original, updates):
+        """Recursively merge `updates` into `original`.
+
+        Rules:
+        - Dictionaries are merged recursively.
+        - Lists and scalar values are replaced entirely.
+        - Keys missing from `updates` remain unchanged.
+        """
+        if isinstance(original, dict) and isinstance(updates, dict):
+            merged = copy.deepcopy(original)
+            for key, value in updates.items():
+                if key in merged:
+                    if isinstance(merged[key], dict) and isinstance(value, dict):
+                        merged[key] = self.deep_merge(merged[key], value)
+                    else:
+                        merged[key] = copy.deepcopy(value)
+                else:
+                    merged[key] = copy.deepcopy(value)
+            return merged
+
+        return copy.deepcopy(updates)
+
+    def _load_bundle(self, path, label):
+        """Load a bundle file from `path`, or return None with an error."""
+        bundle_path = fullpath(path)
+        if bundle_path and os.path.isfile(bundle_path):
+            return Bundle(bundle_path, env=True)
+
+        error(f"ERROR: {label} '{path}' is not a valid bundle file path")
+        return None
+
+    def _merge_named_list(self, base_bundle, key, base_items, update_items):
+        """Merge a named-item list (projects/options) from update into base."""
+        base_dict = {
+            item.config["name"]: {k: v for k, v in item.config.items() if k != "name"}
+            for item in base_items
+        }
+        update_dict = {
+            item.config["name"]: {k: v for k, v in item.config.items() if k != "name"}
+            for item in update_items
+        }
+
+        merged = self.deep_merge(base_dict, update_dict)
+        base_bundle.config[key] = [{name: value} for name, value in merged.items()]
+
+    def _apply_update(self, bundle, bundle_update):
+        """Fold a single update bundle into `bundle` in place."""
+        header("\nMerging bundle")
+        info(f"    {bundle_update.file()}")
+
+        self._merge_named_list(
+            bundle, "projects", bundle.projects(), bundle_update.projects()
+        )
+        self._merge_named_list(
+            bundle, "options", bundle.options(), bundle_update.options()
+        )
+
+        for key in bundle_update.config.keys():
+            if key not in ("projects", "options"):
+                bundle.config[key] = bundle_update.get(key)
+        success("Bundle succesfully merged")
+
+    def merge(self):
+        bundles = self.get("bundles", [])
+        if not bundles or len(bundles) < 2:
+            error("ERROR: need at least one original bundle and one update bundle")
+            return 1
+
+        header("\nMerging bundles:")
+        for bundle in bundles:
+            info(f" - {bundle}")
+
+        original_path, *update_paths = bundles
+
+        bundle = self._load_bundle(original_path, "original bundle")
+        if bundle is None:
+            return 1
+
+        for path in update_paths:
+            bundle_update = self._load_bundle(path, "update bundle")
+            if bundle_update is None:
+                return 1
+            self._apply_update(bundle, bundle_update)
+
+        output_path = self.get("output", "merged-bundle.yml")
+
+        header("\nWriting merge result into:")
+        info(f" - {output_path}")
+
+        with open(output_path, "w", encoding="utf-8") as f:
+            f.write(bundle.yaml())
+        success("Bundles succesfully merged\n")
+        return 0

ecbundle/project.py

@@ -74,6 +74,9 @@ def require(self):
         else:
             return None
 
+    def get_dict(self):
+        return self.config
+
     def optional(self):
         return self.get("optional", False)
 

tests/bundle_merge/bundle-merge-base.yml

@@ -0,0 +1,22 @@
+name    : merge-test-full
+cmake   : CMAKE_BUILD_TYPE=Release
+
+options :
+
+    - without-mpi :
+        help    : Disable MPI
+        cmake   : ENABLE_MPI=OFF
+
+    - with-gpu :
+        help    : Enable GPU support
+        cmake   : ENABLE_GPU=ON
+
+projects :
+
+    - project1 :
+        git     : https://github.com/example/project1
+        version : main
+
+    - project2 :
+        git     : https://github.com/example/project2
+        version : main

tests/bundle_merge/bundle-merge-update-options.yml

@@ -0,0 +1,9 @@
+options :
+
+    - without-mpi :
+        help    : MPI disabled (updated)
+        cmake   : ENABLE_MPI=OFF
+
+    - with-openmp :
+        help    : Enable OpenMP
+        cmake   : ENABLE_OMP=ON

tests/bundle_merge/bundle-merge-update-options2.yml

@@ -0,0 +1,9 @@
+options :
+
+    - with-openmp :
+        help    : OpenMP (final)
+        cmake   : ENABLE_OMP=ON
+
+    - with-gpu :
+        help    : GPU (final)
+        cmake   : ENABLE_GPU=ON

tests/bundle_merge/bundle-merge-update-toplevel.yml

@@ -0,0 +1,2 @@
+name    : merge-test-renamed
+cmake   : CMAKE_BUILD_TYPE=Debug

tests/bundle_merge/bundle-merge-update.yml

@@ -0,0 +1,4 @@
+projects :  
+
+    - project1 :  
+        version : updated-branch

tests/bundle_merge/bundle-merge-update2.yml

@@ -0,0 +1,6 @@
+projects :  
+
+    - project1 :  
+        version : final-branch
+
+