From 145852ea6935c9ca96fddf047e9eefd7a7aeee67 Mon Sep 17 00:00:00 2001 From: Timo Wilken Date: Mon, 9 Oct 2023 18:15:06 +0200 Subject: [PATCH] Support package variants for dev packages This allows compiling development packages partially (e.g. using a non-standard CMake target) to save time when developing locally. It also allows packages to depend on "variants" of other packages, e.g. O2Physics -> O2#Analysis and O2sim -> O2#Simulation. aliBuild merges variants if multiple are required, e.g. aliBuild build O2sim O2Physics in the above case would build O2 with ALIBUILD_BUILD_VARIANT=Analysis,Simulation. "Full" builds (no variants) always take precedence, and are the only possible behaviour for non-development packages (e.g. to allow reusing tarballs as before, and to avoid heisenbugs in CI). --- alibuild_helpers/build.py | 12 ++++++++++++ alibuild_helpers/utilities.py | 20 +++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/alibuild_helpers/build.py b/alibuild_helpers/build.py index 93104cf9..4a513bdb 100644 --- a/alibuild_helpers/build.py +++ b/alibuild_helpers/build.py @@ -457,6 +457,13 @@ def doBuild(args, parser): ", ".join({x.strip() for x, _ in develPkgsUpper} - set(develPkgs))) return 1 + # We must not apply variants to non-development packages (so that the full + # package can be downloaded from the remote, for example). Delete the + # relevant packages' variants, now that we know the dev packages. + for spec in specs.values(): + if spec["package"] not in develPkgs: + spec.pop("variants", None) + if buildOrder: banner("Packages will be built in the following order:\n - %s", "\n - ".join(x+" (development package)" if x in develPkgs else "%s@%s" % (x, specs[x]["tag"]) @@ -510,6 +517,9 @@ def doBuild(args, parser): out = git(("rev-parse", "HEAD"), directory=spec["source"]) spec["commit_hash"] = out.strip() spec["devel_hash"] = spec["commit_hash"] + hash_local_changes(spec["source"]) + if spec.get("variants", ()): + # spec["variants"] is a set, so its order is undefined. Sort it. + spec["devel_hash"] += "/" + ",".join(sorted(spec["variants"])) out = git(("rev-parse", "--abbrev-ref", "HEAD"), directory=spec["source"]) if out == "HEAD": out = git(("rev-parse", "HEAD"), directory=spec["source"])[:10] @@ -1058,6 +1068,8 @@ def doBuild(args, parser): ("FULL_BUILD_REQUIRES", " ".join(spec["full_build_requires"])), ("FULL_REQUIRES", " ".join(spec["full_requires"])), ("WRITE_REPO", spec.get("write_repo", source)), + # spec["variants"] is a set, so its order is undefined. Sort it. + ("ALIBUILD_BUILD_VARIANT", ",".join(sorted(spec.get("variants", ())))), ] # Add the extra environment as passed from the command line. buildEnvironment += [e.partition('=')[::2] for e in args.environment] diff --git a/alibuild_helpers/utilities.py b/alibuild_helpers/utilities.py index c7361355..6a1b8b78 100644 --- a/alibuild_helpers/utilities.py +++ b/alibuild_helpers/utilities.py @@ -358,12 +358,30 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, packages = packages[:] validDefaults = [] # empty list: all OK; None: no valid default; non-empty list: list of valid ones while packages: - p = packages.pop(0) + # Find requested variants of packages. + # We only apply variants to development packages (we always store + # requested variants here, but delete them from non-dev packages later). + # Variants allow building only a subset of a package (e.g. a specific + # CMake target), saving build time. + p, has_variants, variants = packages.pop(0).partition("#") + variants = {var.strip() for var in variants.split(",") if var} + if p in specs: + if not has_variants: + # If we now want to build without variants (i.e. the whole package), + # wipe out any previously-configured variants for this package. + specs[p].pop("variants", None) + elif specs[p].get("variants"): + # The existing package has variants, and this repetition of it does + # too. Merge the variants lists. + specs[p]["variants"] |= variants continue + lowerPkg = p.lower() filename = taps.get(lowerPkg, "%s/%s.sh" % (configDir, lowerPkg)) err, spec, recipe = parseRecipe(getRecipeReader(filename, configDir)) + if has_variants and variants: + spec["variants"] = variants dieOnError(err, err) dieOnError(spec["package"].lower() != lowerPkg, "%s.sh has different package field: %s" % (p, spec["package"]))