diff --git a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml index 9a1afbf7..616131af 100644 --- a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml @@ -1,6 +1,8 @@ # Defaults file for the compatibility layer role. --- -eessi_version: "2025.06" +eessi_version: "2026.06" +eessi_host_os: "linux" +eessi_host_arch: "x86_64" custom_overlays: - name: eessi @@ -14,32 +16,12 @@ gentoo_prefix_path: /cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/c # How to build the prefix. gentoo_git_repo: https://github.com/gentoo/gentoo.git -# Select a specific commit in the gentoo_git_repo that should be used for the bootstrap, -# e.g. by checking: https://github.com/gentoo/gentoo/commits/master -# June 7 2025 (083e38cef302128d595e9f9cfd029ad8f67ec2b7) commit that includes Lmod 8.7.60 and glibc 2.41 -gentoo_git_commit: 083e38cef302128d595e9f9cfd029ad8f67ec2b7 +gentoo_git_branch: master prefix_required_space: 15 GB -prefix_user_defined_trusted_dirs: - - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/override" - - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/nvidia" - - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/amd" -prefix_mask_packages: | - # stick to GCC 13.x; using a too recent compiler in the compat layer may complicate stuff in the software layer, - # see for example https://github.com/EESSI/software-layer/issues/151 - >=sys-devel/gcc-14 -prefix_unmask_packages: | - # if necessary, unmask for instance an older GCC to make it installable -prefix_bootstrap_use_flags: | - # only build libnss, don't build the daemon (use the one from the host) - sys-auth/nss-pam-ldapd -nslcd - # don't build the SSSD daemon (and man pages) either - sys-auth/sssd -daemon -man - # only install Python 3.13 - */* PYTHON_TARGETS: -* python3_13 - */* PYTHON_SINGLE_TARGET: -* python3_13 - # Modules: disable installation of man pages and shell setup files to be able - # to also install Lmod. enable *conflict unload* and *require via* features. - sys-cluster/modules -shell-setup -man-install +new-features +prefix_user_defined_trusted_dirs: [] +prefix_mask_packages: +prefix_unmask_packages: +prefix_bootstrap_use_flags: prefix_use_builtin_bootstrap: false prefix_custom_bootstrap_script: local: "{{ playbook_dir }}/../../bootstrap-prefix.sh" @@ -66,63 +48,11 @@ prefix_locales: package_sets: - "eessi-{{ eessi_version }}-{{ eessi_host_os }}-{{ eessi_host_arch }}" -prefix_packages: - -prefix_remove_packages: - - app-eselect/eselect-rust - - dev-lang/go - - dev-lang/go-bootstrap - - dev-lang/rust - - dev-lang/rust-bin - - dev-python/setuptools-rust - - dev-util/cmake - - dev-util/hermes - - dev-util/ninja - - virtual/rust +prefix_packages: [] +prefix_remove_packages: [] reframe_venv_dir: /tmp/reframe_venv # List of locations that should get a symlink $EPREFIX/$LOCATION -> $LOCATION. # This ensures that things like user/group ids are correct/looked up in the right way in the Prefix environment. -symlinks_to_host: - # required to ensure local user groups are known - - /etc/group - - # required to ensure local users are known (see https://github.com/EESSI/compatibility-layer/issues/15) - - /etc/passwd - - # required to ensure that hosts specified in the host's hosts file can be resolved by tools in the Prefix - - /etc/hosts - - # required to ensure name-service information is taken from the right source (e.g. ldap) - # - /etc/nsswitch.conf - - # required to use the DNS resolver from the host (should be done automatically) - - /etc/resolv.conf - - # required to pick up the right timezone from the host - - /etc/localtime - - # required if Centrify is used in nsswitch.conf - - /lib64/libnss_centrifydc.so.2 - - # required if LDAP is used in nsswitch.conf - # - /lib64/libnss_ldap.so.2 - - # required if SSSD is used in nsswitch.conf - # - /lib64/libnss_sss.so.2 - - # required to make runtime data available for lots of tools (including who and w) - # - /var/run - - # required by the last command - # - /var/log/wtmp - -# - /var/lib/munge -# - /var/lib/unbound -# - /var/lib/VirtualGL -# - /var/log/munge -# - /var/spool -# - /var/tmp -# - /run/dbus -# - /tmp +symlinks_to_host: [] diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index db3c6471..c38f4b68 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -29,9 +29,20 @@ selectattr('eclass-overrides', 'equalto', True) | map(attribute='name') | join(' ') }} +- name: "Import the task for checking out the specific commit of git-based overlays" + ansible.builtin.include_tasks: sync_git_overlay.yml + vars: + gitrepo: "{{ item }}" + # also include the Gentoo git repo here in case we want to switch to a different commit + loop: >- + {{ + custom_overlays | selectattr('source', 'equalto', 'git') + + [{'name': 'gentoo', 'url': gentoo_git_repo, 'branch': gentoo_git_branch, 'commit': gentoo_git_commit}] + }} + # We do an explicit sync here (instead of using a handler), -# so we can be sure that the overlays are up-to-date. -- name: Sync the overlays to make sure that they are up to date +# so we can be sure that non-git overlays are also up-to-date. +- name: Sync overlays with auto-sync enabled to make sure that they are up to date community.general.portage: sync: 'yes' verbose: true diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/create_host_symlinks.yml b/ansible/playbooks/roles/compatibility_layer/tasks/create_host_symlinks.yml index 8737e581..9cb37212 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/create_host_symlinks.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/create_host_symlinks.yml @@ -4,13 +4,13 @@ - name: Check type of original files/directories ansible.builtin.stat: - path: "{{ gentoo_prefix_path }}{{ item }}" + path: "{{ item.dest if item is mapping else gentoo_prefix_path + item }}" with_items: "{{ symlinks_to_host }}" register: orig_types - name: Remove original files/directories ansible.builtin.file: - path: "{{ gentoo_prefix_path }}{{ item.item }}" + path: "{{ item.item.dest if item is mapping else gentoo_prefix_path + item.item }}" state: absent with_items: "{{ orig_types.results }}" when: @@ -19,8 +19,8 @@ - name: Make symlinks ansible.builtin.file: - src: "{{ item }}" - dest: "{{ gentoo_prefix_path }}{{ item }}" + src: "{{ item.src if item is mapping else item }}" + dest: "{{ item.dest if item is mapping else gentoo_prefix_path + item }}" state: link force: true with_items: "{{ symlinks_to_host }}" diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml b/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml index 4d0cb40e..d8d1efea 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml @@ -1,6 +1,6 @@ # Install a specified list of sets and packages. --- -- name: Install package set {{ package_sets }} +- name: Install package sets community.general.portage: package: "@{{ item }}" state: present @@ -8,8 +8,34 @@ tags: - set -- name: Install additional packages +- name: Determine packages to be installed for this architecture + ansible.builtin.set_fact: + # Join the following subsets of items from the package list: + # - packages without architecture constraints + # - packages with a "include_on" constraint that includes this architecture + # - packages with a "exclude_on" constraint that don't include this architecture + eessi_os_arch_packages: | + {{ prefix_packages + | rejectattr('include_on', 'defined') + | rejectattr('exclude_on', 'defined') + + prefix_packages + | selectattr('include_on', 'defined') + | selectattr('include_on', 'contains', eessi_host_os + '-' + eessi_host_arch) + + prefix_packages + | rejectattr('include_on', 'defined') + | selectattr('exclude_on', 'defined') + | rejectattr('exclude_on', 'contains', eessi_host_os + '-' + eessi_host_arch) + }} + +- name: Install EESSI packages community.general.portage: - package: "{{ item }}" + # Construct package atom: + # category/name + # =category/name-version + # =category/name-version::overlay + package: "{{ '=' if item.version is defined }}\ + {{ item.name }}\ + {{ '-' + item.version if item.version is defined }}\ + {{ '::' + item.overlay if item.overlay is defined }}" state: present - with_items: "{{ prefix_packages }}" + loop: "{{ eessi_os_arch_packages }}" diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/install_prefix.yml b/ansible/playbooks/roles/compatibility_layer/tasks/install_prefix.yml index f89df057..aa13396c 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/install_prefix.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/install_prefix.yml @@ -94,33 +94,14 @@ tags: - build_prefix -- name: "Clone the Gentoo git repository into the overlay directory" - ansible.builtin.git: - repo: "{{ gentoo_git_repo }}" - dest: "{{ gentoo_prefix_path }}/var/db/repos/gentoo" - depth: 1 - force: true - version: master - tags: - - build_prefix - -- name: "Fetch the specific git commit to be used for the bootstrap" - ansible.builtin.git: - repo: "{{ gentoo_git_repo }}" - dest: "{{ gentoo_prefix_path }}/var/db/repos/gentoo" - clone: false - refspec: "{{ gentoo_git_commit }}" - depth: 1 - version: master - tags: - - build_prefix - -- name: "Checkout the fetched git commit" - ansible.builtin.git: - repo: "{{ gentoo_git_repo }}" - dest: "{{ gentoo_prefix_path }}/var/db/repos/gentoo" - clone: false - version: "{{ gentoo_git_commit }}" +- name: "Import the task for checking out the specific commit of git-based overlays" + ansible.builtin.import_tasks: sync_git_overlay.yml + vars: + gitrepo: + name: "gentoo" + url: "{{ gentoo_git_repo }}" + branch: "{{ gentoo_git_branch }}" + commit: "{{ gentoo_git_commit }}" tags: - build_prefix diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/main.yml b/ansible/playbooks/roles/compatibility_layer/tasks/main.yml index 7a45863a..a8f935a5 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/main.yml @@ -11,6 +11,22 @@ target CPU architecture {{ eessi_host_arch }}. when: not (ansible_architecture == eessi_host_arch) +- name: Check for a vars file for this EESSI version + ansible.builtin.stat: + path: "{{ role_path }}/vars/{{ eessi_version }}.yml" + delegate_to: localhost + register: eessi_vars_file + +- name: Fail if no vars file is available for the specified/default EESSI version + ansible.builtin.fail: + msg: | + There is no vars file for EESSI version {{ eessi_version }} in the role's vars subdirectory. + Please create one and try again. + when: not eessi_vars_file.stat.exists + +- name: Include the variables file for this EESSI version + ansible.builtin.include_vars: "{{ role_path }}/vars/{{ eessi_version }}.yml" + - name: Check if a Prefix installation is found at the specified location ansible.builtin.stat: path: "{{ gentoo_prefix_path }}/startprefix" diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/prefix_configuration.yml b/ansible/playbooks/roles/compatibility_layer/tasks/prefix_configuration.yml index 3453b927..d26a5696 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/prefix_configuration.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/prefix_configuration.yml @@ -32,3 +32,7 @@ state: present with_items: "{{ prefix_locales }}" notify: Generate locales + +- name: Flush handlers to generate new locales. + ansible.builtin.meta: flush_handlers +... diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/sync_git_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/sync_git_overlay.yml new file mode 100644 index 00000000..7bd8ab7a --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/tasks/sync_git_overlay.yml @@ -0,0 +1,25 @@ +--- +- name: "Clone the git repository of a git-based overlay into the prefix overlay directory" + ansible.builtin.git: + repo: "{{ gitrepo.url }}" + dest: "{{ gentoo_prefix_path }}/var/db/repos/{{ gitrepo.name }}" + depth: 1 + force: true + version: "{{ gitrepo.branch | default('HEAD') }}" + +- name: "Fetch the specific git commit to be used for the git-based overlay" + ansible.builtin.git: + repo: "{{ gitrepo.url }}" + dest: "{{ gentoo_prefix_path }}/var/db/repos/{{ gitrepo.name }}" + clone: false + refspec: "{{ gitrepo.commit }}" + depth: 1 + version: "{{ gitrepo.branch | default('HEAD') }}" + +- name: "Checkout the fetched git commit" + ansible.builtin.git: + repo: "{{ gitrepo.url }}" + dest: "{{ gentoo_prefix_path }}/var/db/repos/{{ gitrepo.name }}" + clone: false + version: "{{ gitrepo.commit }}" +... diff --git a/ansible/playbooks/roles/compatibility_layer/templates/overlay.conf.j2 b/ansible/playbooks/roles/compatibility_layer/templates/overlay.conf.j2 index 023ef327..d4b83ea5 100644 --- a/ansible/playbooks/roles/compatibility_layer/templates/overlay.conf.j2 +++ b/ansible/playbooks/roles/compatibility_layer/templates/overlay.conf.j2 @@ -6,3 +6,10 @@ sync-uri = {{ item.url }} {% if item.branch is defined %} sync-git-clone-extra-opts = --branch {{ item.branch }} {% endif %} +{% if item.autosync is defined %} +auto-sync = {{ item.autosync }} +{% elif item.source == 'git' %} +auto-sync = no +{% else %} +auto-sync = yes +{% endif %} diff --git a/ansible/playbooks/roles/compatibility_layer/vars/2023.06.yml b/ansible/playbooks/roles/compatibility_layer/vars/2023.06.yml new file mode 100644 index 00000000..ae719b6c --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/vars/2023.06.yml @@ -0,0 +1,99 @@ +# Vars file for the compatibility layer of EESSI version 2023.06 +--- +eessi_version: "2023.06" + +custom_overlays: + - name: eessi + source: git + url: https://github.com/EESSI/gentoo-overlay.git + eclass-overrides: true + +# Select a specific commit in the gentoo_git_repo that should be used for the bootstrap, +# e.g. by checking: https://github.com/gentoo/gentoo/commits/master +# April 17 (29492845e41ea6a0a4a9769c7e0ce287d106079b) commit is after fix for Lmod +# gentoo_git_commit: 29492845e41ea6a0a4a9769c7e0ce287d106079b +# June 8 (aab8473aa90e0287553b3348a5c5b17872df4b7b) commit that was current when fetching luaposix +gentoo_git_commit: aab8473aa90e0287553b3348a5c5b17872df4b7b + +prefix_user_defined_trusted_dirs: + - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib" + +prefix_mask_packages: | + # stick to GCC 10.x; using a too recent compiler in the compat layer complicates stuff in the software layer, + # see for example https://github.com/EESSI/software-layer/issues/151 + >=sys-devel/gcc-11 + # mask OpenSSL 3.x, stick to OpenSSL 1.1.x for now to avoid problems with: + # - older versions of Rust (see https://github.com/EESSI/software-layer/issues/257) + # - older versions of cryptograhy in Python (see https://github.com/EESSI/software-layer/issues/258) + >=dev-libs/openssl-3 + +prefix_unmask_packages: | + # unmask older GCC to make it installable + =sys-devel/gcc-9* + +prefix_bootstrap_use_flags: | + # only build libnss, don't build the daemon (use the one from the host) + sys-auth/nss-pam-ldapd -nslcd + # don't build the SSSD daemon (and man pages) either + sys-auth/sssd -daemon -man + # make sure that gold linker is installed with binutils + sys-devel/binutils gold + # only install Python 3.11 + */* PYTHON_TARGETS: -* python3_11 + */* PYTHON_SINGLE_TARGET: -* python3_11 + +prefix_remove_packages: + - app-eselect/eselect-rust + - dev-lang/go + - dev-lang/go-bootstrap + - dev-lang/rust + - dev-lang/rust-bin + - dev-python/setuptools-rust + - dev-util/cmake + - dev-util/ninja + - virtual/rust + +# List of locations that should get a symlink $EPREFIX/$LOCATION -> $LOCATION. +# This ensures that things like user/group ids are correct/looked up in the right way in the Prefix environment. +symlinks_to_host: + # required to ensure local user groups are known + - /etc/group + + # required to ensure local users are known (see https://github.com/EESSI/compatibility-layer/issues/15) + - /etc/passwd + + # required to ensure that hosts specified in the host's hosts file can be resolved by tools in the Prefix + - /etc/hosts + + # required to ensure name-service information is taken from the right source (e.g. ldap) + # - /etc/nsswitch.conf + + # required to use the DNS resolver from the host (should be done automatically) + - /etc/resolv.conf + + # required to pick up the right timezone from the host + - /etc/localtime + + # required if Centrify is used in nsswitch.conf + - /lib64/libnss_centrifydc.so.2 + + # required if LDAP is used in nsswitch.conf + # - /lib64/libnss_ldap.so.2 + + # required if SSSD is used in nsswitch.conf + # - /lib64/libnss_sss.so.2 + + # required to make runtime data available for lots of tools (including who and w) + # - /var/run + + # required by the last command + # - /var/log/wtmp + +# - /var/lib/munge +# - /var/lib/unbound +# - /var/lib/VirtualGL +# - /var/log/munge +# - /var/spool +# - /var/tmp +# - /run/dbus +# - /tmp diff --git a/ansible/playbooks/roles/compatibility_layer/vars/2025.06.yml b/ansible/playbooks/roles/compatibility_layer/vars/2025.06.yml new file mode 100644 index 00000000..90161701 --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/vars/2025.06.yml @@ -0,0 +1,96 @@ +# Vars file for the compatibility layer of EESSI version 2025.06 +--- +eessi_version: "2025.06" + +custom_overlays: + - name: eessi + source: git + url: https://github.com/EESSI/gentoo-overlay.git + eclass-overrides: true + +# Select a specific commit in the gentoo_git_repo that should be used for the bootstrap, +# e.g. by checking: https://github.com/gentoo/gentoo/commits/master +# June 7 2025 (083e38cef302128d595e9f9cfd029ad8f67ec2b7) commit that includes Lmod 8.7.60 and glibc 2.41 +gentoo_git_commit: 083e38cef302128d595e9f9cfd029ad8f67ec2b7 + +prefix_user_defined_trusted_dirs: + - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/override" + - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/nvidia" + - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/amd" + +prefix_mask_packages: | + # stick to GCC 13.x; using a too recent compiler in the compat layer may complicate stuff in the software layer, + # see for example https://github.com/EESSI/software-layer/issues/151 + >=sys-devel/gcc-14 + +prefix_unmask_packages: | + # if necessary, unmask for instance an older GCC to make it installable + +prefix_bootstrap_use_flags: | + # only build libnss, don't build the daemon (use the one from the host) + sys-auth/nss-pam-ldapd -nslcd + # don't build the SSSD daemon (and man pages) either + sys-auth/sssd -daemon -man + # only install Python 3.13 + */* PYTHON_TARGETS: -* python3_13 + */* PYTHON_SINGLE_TARGET: -* python3_13 + # Modules: disable installation of man pages and shell setup files to be able + # to also install Lmod. enable *conflict unload* and *require via* features. + sys-cluster/modules -shell-setup -man-install +new-features + +prefix_remove_packages: + - app-eselect/eselect-rust + - dev-lang/go + - dev-lang/go-bootstrap + - dev-lang/rust + - dev-lang/rust-bin + - dev-python/setuptools-rust + - dev-util/cmake + - dev-util/hermes + - dev-util/ninja + - virtual/rust + +# List of locations that should get a symlink $EPREFIX/$LOCATION -> $LOCATION. +# This ensures that things like user/group ids are correct/looked up in the right way in the Prefix environment. +symlinks_to_host: + # required to ensure local user groups are known + - /etc/group + + # required to ensure local users are known (see https://github.com/EESSI/compatibility-layer/issues/15) + - /etc/passwd + + # required to ensure that hosts specified in the host's hosts file can be resolved by tools in the Prefix + - /etc/hosts + + # required to ensure name-service information is taken from the right source (e.g. ldap) + # - /etc/nsswitch.conf + + # required to use the DNS resolver from the host (should be done automatically) + - /etc/resolv.conf + + # required to pick up the right timezone from the host + - /etc/localtime + + # required if Centrify is used in nsswitch.conf + - /lib64/libnss_centrifydc.so.2 + + # required if LDAP is used in nsswitch.conf + # - /lib64/libnss_ldap.so.2 + + # required if SSSD is used in nsswitch.conf + # - /lib64/libnss_sss.so.2 + + # required to make runtime data available for lots of tools (including who and w) + # - /var/run + + # required by the last command + # - /var/log/wtmp + +# - /var/lib/munge +# - /var/lib/unbound +# - /var/lib/VirtualGL +# - /var/log/munge +# - /var/spool +# - /var/tmp +# - /run/dbus +# - /tmp diff --git a/ansible/playbooks/roles/compatibility_layer/vars/2026.06.yml b/ansible/playbooks/roles/compatibility_layer/vars/2026.06.yml new file mode 100644 index 00000000..03c808af --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/vars/2026.06.yml @@ -0,0 +1,205 @@ +# Vars file for the compatibility layer of EESSI version 2026.06 +--- +eessi_version: "2026.06" +eessi_host_os: linux +eessi_host_arch: "{{ ansible_architecture }}" + +custom_overlays: + - name: eessi + source: git + url: https://github.com/EESSI/gentoo-overlay.git + eclass-overrides: true + autosync: false + commit: a8932a17c5d3a967a799c4ea105644ec295526e2 + +# Select a specific commit in the gentoo_git_repo that should be used for the bootstrap, +# e.g. by checking: https://github.com/gentoo/gentoo/commits/master +# May 22 2026: commit that includes Lmod 9.2.2 +gentoo_git_commit: 06c43f523fcc82bb634720e25c91cf2cfa3b5a5c + +prefix_user_defined_trusted_dirs: + - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/override" + - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/nvidia" + - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/amd" + +prefix_mask_packages: | + # stick to GCC 14.x; using a too recent compiler in the compat layer may complicate stuff in the software layer, + # see for example https://github.com/EESSI/software-layer/issues/151 + >=sys-devel/gcc-15 + +prefix_unmask_packages: | + # if necessary, unmask for instance an older GCC to make it installable + +prefix_bootstrap_use_flags: | + # only build libnss, don't build the daemon (use the one from the host) + sys-auth/nss-pam-ldapd -nslcd + # don't build the SSSD daemon (and man pages) either + sys-auth/sssd -daemon -man + # only install Python 3.13 + */* PYTHON_TARGETS: -* python3_13 + */* PYTHON_SINGLE_TARGET: -* python3_13 + # Modules: disable installation of man pages and shell setup files to be able + # to also install Lmod. enable *conflict unload* and *require via* features. + sys-cluster/modules -shell-setup -man-install +new-features + +package_sets: [] +prefix_packages: + - name: app-arch/xz-utils + version: 5.8.2 + overlay: eessi + - name: app-misc/elfx86exts + version: 0.6.2 + overlay: eessi + - name: app-shells/bash-completion + - name: dev-python/pip + - name: dev-python/pycodestyle + - name: dev-python/pyyaml + - name: dev-python/rich + - name: dev-util/direnv + version: 2.36.0 + overlay: eessi + - name: dev-util/patchelf + - name: media-fonts/dejavu + - name: media-fonts/liberation-fonts + - name: sys-apps/archspec + version: 0.2.5 + overlay: eessi + - name: sys-apps/dbus + - name: sys-auth/nss-pam-ldapd + version: 0.9.13 + overlay: eessi + - name: sys-auth/munge + version: 0.5.16 + overlay: eessi + - name: sys-auth/sssd + version: 2.11.0 + overlay: eessi + - name: sys-cluster/lmod + - name: sys-cluster/modules + version: 5.6.1 + overlay: eessi + - name: sys-cluster/rdma-core + exclude_on: + - macos-aarch64 + - macos-x86_64 + - name: sys-fabric/opa-psm2 + version: 12.0.1 + overlay: eessi + include_on: + - linux-x86_64 + - name: sys-libs/pam + exclude_on: + - macos-aarch64 + - macos-x86_64 + - name: sys-process/numactl + +prefix_remove_packages: + - app-eselect/eselect-rust + - dev-lang/go + - dev-lang/go-bootstrap + - dev-lang/rust + - dev-lang/rust-bin + - dev-python/setuptools-rust + - dev-util/cmake + - dev-util/hermes + - dev-util/ninja + - virtual/rust + +# List of locations that should get a (variant) symlink $EPREFIX/$LOCATION -> $LOCATION. +# This ensures that things like user/group ids are correct/looked up in the right way in the Prefix environment. +symlinks_to_host: + # required to ensure local user groups are known + - dest: "{{ gentoo_prefix_path }}/etc/ethers" + src: "$(EESSI_COMPAT_ETC_ETHERS):-/etc/ethers" + + - dest: "{{ gentoo_prefix_path }}/etc/fstab" + src: "$(EESSI_COMPAT_ETC_FSTAB):-/etc/fstab" + + - dest: "{{ gentoo_prefix_path }}/etc/group" + src: "$(EESSI_COMPAT_ETC_GROUP):-/etc/group" + + - dest: "{{ gentoo_prefix_path }}/etc/gshadow" + src: "$(EESSI_COMPAT_ETC_GSHADOW):-/etc/gshadow" + + # required to ensure that hosts specified in the host's hosts file can be resolved by tools in the Prefix + - dest: "{{ gentoo_prefix_path }}/etc/hosts" + src: "$(EESSI_COMPAT_ETC_HOSTS):-/etc/hosts" + + - dest: "{{ gentoo_prefix_path }}/etc/hosts.equiv" + src: "$(EESSI_COMPAT_ETC_HOSTS_EQUIV):-/etc/hosts.equiv" + + - dest: "{{ gentoo_prefix_path }}/etc/ld.so.preload" + src: "$(EESSI_COMPAT_ETC_LD_SO_PRELOAD):-/etc/ld.so.preload" + + # required to pick up the right timezone from the host + - dest: "{{ gentoo_prefix_path }}/etc/localtime" + src: "$(EESSI_COMPAT_ETC_LOCALTIME):-/etc/localtime" + + - dest: "{{ gentoo_prefix_path }}/etc/netgroup" + src: "$(EESSI_COMPAT_ETC_NETGROUP):-/etc/netgroup" + + - dest: "{{ gentoo_prefix_path }}/etc/networks" + src: "$(EESSI_COMPAT_ETC_NETWORKS):-/etc/networks" + + # required to ensure name-service information is taken from the right source (e.g. ldap) + - dest: "{{ gentoo_prefix_path }}/etc/nsswitch.conf" + src: "$(EESSI_COMPAT_ETC_NSSWITCH_CONF):-/etc/nsswitch.conf" + + # required to ensure local users are known (see https://github.com/EESSI/compatibility-layer/issues/15) + - dest: "{{ gentoo_prefix_path }}/etc/passwd" + src: "$(EESSI_COMPAT_ETC_PASSWD):-/etc/passwd" + + - dest: "{{ gentoo_prefix_path }}/etc/protocols" + src: "$(EESSI_COMPAT_ETC_PROTOCOLS):-/etc/protocols" + + # required to use the DNS resolver from the host (should be done automatically) + - dest: "{{ gentoo_prefix_path }}/etc/resolv.conf" + src: "$(EESSI_COMPAT_ETC_RESOLV_CONF):-/etc/resolv.conf" + + - dest: "{{ gentoo_prefix_path }}/etc/rpc" + src: "$(EESSI_COMPAT_ETC_RPC):-/etc/rpc" + + - dest: "{{ gentoo_prefix_path }}/etc/services" + src: "$(EESSI_COMPAT_ETC_SERVICES):-/etc/services" + + - dest: "{{ gentoo_prefix_path }}/etc/shadow" + src: "$(EESSI_COMPAT_ETC_SHADOW):-/etc/shadow" + + - dest: "{{ gentoo_prefix_path }}/usr/sbin/sendmail" + src: "$(EESSI_COMPAT_SENDMAIL):-/usr/sbin/sendmail" + + - dest: "{{ gentoo_prefix_path }}/var/db/passwd.db" + src: "$(EESSI_COMPAT_VAR_DB_PASSWD_DB):-/var/db/passwd.db" + + - dest: "{{ gentoo_prefix_path }}/var/log/lastlog" + src: "$(EESSI_COMPAT_VAR_LOG_LASTLOG):-/var/log/lastlog" + + # required by the last command + - dest: "{{ gentoo_prefix_path }}/var/log/wtmp" + src: "$(EESSI_COMPAT_VAR_LOG_WTMP):-/var/log/wtmp" + + - dest: "{{ gentoo_prefix_path }}/var/mail" + src: "$(EESSI_COMPAT_VAR_MAIL):-/var/mail" + + # required to make runtime data available for lots of tools (including who and w) + - dest: "{{ gentoo_prefix_path }}/var/run" + src: "$(EESSI_COMPAT_VAR_RUN):-/var/run" + + - dest: "{{ gentoo_prefix_path }}/var/spool/rwho" + src: "$(EESSI_COMPAT_VAR_SPOOL_RWHO):-/var/spool/rwho" + +# - dest: "{{ gentoo_prefix_path }}/var/tmp" +# src: "$(EESSI_COMPAT_VAR_TMP):-/var/tmp" + + # required if Centrify is used in nsswitch.conf + - dest: "{{ gentoo_prefix_path }}/lib64/libnss_centrifydc.so.2" + src: "$(EESSI_COMPAT_LIBNSS_CENTRIFYDB):-/lib64/libnss_centrifydc.so.2" + +# - /var/lib/munge +# - /var/lib/unbound +# - /var/lib/VirtualGL +# - /var/log/munge +# - /var/spool +# - /var/tmp +# - /run/dbus +# - /tmp diff --git a/bootstrap-prefix.sh b/bootstrap-prefix.sh index 97bc0640..d75b2882 100644 --- a/bootstrap-prefix.sh +++ b/bootstrap-prefix.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash #shellcheck disable=SC1091,SC2015,SC2016,SC2030,SC2031,SC2038,SC2185,SC2120 -# Copyright 2006-2024 Gentoo Authors +# Copyright 2006-2025 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 trap 'exit 1' TERM INT QUIT ABRT @@ -34,6 +34,7 @@ econf() { --sysconfdir="${ROOT}"/tmp/etc \ --localstatedir="${ROOT}"/tmp/var/lib \ --build="${CHOST}" \ + --disable-dependency-tracking \ "$@" || return 1 } @@ -143,11 +144,6 @@ configure_cflags() { export CXX="${CXX-g++} -m64" export HOSTCC="${CC}" ;; - i*86-apple-darwin1*) - export CC="${CC-gcc} -m32" - export CXX="${CXX-g++} -m32" - export HOSTCC="${CC}" - ;; i*86-pc-linux-gnu) if [[ $(${CC} -dumpspecs | grep -A1 multilib_default) != *m32 ]]; then export CC="${CC-gcc} -m32" @@ -164,20 +160,20 @@ configure_toolchain() { linker="sys-devel/binutils" local gcc_deps="dev-libs/gmp dev-libs/mpfr dev-libs/mpc dev-libs/libffi" compiler="${gcc_deps} sys-devel/gcc-config sys-devel/gcc" - compiler_stage1="${gcc_deps} sys-devel/gcc-config" + compiler_stage1="${gcc_deps} sys-devel/gcc-config sys-devel/gcc" compiler_type="gcc" # The host may not have a functioning C++ toolchain, but all # compilers available to us require C++ to build. The last known # version not to require C++ is gcc-4.7. - # We can bootstrap 4.7 in stage1 perhaps if we find envs that do - # not have a functioning C++ toolchain, but for now we assume this - # is not a problem. - # On top of this since gcc-11, C++11 is necessary. This was - # introduced in gcc-4.8, but apparently gcc-5 is still buildable - # with Apple's gcc-apple-4.0.1, so that's a good candidate - # The Prefix tree only contains gcc-12 as of this writing. - # The bootstrap Python 3.7 we have in use requires C11, so Apple's + # We can bootstrap 4.7 in stage1 perhaps if we find envs that do not + # have a functioning C++ toolchain, but for now we assume this is + # not a problem. + # On top of this since gcc-11, C++11 is necessary. gcc-15 requires + # C++14. C++11 support was introduced in gcc-4.8, but apparently + # gcc-5 is still buildable with Apple's gcc-apple-4.0.1, so that's a + # good candidate. See https://gcc.gnu.org/projects/cxx-status.html + # The bootstrap Python 3.11 we have in use requires C11, so Apple's # 4.x line is no longer enough for that. CC=gcc @@ -186,24 +182,14 @@ configure_toolchain() { case ${CHOST}:${DARWIN_USE_GCC} in *darwin*:1) einfo "Triggering Darwin with GCC toolchain" - compiler_stage1+=" sys-apps/darwin-miscutils" - compiler_stage1+=" sys-devel/gcc" - - # binutils-apple/xtools doesn't work (yet) on arm64. The - # profiles will mask and keep using native-cctools for that, - # otherwise stage3 and @system will take care of switching - # to binutils-apple. - # one problem: when we have a really old linker, we need - # to use it sooner or else packages like libffi won't - # compile. - case ${CHOST} in - *-darwin[89]) - linker="=sys-devel/binutils-apple-3.2.6*" - ;; - *) - linker="sys-devel/native-cctools" - ;; - esac + + # In the past we have used binutils-apple/xtools, but that + # doesn't work on arm64 and it isn't updated either. + # Historically (mostly on PPC) we really needed our own + # linker, but that support was dropped. Things like mold or + # sold don't work, so our only option is native-cctools, use + # the linker as provided by the host (Xcode tools). + linker="sys-devel/native-cctools" ;; *-darwin*) local ccvers @@ -211,26 +197,10 @@ configure_toolchain() { einfo "Triggering Darwin with LLVM/Clang toolchain" # for compilers choice, see bug: # https://bugs.gentoo.org/show_bug.cgi?id=538366 - compiler_stage1="sys-apps/darwin-miscutils" - compiler_type="clang" ccvers="$(unset CHOST; ${CC} --version 2>/dev/null)" - llvm_deps="dev-build/ninja" case "${ccvers}" in *"Apple clang version "*|*"Apple LLVM version "*) - # this is Clang, recent enough to compile recent clang - compiler_stage1+=" - ${llvm_deps} - sys-libs/compiler-rt - sys-devel/llvm - sys-devel/lld - sys-devel/clang-common - sys-devel/clang - " - CC=clang - CXX=clang++ - linker= - [[ "${BOOTSTRAP_STAGE}" == stage2 ]] && \ - linker=sys-devel/lld + : # this is Clang, recent enough to compile recent clang ;; *) eerror "unknown/unsupported compiler" @@ -238,6 +208,20 @@ configure_toolchain() { ;; esac + llvm_deps="dev-build/ninja" + compiler_stage1=" + ${llvm_deps} + sys-libs/compiler-rt + sys-devel/llvm + sys-devel/lld + sys-devel/clang-common + sys-devel/clang + " + CC=clang + CXX=clang++ + linker= + [[ "${BOOTSTRAP_STAGE}" == stage2 ]] && \ + linker=sys-devel/lld compiler=" ${llvm_deps} sys-libs/compiler-rt @@ -250,12 +234,8 @@ configure_toolchain() { sys-devel/clang " ;; - *-linux*) - is-rap && einfo "Triggering Linux RAP bootstrap" - compiler_stage1+=" sys-devel/gcc" - ;; *) - compiler_stage1+=" sys-devel/gcc" + is-rap && einfo "Triggering Linux RAP bootstrap" ;; esac @@ -303,24 +283,16 @@ bootstrap_setup() { [[ ${OFFLINE_MODE} ]] && \ echo 'FETCHCOMMAND="bash -c \"echo I need \${FILE} from \${URI} in \${DISTDIR}; read\""' - if [[ ${CHOST} == i*86-apple-darwin9 ]] ; then - # There's no legitimate reason to use 10.5 with x86 (10.6 and - # 10.7 run on every device that ever ran 10.5 x86) but it's - # vastly easier to access and faster than ppc. Don't want to - # burden the tree with this aid-arch, so just use the ppc - # keyword. - echo - echo 'ACCEPT_KEYWORDS="~ppc-macos"' - fi - - if is-rap ; then - # https://bugs.gentoo.org/933100 - # mainline Portage doesn't set these like Prefix branch - # does, so hardwire the IDs here - echo - echo "PORTAGE_INST_UID=$(id --user)" - echo "PORTAGE_INST_GID=$(id --group)" - fi + # https://bugs.gentoo.org/933100 + # Prefix Portage branch sets this in make.globals, mainline + # does not, which breaks RAP. Unconditionally set the vars + # here, as it makes more sense, while we figure out a way to + # migrate Prefix to not pushing this in make.globals. + echo + echo "PORTAGE_USERNAME=\"$(id --name --user)\"" + echo "PORTAGE_GRPNAME=\"$(id --name --group)\"" + echo "PORTAGE_INST_UID=\"$(id --user)\"" + echo "PORTAGE_INST_GID=\"$(id --group)\"" } > "${MAKE_CONF_DIR}/0100_bootstrap_prefix_make.conf" fi @@ -353,6 +325,7 @@ bootstrap_setup() { bootstrap_profile() { local profile local profile_linux + local profile_ver # 2.6.32.1 -> 2*256^3 + 6*256^2 + 32 * 256 + 1 = 33955841 kver() { uname -r|cut -d- -f1|awk -F. '{for (i=1; i<=NF; i++){s+=lshift($i,(4-i)*8)};print s}'; } @@ -370,21 +343,14 @@ bootstrap_profile() { } if is-rap ; then - profile_linux="default/linux/ARCH/17.0/prefix/$(profile-kernel)" + profile_linux="default/linux/ARCH/VER/prefix/$(profile-kernel)" + profile_ver=23.0 else profile_linux="prefix/linux/ARCH" fi case ${CHOST} in - powerpc-apple-darwin9) - rev=${CHOST##*darwin} - profile="prefix/darwin/macos/10.$((rev - 4))/ppc" - ;; - i*86-apple-darwin9) - rev=${CHOST##*darwin} - profile="prefix/darwin/macos/10.$((rev - 4))/x86" - ;; - i*86-apple-darwin1[578]) + i*86-apple-darwin*|powerpc-apple-darwin*) eerror "REMOVED ARCH: this 32-bit MacOS architecture was removed," eerror "bootstrapping is impossible" exit 1 @@ -393,11 +359,12 @@ bootstrap_profile() { rev=${CHOST##*darwin} profile="prefix/darwin/macos/10.$((rev - 4))/x64" ;; - *64-apple-darwin2[0123456789]) + *64-apple-darwin2[01234]) # Big Sur is 11.0 darwin20 # Monterey is 12.0 darwin21 # Ventura is 13.0 darwin22 # Sanoma is 14.0 darwin23 + # Sequoia is 15.0 darwin24 rev=${CHOST##*darwin} case ${CHOST%%-*} in x86_64) arch=x64 ;; @@ -406,36 +373,52 @@ bootstrap_profile() { esac profile="prefix/darwin/macos/$((rev - 9)).0/${arch}" ;; + *64-apple-darwin2[56789]) + # Tahoe is 26.0 darwin25 + rev=${CHOST##*darwin} + case ${CHOST%%-*} in + x86_64) arch=x64 ;; + arm64) arch=arm64 ;; + *) arch=error ;; + esac + profile="prefix/darwin/macos/$((rev + 1)).0/${arch}" + ;; i*86-pc-linux-gnu) profile=${profile_linux/ARCH/x86} - ;; - riscv64-*-linux-gnu) - profile=${profile_linux/ARCH/riscv} - profile=${profile/17.0/20.0/rv64gc/lp64d} + profile=${profile/VER/${profile_ver}/i686} ;; x86_64-pc-linux-gnu) profile=${profile_linux/ARCH/amd64} - profile=${profile/17.0/17.1/no-multilib} + profile=${profile/VER/${profile_ver}/no-multilib} ;; powerpc-unknown-linux-gnu) + # no RAP support profile=${profile_linux/ARCH/ppc} + profile=${profile/VER/${profile_ver}} ;; powerpc64-unknown-linux-gnu) + # no RAP support profile=${profile_linux/ARCH/ppc64} + profile=${profile/VER/${profile_ver}} ;; powerpc64le-unknown-linux-gnu) profile=${profile_linux/ARCH/ppc64le} + profile=${profile/VER/${profile_ver}} ;; + riscv64-*-linux-gnu|\ riscv-pc-unknown-linux-gnu) profile=${profile_linux/ARCH/riscv} - profile=${profile/17.0/20.0/rv64gc/lp64d} + profile=${profile/VER/${profile_ver}/rv64/lp64d} ;; aarch64-unknown-linux-gnu) profile=${profile_linux/ARCH/arm64} + profile=${profile/VER/${profile_ver}} ;; armv7*-unknown-linux-gnueabi*) profile=${profile_linux/ARCH/arm} - profile=${profile/17.0/17.0/armv7a} + [[ ${CHOST} == *gnueabihf ]] \ + && profile=${profile/VER/${profile_ver}/armv7a_hf} \ + || profile=${profile/VER/${profile_ver}/armv7a_sf} ;; x86_64-pc-solaris2.11) profile="prefix/sunos/solaris/5.11/x64" @@ -536,7 +519,7 @@ bootstrap_tree() { # retain this comment and the line below to # keep this snapshot around in the snapshots # MKSNAPSHOT-ANCHOR -- directory of rsync slaves - local PV="20250525" + local PV="20251216" # RAP uses the latest gentoo main repo snapshot to bootstrap. is-rap && LATEST_TREE_YES=1 @@ -598,7 +581,8 @@ bootstrap_startscript() { prepare_portage() { # see bootstrap_portage for explanations. - mkdir -p "${ROOT}"/bin/. "${ROOT}"/var/log + mkdir -p "${ROOT}"/usr/bin/. "${ROOT}"/var/log + [[ -x ${ROOT}/bin ]] || ln -s usr/bin "${ROOT}"/bin [[ -x ${ROOT}/bin/bash ]] || ln -s "${ROOT}"{/tmp,}/bin/bash || return 1 [[ -x ${ROOT}/bin/sh ]] || ln -s bash "${ROOT}"/bin/sh || return 1 } @@ -608,8 +592,8 @@ bootstrap_portage() { # STABLE_PV that is known to work. Intended for power users only. ## It is critical that STABLE_PV is the lastest (non-masked) version that is ## included in the snapshot for bootstrap_tree. - STABLE_PV="3.0.56.1" - [[ ${TESTING_PV} == latest ]] && TESTING_PV="3.0.56.1" + STABLE_PV="3.0.72" + [[ ${TESTING_PV} == latest ]] && TESTING_PV="3.0.72" PV="${TESTING_PV:-${STABLE_PV}}" A=prefix-portage-${PV}.tar.bz2 einfo "Bootstrapping ${A%.tar.*}" @@ -634,9 +618,10 @@ bootstrap_portage() { -i lib/_emerge/AbstractEbuildProcess.py || \ return 1 - # host-provided wget may lack certificates, stage1 wget is without ssl + # host may lack (sufficient) certificates [[ $(wget -h) == *"--no-check-certificate"* ]] && - sed -e '/wget/s/ --passive-ftp /&--no-check-certificate /' -i cnf/make.globals + sed -e '/wget/s/ --passive-ftp /&--no-check-certificate /' \ + -i cnf/make.globals # Portage checks for valid shebangs. These may (xz-utils) originate # in CONFIG_SHELL (AIX), which originates in PORTAGE_BASH then. @@ -653,7 +638,6 @@ bootstrap_portage() { --with-offset-prefix="${ROOT}"/tmp \ --with-portage-user="$(id -un)" \ --with-portage-group="$(id -gn)" \ - --with-extra-path="${PATH}" \ || return 1 emake || return 1 @@ -667,8 +651,12 @@ bootstrap_portage() { # As such, portage complains.. mkdir -p "${ROOT}"/tmp/var/log - # in Prefix the sed wrapper is deadly, so kill it - rm -f "${ROOT}"/tmp/usr/lib/portage/bin/ebuild-helpers/sed + # phase-helpers.sh now uses gtar instead of tar, so ensure we have + # it available + if [[ -x "${ROOT}"/tmp/bin/tar ]] ; then + rm -f "${ROOT}"/tmp/bin/gtar + ( cd "${ROOT}"/tmp/bin ; ln -s tar gtar ) + fi local tmpportdir=${ROOT}/tmp/${PORTDIR#"${ROOT}"} [[ -e "${tmpportdir}" ]] || ln -s "${PORTDIR}" "${tmpportdir}" @@ -835,6 +823,13 @@ bootstrap_gnu() { fi local -a myconf + + # no point in doing NLS at this stage + myconf+=( --disable-nls ) + # if year 2038 is a problem, don't error out on it, we're not there + # yet, and this is only temporary, just for now + myconf+=( --disable-year2038 ) + if [[ ${PN}-${PV} == "make-4.2.1" ]] ; then if [[ ${CHOST} == *-linux-gnu* ]] ; then # force this, macros aren't set correctly with newer glibc @@ -862,40 +857,30 @@ bootstrap_gnu() { fix_config_sub if [[ ${PN} == "grep" ]] ; then - # Solaris and OSX don't like it when --disable-nls is set, - # so just don't set it at all. # Solaris 11 has a messed up prce installation. We don't need # it anyway, so just disable it myconf+=( "--disable-perl-regexp" ) fi - if [[ ${PN} == "mpfr" || ${PN} == "mpc" || ${PN} == "gcc" ]] ; then - [[ -e "${ROOT}"/tmp/usr/include/gmp.h ]] \ - && myconf+=( "--with-gmp=${ROOT}/tmp/usr" ) - fi - if [[ ${PN} == "mpc" || ${PN} == "gcc" ]] ; then - [[ -e "${ROOT}"/tmp/usr/include/mpfr.h ]] \ - && myconf+=( "--with-mpfr=${ROOT}/tmp/usr" ) - fi if [[ ${PN} == "gcc" ]] ; then - [[ -e "${ROOT}"/tmp/usr/include/mpc.h ]] \ - && myconf+=( "--with-mpc=${ROOT}/tmp/usr" ) - myconf+=( "--enable-languages=c,c++" "--disable-bootstrap" "--disable-multilib" - "--disable-nls" "--disable-libsanitizer" + "--with-local-prefix=${ROOT}/tmp/usr" + "--with-ld=${ROOT}/tmp/usr/bin/ldwrapper" + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55930 + "--enable-dependency-tracking" ) if [[ ${CHOST} == *-darwin* ]] ; then myconf+=( "--with-native-system-header-dir=${ROOT}/MacOSX.sdk/usr/include" - "--with-ld=${ROOT}/tmp/usr/bin/ldwrapper" ) fi + # keep this mild soas not to trigger any odd bugs/problems export CFLAGS="-O1 -pipe" export CXXFLAGS="-O1 -pipe" fi @@ -935,15 +920,21 @@ bootstrap_gnu() { if [[ -x ${ROOT}/tmp/usr/bin/openssl ]] ; then myconf+=( "-with-ssl=openssl" - "--with-libssl-prefix=${ROOT}/tmp/usr" ) - export CPPFLAGS="${CPPFLAGS} -I${ROOT}/tmp/usr/include" - export LDFLAGS="${LDFLAGS} -L${ROOT}/tmp/usr/lib" + # avoid pkg-config call + export OPENSSL_CFLAGS="-I${ROOT}/tmp" + export OPENSSL_LIBS="-lssl -lcrypto" else myconf+=( "--without-ssl" ) fi fi + if [[ ${PN} == "tar" ]] ; then + # tar really insists on doing iconv stuff, even when it doesn't + # work (like on macOS), so force it off + sed -i -e '/HAVE_ICONV/d' config.h.in || die + fi + # SuSE 11.1 has GNU binutils-2.20, choking on crc32_x86 [[ ${PN} == "xz" ]] && myconf+=( "--disable-assembler" ) @@ -991,14 +982,19 @@ bootstrap_gnu() { } python_ver() { - # keep this number in line with PV below for stage1,2 - # also, note that this version must match the Python version in the - # snapshot for stage3, else packages will break with some python - # mismatch error due to Portage using a different version after it - # upgraded itself with a newer Python - echo 3.11 - export PYTHON_FULL_VERSION="3.11.7-gentoo-prefix-patched" - # keep this number in line with PV below for stage1,2 + # stage1 and 2 should call python_ver if they need the version, such + # that we can change it in a single place (here) + local PYVER=3.11 + echo ${PYVER} + export PYTHON_FULL_VERSION="${PYVER}.7-gentoo-prefix-patched" + + # note on upgrading Python versions + # At least Python 3.13 requires pkg-config and a lot of external + # dependencies which makes bootstrapping more complicated. For that + # reason, the Python version is now locked to 3.11, bug #966544 + # This ultimately means that the bootstrap Python here, and the + # Portage snapshot we use should work together, and that we must + # update Python versions from Portage. } bootstrap_python() { @@ -1045,17 +1041,6 @@ bootstrap_python() { # fix result export LIBS="${LIBS} -lresolv" ;; - (*-darwin9) - # Darwin 9's kqueue seems to act up (at least at this stage), so - # make Python's selectors resort to poll() or select() for the - # time being - sed -i \ - -e 's/kqueue/kqueue_DISABLED/' \ - configure - # fixup thread id detection (only needed on vanilla Python tar) - efetch "https://dev.gentoo.org/~sam/distfiles/dev-lang/python/python-3.9.6-darwin9_pthreadid.patch" - patch -p1 < "${DISTDIR}"/python-3.9.6-darwin9_pthreadid.patch - ;; (*-openbsd*) # OpenBSD is not a multilib system sed -i \ @@ -1102,28 +1087,6 @@ bootstrap_python() { ;; esac - # python refuses to find the zlib headers that are built in the offset, - # same for libffi, which installs into compiler's multilib-osdir - export CPPFLAGS="-I${ROOT}/tmp/usr/include" - export LDFLAGS="${CFLAGS} -L${ROOT}/tmp/usr/lib" - # set correct flags for runtime for ELF platforms - case ${CHOST} in - *-linux*) - # GNU ld - LDFLAGS="${LDFLAGS} -Wl,-rpath,${ROOT}/tmp/usr/lib ${libdir}" - LDFLAGS="${LDFLAGS} -Wl,-rpath,${libdir#-L}" - ;; - *-openbsd*) - # LLD - LDFLAGS="${LDFLAGS} -Wl,-rpath,${ROOT}/tmp/usr/lib" - ;; - *-solaris*) - # Sun ld - LDFLAGS="${LDFLAGS} -R${ROOT}/tmp/usr/lib ${libdir}" - LDFLAGS="${LDFLAGS} -R${libdir#-L}" - ;; - esac - # if the user has a $HOME/.pydistutils.cfg file, the python # installation is going to be screwed up, as reported by users, so # just make sure Python won't find it @@ -1272,7 +1235,7 @@ bootstrap_zlib_core() { } bootstrap_zlib() { - bootstrap_zlib_core 1.2.11 || \ + bootstrap_zlib_core 1.3.1 || bootstrap_zlib_core 1.2.11 || \ bootstrap_zlib_core 1.2.8 || bootstrap_zlib_core 1.2.7 || \ bootstrap_zlib_core 1.2.6 || bootstrap_zlib_core 1.2.5 } @@ -1286,14 +1249,17 @@ bootstrap_libffi() { } bootstrap_gmp() { + bootstrap_gnu gmp 6.3.0 || \ bootstrap_gnu gmp 6.2.1 } bootstrap_mpfr() { + bootstrap_gnu mpfr 4.2.1 || \ bootstrap_gnu mpfr 4.1.0 } bootstrap_mpc() { + bootstrap_gnu mpc 1.3.1 || \ bootstrap_gnu mpc 1.2.1 } @@ -1311,22 +1277,42 @@ bootstrap_ldwrapper() { cp "${DISTDIR}/${A}" . || return 1 einfo "Compiling ${A%.c}" - ${CC:-gcc} \ + v ${CC:-gcc} \ -o ldwrapper \ -DCHOST="\"${CHOST}\"" \ - -DEPREFIX="\"${ROOT}\"" \ + -DEPREFIX="\"${ROOT}/tmp\"" \ ldwrapper.c || return 1 einfo "Installing ${A%.c}" mkdir -p "${ROOT}"/tmp/usr/bin - cp -a ldwrapper "${ROOT}"/tmp/usr/bin/ || return 1 + cp -p ldwrapper "${ROOT}"/tmp/usr/bin/ || return 1 einfo "${A%.c} bootstrapped" } bootstrap_gcc5() { + # GCC doesn't respect CPPFLAGS because of its own meddling/cleansing + # so provide a wrapper here to force mpfr, mpc + # installed packages to be found + mkdir -p "${ROOT}"/tmp/usr/local/bin + rm -f "${ROOT}"/tmp/usr/local/bin/my{gcc,g++} + cat > "${ROOT}/tmp/usr/local/bin/mygcc" <<-EOS + #!/usr/bin/env sh + exec ${CC} "\$@" ${CPPFLAGS} + EOS + cat > "${ROOT}/tmp/usr/local/bin/myg++" <<-EOS + #!/usr/bin/env sh + exec ${CXX} "\$@" ${CPPFLAGS} + EOS + chmod 755 "${ROOT}/tmp/usr/local/bin/my"{gcc,g++} + export CC="${ROOT}"/tmp/usr/local/bin/mygcc + export CXX="${ROOT}"/tmp/usr/local/bin/myg++ + # bootstraps with gcc-4.0.1 (Darwin 8), provides C11 bootstrap_gnu gcc 5.5.0 + + # ensure the wrappers referring to the host provided compiler are gone + rm -f "${ROOT}"/tmp/usr/local/bin/my{gcc,g++} } bootstrap_sed() { @@ -1335,13 +1321,13 @@ bootstrap_sed() { } bootstrap_findutils() { - bootstrap_gnu findutils 4.9.0 || - bootstrap_gnu findutils 4.7.0 || - bootstrap_gnu findutils 4.5.10 || - bootstrap_gnu findutils 4.2.33 + # portage 3.0.72 requires --files-from0, which is in 4.9 and up + bootstrap_gnu findutils 4.10.0 || + bootstrap_gnu findutils 4.9.0 } bootstrap_wget() { + bootstrap_gnu wget 1.25.0 || \ bootstrap_gnu wget 1.20.1 || \ bootstrap_gnu wget 1.17.1 || bootstrap_gnu wget 1.13.4 } @@ -1350,7 +1336,7 @@ bootstrap_grep() { # don't use 2.13, it contains a bug that bites, bug #425668 # 2.9 is the last version provided as tar.gz (platforms without xz) # 2.7 is necessary for Solaris/OpenIndiana (2.8, 2.9 fail to configure) - bootstrap_gnu grep 3.3 || \ + bootstrap_gnu grep 3.12 || bootstrap_gnu grep 3.3 || \ bootstrap_gnu grep 2.9 || bootstrap_gnu grep 2.7 || \ bootstrap_gnu grep 2.14 || bootstrap_gnu grep 2.12 } @@ -1359,13 +1345,15 @@ bootstrap_coreutils() { # 8.16 is the last version released as tar.gz # 8.18 is necessary for macOS High Sierra (darwin17) and converted # to tar.gz for this case - bootstrap_gnu coreutils 9.5 || \ + bootstrap_gnu coreutils 9.8 || bootstrap_gnu coreutils 9.5 || \ bootstrap_gnu coreutils 8.32 || bootstrap_gnu coreutils 8.30 || \ bootstrap_gnu coreutils 8.16 || bootstrap_gnu coreutils 8.17 } bootstrap_tar() { - bootstrap_gnu tar 1.32 || bootstrap_gnu tar 1.26 + bootstrap_gnu tar 1.35 || bootstrap_gnu tar 1.32 + # tar <=1.26 handles -I "bzip2 -c" wrongly, which is used by Portage + # nowadays (3.0.72) } bootstrap_make() { @@ -1379,6 +1367,7 @@ bootstrap_make() { bootstrap_patch() { # 2.5.9 needed for OSX 10.6.x still? + bootstrap_gnu patch 2.8 || bootstrap_gnu patch 2.7.5 || bootstrap_gnu patch 2.7.4 || bootstrap_gnu patch 2.7.3 || @@ -1386,8 +1375,10 @@ bootstrap_patch() { } bootstrap_gawk() { - bootstrap_gnu gawk 5.0.1 || bootstrap_gnu gawk 4.0.1 || \ - bootstrap_gnu gawk 3.1.8 + bootstrap_gnu gawk 5.3.2 || + bootstrap_gnu gawk 5.0.1 || + bootstrap_gnu gawk 4.0.1 || + bootstrap_gnu gawk 3.1.8 } bootstrap_binutils() { @@ -1399,6 +1390,7 @@ bootstrap_texinfo() { } bootstrap_bash() { + bootstrap_gnu bash 5.3 || bootstrap_gnu bash 5.2 || bootstrap_gnu bash 5.1 || bootstrap_gnu bash 5.0 @@ -1412,15 +1404,18 @@ bootstrap_bison() { } bootstrap_m4() { + bootstrap_gnu m4 1.4.20 || \ bootstrap_gnu m4 1.4.19 || \ bootstrap_gnu m4 1.4.18 # version is patched, so beware } bootstrap_gzip() { + bootstrap_gnu gzip 1.14 || bootstrap_gnu gzip 1.4 } bootstrap_xz() { + GNU_URL=http://tukaani.org/xz bootstrap_gnu xz 5.4.5 || \ GNU_URL=http://tukaani.org/xz bootstrap_gnu xz 5.2.4 || \ GNU_URL=http://tukaani.org/xz bootstrap_gnu xz 5.2.3 @@ -1488,6 +1483,7 @@ bootstrap_stage1() { done BOOTSTRAP_STAGE="stage1" configure_toolchain || return 1 + configure_cflags || return 1 export CC CXX # default: empty = NO @@ -1539,72 +1535,34 @@ bootstrap_stage1() { ( cd "${ROOT}" && ln -s "${SDKPATH}" MacOSX.sdk ) einfo "using system sources from ${SDKPATH}" - # GCC 14 cannot be compiled by versions of Clang at least on + # benefit from 4.2 if it's present (it only will be if the + # default is 4.0.1) + if [[ -e /usr/bin/gcc-4.2 ]] ; then + export CC=gcc-4.2 + export CXX=g++-4.2 + fi + + # GCC >=14 cannot be compiled by versions of Clang at least on # Darwin17, so go the safe route and get GCC-5 which is sufficient - # and the last one we can compile without C11. This also compiles - # on Darwin 8 and 9. + # and the last one we can compile without C11. # see also configure_toolchain case ${CHOST} in *-darwin2[23456789]) : ;; # host toolchain can compile gcc-14 - *-darwin[89]) USEGCC5=yes ;; *86*-darwin*) USEGCC5=yes ;; # arm64/M1 isn't supported by old GCC-5! esac fi - if [[ -n ${USEGCC5} ]] ; then - # benefit from 4.2 if it's present - if [[ -e /usr/bin/gcc-4.2 ]] ; then - export CC=gcc-4.2 - export CXX=g++-4.2 - fi - - [[ -e ${ROOT}/tmp/usr/include/gmp.h ]] \ - || (bootstrap_gmp) || return 1 - [[ -e ${ROOT}/tmp/usr/include/mpfr.h ]] \ - || (bootstrap_mpfr) || return 1 - [[ -e ${ROOT}/tmp/usr/include/mpc.h ]] \ - || (bootstrap_mpc) || return 1 - [[ -x ${ROOT}/tmp/usr/bin/ldwrapper ]] \ - || (bootstrap_ldwrapper) || return 1 - # get ldwrapper target in PATH - export BINUTILS_CONFIG_LD="$(type -P ld)" - # force deployment target in GCCs build, GCC-5 doesn't quite get - # the newer macOS versions (20+) and thus confuses ld when it - # passes on the deployment version. Use High Sierra as it has - # everything we need - [[ ${CHOST##*darwin} -gt 10 ]] && export MACOSX_DEPLOYMENT_TARGET=10.13 - [[ -x ${ROOT}/tmp/usr/bin/gcc ]] \ - || (bootstrap_gcc5) || return 1 - - if [[ ${CHOST##*darwin} -gt 10 ]] ; then - # install wrappers in tmp/usr/local/bin which comes before - # /tmp/usr/bin in PATH - mkdir -p "${ROOT}"/tmp/usr/local/bin - rm -f "${ROOT}"/tmp/usr/local/bin/{gcc,${CHOST}-gcc} - cat > "${ROOT}/tmp/usr/local/bin/${CHOST}-gcc" <<-EOS - #!/usr/bin/env sh - export MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} - export BINUTILS_CONFIG_LD="$(type -P ld)" - exec "${ROOT}"/tmp/usr/bin/${CHOST}-gcc "\$@" - EOS - chmod 755 "${ROOT}/tmp/usr/local/bin/${CHOST}-gcc" - ln -s ${CHOST}-gcc "${ROOT}"/tmp/usr/local/bin/gcc - - rm -f "${ROOT}"/tmp/usr/local/bin/{g++,${CHOST}-g++} - cat > "${ROOT}"/tmp/usr/local/bin/${CHOST}-g++ <<-EOS - #!/usr/bin/env sh - export MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} - export BINUTILS_CONFIG_LD="$(type -P ld)" - exec "${ROOT}"/tmp/usr/bin/${CHOST}-g++ "\$@" - EOS - chmod 755 "${ROOT}"/tmp/usr/local/bin/${CHOST}-g++ - ln -s ${CHOST}-g++ "${ROOT}"/tmp/usr/local/bin/g++ - fi - - # reset after gcc-4.2 usage - export CC=gcc - export CXX=g++ + # For GCC-14 and above which are in the snapshots at this time, we + # need C++14 and C11, which GCC-5 provides + if [[ ${USEGCC5} != yes ]] ; then + # both GCC and Clang accept -dumpversion, so if it doesn't work, + # we can assume this isn't sufficient + ccver=$(${CC} -dumpversion 2>/dev/null) + # we assume getting something like 4.8.3 or 17, since we need + # just the major version, it's simple processing here + ccver=${ccver%%.*} + [[ ${ccver:-0} -lt 5 ]] && USEGCC5=yes fi # Some host tools need to be wrapped to be useful for us. @@ -1625,39 +1583,6 @@ bootstrap_stage1() { cp "${ROOT}"/tmp/usr/local/bin/java{,c} chmod 755 "${ROOT}"/tmp/usr/local/bin/java{,c} ;; - *-linux*) - if [[ ! -x "${ROOT}"/tmp/usr/bin/gcc ]] \ - && [[ $(gcc -print-prog-name=as),$(gcc -print-prog-name=ld) != /*,/* ]] - then - # RHEL's system gcc is set up to use binutils via PATH search. - # If the version of our binutils an older one, they may not - # provide what the system gcc is configured to use. - # We need to direct the system gcc to find the system binutils. - EXEC="$(PATH="${ORIGINAL_PATH}" type -P gcc)" - if [[ -z ${EXEC} ]] ; then - eerror "could not find 'gcc' in your PATH!" - eerror "please install gcc or provide access via PATH or symlink" - return 1 - fi - cat >> "${ROOT}"/tmp/usr/local/bin/gcc <<-EOF - #! /bin/sh - PATH="${ORIGINAL_PATH}" export PATH - exec "${EXEC}" "\$@" - EOF - EXEC="$(PATH="${ORIGINAL_PATH}" type -P g++)" - if [[ -z ${EXEC} ]] ; then - eerror "could not find 'g++' in your PATH!" - eerror "please install g++ or provide access via PATH or symlink" - return 1 - fi - cat >> "${ROOT}"/tmp/usr/local/bin/g++ <<-EOF - #! /bin/sh - PATH="${ORIGINAL_PATH}" export PATH - exec "${EXEC}" "\$@" - EOF - chmod 755 "${ROOT}"/tmp/usr/local/bin/g{cc,++} - fi - ;; esac # Host compiler can output a variety of libdirs. At stage1, @@ -1695,8 +1620,9 @@ bootstrap_stage1() { || (bootstrap_make) || return 1 [[ ${OFFLINE_MODE} ]] || [[ -x ${ROOT}/tmp/usr/bin/openssl ]] \ || (bootstrap_libressl) # try without on failure - [[ ${OFFLINE_MODE} ]] || type -P wget > /dev/null \ - || (bootstrap_wget) || return 1 + [[ ${OFFLINE_MODE} ]] \ + || [[ $(wget --version 2>&1) == *GNU" Wget "1.* ]] \ + || (bootstrap_wget) || return 1 # need wget-1, bug #953843 [[ -x ${ROOT}/tmp/usr/bin/sed ]] \ || [[ $(sed --version 2>&1) == *GNU* ]] \ || (bootstrap_sed) || return 1 @@ -1706,7 +1632,7 @@ bootstrap_stage1() { || [[ $(patch --version 2>&1) == *"patch 2."[6-9]*GNU* ]] \ || (bootstrap_patch) || return 1 [[ -x ${ROOT}/tmp/usr/bin/m4 ]] \ - || [[ $(m4 --version 2>&1) == *GNU*1.4.1?* ]] \ + || [[ $(m4 --version 2>&1) == *"GNU M4) "1.4.[1234]* ]] \ || (bootstrap_m4) || return 1 [[ -x ${ROOT}/tmp/usr/bin/bison ]] \ || [[ $(bison --version 2>&1) == *"GNU Bison) "2.[3-7]* ]] \ @@ -1718,12 +1644,12 @@ bootstrap_stage1() { [[ $(uniq --version 2>&1) == *"(GNU coreutils) "[6789]* ]] \ || (bootstrap_coreutils) || return 1 fi - [[ -x ${ROOT}/tmp/usr/bin/find ]] \ - || [[ $(find --version 2>&1) == *GNU* ]] \ + || [[ $(find --version 2>&1) == *"GNU 4.9"* ]] \ + || [[ $(find --version 2>&1) == *"GNU 4."[12][012346789]* ]] \ || (bootstrap_findutils) || return 1 [[ -x ${ROOT}/tmp/usr/bin/tar ]] \ - || [[ $(tar --version 2>&1) == *GNU* ]] \ + || [[ $(tar --version 2>&1) == *"GNU 1."[3456789]* ]] \ || (bootstrap_tar) || return 1 [[ -x ${ROOT}/tmp/usr/bin/grep ]] \ || [[ $(grep --version 2>&1) == *GNU* ]] \ @@ -1759,6 +1685,93 @@ bootstrap_stage1() { || [[ ${DARWIN_USE_GCC} == 1 ]] \ || (bootstrap_cmake) || return 1 + # get a sufficient compiler if we have to + if [[ -n ${USEGCC5} ]] ; then + [[ -e ${ROOT}/tmp/usr/include/gmp.h ]] \ + || (bootstrap_gmp) || return 1 + [[ -e ${ROOT}/tmp/usr/include/mpfr.h ]] \ + || (bootstrap_mpfr) || return 1 + [[ -e ${ROOT}/tmp/usr/include/mpc.h ]] \ + || (bootstrap_mpc) || return 1 + [[ -x ${ROOT}/tmp/usr/bin/ldwrapper ]] \ + || (bootstrap_ldwrapper) || return 1 + # get ldwrapper target in PATH + BINUTILS_CONFIG_LD="$(type -P ld)" + export BINUTILS_CONFIG_LD + # force deployment target in GCCs build, GCC-5 doesn't quite get + # the newer macOS versions (20+) and thus confuses ld when it + # passes on the deployment version. Use High Sierra as it has + # everything we need + [[ ${CHOST##*darwin} -gt 10 ]] && export MACOSX_DEPLOYMENT_TARGET=10.13 + [[ -x ${ROOT}/tmp/usr/bin/gcc ]] \ + || (bootstrap_gcc5) || return 1 + + if [[ ${CHOST##*darwin} -gt 10 ]] ; then + # install wrappers in tmp/usr/local/bin which comes before + # /tmp/usr/bin in PATH + mkdir -p "${ROOT}"/tmp/usr/local/bin + rm -f "${ROOT}"/tmp/usr/local/bin/{gcc,"${CHOST}"-gcc} + cat > "${ROOT}/tmp/usr/local/bin/${CHOST}-gcc" <<-EOS + #!/usr/bin/env sh + export MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} + export BINUTILS_CONFIG_LD="$(type -P ld)" + exec "${ROOT}"/tmp/usr/bin/${CHOST}-gcc "\$@" + EOS + chmod 755 "${ROOT}/tmp/usr/local/bin/${CHOST}-gcc" + ln -s "${CHOST}"-gcc "${ROOT}"/tmp/usr/local/bin/gcc + + rm -f "${ROOT}"/tmp/usr/local/bin/{g++,"${CHOST}"-g++} + cat > "${ROOT}"/tmp/usr/local/bin/"${CHOST}"-g++ <<-EOS + #!/usr/bin/env sh + export MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} + export BINUTILS_CONFIG_LD="$(type -P ld)" + exec "${ROOT}"/tmp/usr/bin/${CHOST}-g++ "\$@" + EOS + chmod 755 "${ROOT}"/tmp/usr/local/bin/"${CHOST}"-g++ + ln -s "${CHOST}"-g++ "${ROOT}"/tmp/usr/local/bin/g++ + fi + + # make sure we use the now bootstrapped compiler + export CC=gcc + export CXX=g++ + fi + + case ${CHOST} in + *-linux*) + if [[ ! -x "${ROOT}"/tmp/usr/bin/gcc ]] \ + && [[ $(gcc -print-prog-name=as),$(gcc -print-prog-name=ld) != /*,/* ]] + then + # RHEL's system gcc is set up to use binutils via PATH search. + # If the version of our binutils an older one, they may not + # provide what the system gcc is configured to use. + # We need to direct the system gcc to find the system binutils. + EXEC="$(PATH="${ORIGINAL_PATH}" type -P gcc)" + if [[ -z ${EXEC} ]] ; then + eerror "could not find 'gcc' in your PATH!" + eerror "please install gcc or provide access via PATH or symlink" + return 1 + fi + cat >> "${ROOT}"/tmp/usr/local/bin/gcc <<-EOF + #! /bin/sh + PATH="${ORIGINAL_PATH}" export PATH + exec "${EXEC}" "\$@" + EOF + EXEC="$(PATH="${ORIGINAL_PATH}" type -P g++)" + if [[ -z ${EXEC} ]] ; then + eerror "could not find 'g++' in your PATH!" + eerror "please install gcc-c++ or provide access via PATH or symlink" + return 1 + fi + cat >> "${ROOT}"/tmp/usr/local/bin/g++ <<-EOF + #! /bin/sh + PATH="${ORIGINAL_PATH}" export PATH + exec "${EXEC}" "\$@" + EOF + chmod 755 "${ROOT}"/tmp/usr/local/bin/g{cc,++} + fi + ;; + esac + # get ebuilds and support files in place (bootstrap_tree) || return 1 @@ -1845,6 +1858,7 @@ do_emerge_pkgs() { "-fortran" # gcc "-gdbm" "-nls" + "-openssl" # coreutils "-pcre" "-python" "-qmanifest" # portage-utils @@ -1953,6 +1967,7 @@ bootstrap_stage2() { BOOTSTRAP_STAGE="stage2" configure_toolchain || return 1 configure_cflags || return 1 export CONFIG_SHELL="${ROOT}"/tmp/bin/bash + export BINUTILS_CONFIG_LD="$(type -P ld)" # in case of bootstrapped GCC export CC CXX emerge_pkgs() { @@ -2085,9 +2100,17 @@ bootstrap_stage2() { echo "sys-devel/gcc -pie" >> "${ROOT}"/tmp/etc/portage/package.use fi - # don't use CET, we don't know if the host compiler supports it - echo "sys-devel/binutils -cet" >> \ - "${ROOT}"/tmp/etc/portage/profile/package.use.force + # don't use zstd at this point, the host may not have it installed + # don't use CET with binutils, we don't know if the host compiler + # supports it #936629 + { + echo "sys-devel/binutils -cet -zstd" + echo "sys-devel/gcc -zstd" + } >> "${ROOT}"/tmp/etc/portage/profile/package.use.force + { + echo "sys-devel/binutils cet zstd" + echo "sys-devel/gcc zstd" + } >> "${ROOT}"/tmp/etc/portage/profile/package.use.mask emerge_pkgs --nodeps "${pkgs[@]}" || return 1 @@ -2097,6 +2120,9 @@ bootstrap_stage2() { emerge_pkgs --nodeps "${pkg}" || return 1 done + # we got a linker now + unset BINUTILS_CONFIG_LD + # GCC doesn't respect CPPFLAGS because of its own meddling as well # as toolchain.eclass, so provide a wrapper here to force just # installed packages to be found @@ -2126,14 +2152,6 @@ bootstrap_stage2() { CC=mygcc CXX=myg++ \ PYTHON_COMPAT_OVERRIDE=python$(python_ver) \ emerge_pkgs --nodeps "${pkg}" || return 1 - - if [[ "${pkg}" == *sys-devel/llvm* || ${pkg} == *sys-devel/clang* ]] ; - then - # we need llvm/clang ASAP for libcxx* doesn't build - # without C++11 - [[ -x ${ROOT}/tmp/usr/bin/clang ]] && CC=clang - [[ -x ${ROOT}/tmp/usr/bin/clang++ ]] && CXX=clang++ - fi done if [[ ${compiler_type} == clang ]] ; then @@ -2214,7 +2232,7 @@ bootstrap_stage3() { # At this point, we should have a proper GCC, and don't need to # rely on the system wrappers. Let's get rid of them, so that # they stop mucking up builds. - rm -f "${ROOT}"/tmp/usr/local/bin/{,my,${CHOST}-}{gcc,g++} + rm -f "${ROOT}"/tmp/usr/local/bin/{,my,"${CHOST}"-}{gcc,g++} BOOTSTRAP_STAGE=stage3 configure_toolchain || return 1 @@ -2308,7 +2326,7 @@ bootstrap_stage3() { # to finally overwrite it. if [[ ! -d "${ROOT}"/usr/share/portage ]]; then mkdir -p "${ROOT}"/usr/share - cp -a "${ROOT}"{/tmp,}/usr/share/portage + cp -RpP "${ROOT}"{/tmp,}/usr/share/portage fi local -a linker_pkgs compiler_pkgs @@ -2320,7 +2338,8 @@ bootstrap_stage3() { # script file need meson is beyond me. So, we have no other way # than to fake it here for the time being like in stage2. if [[ ! -e "${ROOT}"/lib/gentoo/functions.sh ]] ; then - mkdir -p "${ROOT}"/lib/gentoo + mkdir -p "${ROOT}"/usr/lib/gentoo + [[ -x ${ROOT}/lib ]] || ln -s usr/lib "${ROOT}"/lib cp "${ROOT}"/tmp/lib/gentoo/functions.sh \ "${ROOT}"/lib/gentoo/functions.sh fi @@ -2388,6 +2407,7 @@ bootstrap_stage3() { pkgs=( sys-devel/binutils-config sys-libs/zlib + app-arch/xz-utils "${linker_pkgs[@]}" ) # use the new dynamic linker in place of rpath from now on. @@ -2404,6 +2424,9 @@ bootstrap_stage3() { pkgs=( sys-apps/attr sys-libs/libcap + sys-libs/zlib + app-arch/bzip2 + dev-lang/perl sys-libs/libxcrypt ) BOOTSTRAP_RAP=yes \ @@ -2416,7 +2439,8 @@ bootstrap_stage3() { else # make libgcc_s.so.1 from stage2 available while we build the # new toolchain - export LD_LIBRARY_PATH=$(dirname "$(gcc -print-libgcc-file-name)") + LD_LIBRARY_PATH="$(dirname "$(gcc -print-libgcc-file-name)")" + export LD_LIBRARY_PATH pkgs=( sys-devel/gnuconfig @@ -2460,11 +2484,11 @@ bootstrap_stage3() { # Make ${CHOST}-libtool (used by compiler-rt's and llvm's ebuild) to # point at the correct libtool in stage3. Resolve it in runtime, to # support llvm version upgrades. - rm -f ${ROOT}/usr/bin/${CHOST}-libtool + rm -f "${ROOT}/usr/bin/${CHOST}-libtool" { echo "#!${ROOT}/bin/sh" echo 'exec llvm-libtool-darwin "$@"' - } > "${ROOT}"/usr/bin/${CHOST}-${bin} + } > "${ROOT}/usr/bin/${CHOST}-${bin}" # Now clang is ready, can use it instead of /usr/bin/gcc # TODO: perhaps symlink the whole etc/portage instead? @@ -2513,7 +2537,7 @@ bootstrap_stage3() { chmod +x "${ROOT}"/usr/bin/ld fi fi - + # Start using apps from the final destination Prefix cat > "${ROOT}"/tmp/etc/env.d/10stage3 <<-EOF PATH="${ROOT}/usr/bin:${ROOT}/bin" @@ -2566,9 +2590,9 @@ bootstrap_stage3() { if [[ ${OFFLINE_MODE} ]]; then # --keep used ${DISTDIR}, which make it easier to download a # snapshot beforehand - emerge-webrsync --keep || return 1 + emerge-webrsync --keep --no-pgp-verify || return 1 else - emerge --color n --sync || emerge-webrsync || return 1 + emerge --color n --sync || emerge-webrsync --no-pgp-verify || return 1 fi # Avoid installing git or encryption just for fun while completing @system @@ -2643,15 +2667,20 @@ set_helper_vars() { SNAPSHOT_URL=${SNAPSHOT_URL:-"${SNAPSHOT_HOST}/snapshots"} # USE-flags to disable during bootstrap for they produce - # unnecessary, or worse: circular deps #901101, #936629 - # - nghttp2 -> cmake -> curl -> nghttp2 (http2) + # unnecessary, or worse: circular deps + # - nghttp2 -> cmake -> curl -> nghttp2 (http2) #901101 + # - ensurepip -> python -> ensurepip (ensurepip) + # - binutils -> zstd -> meson -> python (zstd) #967234 DISABLE_USE=( "-crypt" "-curl_quic_openssl" # curl + "-ensurepip" # python-3.13 "-git" "-http2" # curl "-http3" # curl "-quic" # curl + "-zstd" # binutils/gcc + "-debuginfod" # binutils ) export MAKE CONFIG_SHELL @@ -2736,6 +2765,7 @@ EOF CPATH \ LIBRARY_PATH \ ; do + [[ -n ${SETUP_ENV_ONLY} ]] && continue # we already checked this # starting on purpose a shell here iso ${!flag} because I want # to know if the shell initialisation files trigger this # note that this code is so complex because it handles both @@ -2928,17 +2958,7 @@ EOF # ancient Xcode (3.0/3.1) cat << EOF -Ok, this is an old system, let's just try and see what happens. -EOF - elif [[ $(xcode-select -p) != */CommandLineTools ]] ; then - # to an extent, bug #564814 and bug #562800 - cat << EOF - -Your xcode-select is not set to CommandLineTools. This prevents builds -from succeeding. Switch to command line tools for the bootstrap to -continue. Please execute: - xcode-select -s /Library/Developer/CommandLineTools -and try running me again. +Ok, this is an old system, we don't support it any more unfortunately :( EOF exit 1 fi @@ -2972,9 +2992,12 @@ EOF esac # get rid of excess spaces (at least Solaris wc does) ncpu=$((ncpu + 0)) - # Suggest usage of 50% to 75% of the available CPUs - [[ ${tcpu} -eq 0 ]] && tcpu=1 + # Suggest usage of 50% to 75% of the available CPUs when there is + # at least 4 cores, anything below, just use all (typically VMs to + # test stuff), this gives 1->1, 2->2, 3->3, 4->3, ... + [[ ${ncpu} -eq 0 ]] && ncpu=1 local tcpu=$((((ncpu * 3) + 1) / 4)) + [[ ${ncpu} -lt 4 ]] && tcpu=${ncpu} [[ -n ${USE_CPU_CORES} ]] && tcpu=${USE_CPU_CORES} cat << EOF @@ -2989,7 +3012,7 @@ EOF read -r -p "How many parallel make jobs do you want? [${tcpu}] " ans case "${ans}" in "") - MAKEOPTS="-j${tcpu}" + ans="${tcpu}" ;; *) if [[ ${ans} -le 0 ]] ; then @@ -3014,25 +3037,23 @@ EOF echo "(are you?)" fi fi - MAKEOPTS="-j${ans}" ;; esac - export MAKEOPTS + # bug 966647: retain MAKEOPTS for phase runs + if [[ -z ${SETUP_ENV_ONLY} ]] ; then + MAKEOPTS="-j${ans}" + # technically could consider using --jobs ${ans} --load-average ${ans}.5 + # but there are some bugs in dependencies that way; just use a + # single job to make the output quieter, on error Portage shows + # the full buildlog + EMERGE_DEFAULT_OPTS="--jobs 1" + export MAKEOPTS EMERGE_DEFAULT_OPTS + fi #32/64 bits, multilib local candomultilib=no local t64 t32 case "${CHOST}" in - *86*-darwin1[012345]) - # PPC/Darwin only works in 32-bits mode, so this is Intel - # only, and officially starting from Leopard (10.5, darwin9) - # but this is broken, so stick to 32-bits there, and use it - # from Snow Lepard (10.6). - # with Big Sur (11.0, darwin20) we have x64 or arm64 only - candomultilib=yes - t64=x86_64-${CHOST#*-} - t32=i686-${CHOST#*-} - ;; *-solaris*) # Solaris is a true multilib system from as long as it does # 64-bits, we only need to know if the CPU we use is capable @@ -3555,10 +3576,10 @@ if [[ ${CHOST} == *-linux-* ]] ; then # x86_64-pc-linux-ubuntu16 # I choose the latter because it is compatible with most # UNIX vendors and it allows to fit RAP into platform - dist=$(lsb_release -si) - rel=$(lsb_release -sr) + dist=$(lsb_release -si 2>/dev/null) + rel=$(lsb_release -sr 2>/dev/null) if [[ -z ${dist} ]] || [[ -z ${rel} ]] ; then - source /etc/os-release # this may fail if the file isn't there + source /etc/os-release 2>/dev/null # accept when file isn't there [[ -z ${dist} ]] && dist=${ID} [[ -z ${dist} ]] && dist=${NAME} [[ -z ${rel} ]] && rel=${VERSION_ID} diff --git a/test/compat_layer.py b/test/compat_layer.py index 746a5f1d..0d127e4c 100644 --- a/test/compat_layer.py +++ b/test/compat_layer.py @@ -124,6 +124,8 @@ def __init__(self): @rfm.simple_test class EessiSetTest(RunInGentooPrefixTest): def __init__(self): + # Only EESSI 2025.06 and older used package sets + self.skip_if(self.eessi_version not in ['2023.06', '2025.06']) super().__init__() self.descr = 'Test whether a EESSI set is available for the given architecture, operating system, and version.' self.command = 'emerge --list-sets' @@ -137,6 +139,8 @@ def __init__(self): @rfm.simple_test class EessiSetInstalledTest(RunInGentooPrefixTest): def __init__(self): + # Only EESSI 2025.06 and older used package sets + self.skip_if(self.eessi_version not in ['2023.06', '2025.06']) super().__init__() self.descr = 'Test whether a the packages of the EESSI set have been installed.' self.command = 'qlist -IRv' @@ -183,7 +187,8 @@ def __init__(self): super().__init__() self.descr = 'Verify that all required symlinks to host files have been created.' symlink_path = os.path.join(self.compat_dir, self.symlink_to_host) - self.command = f'readlink {symlink_path}' + # cut off variant symlink output + self.command = f"readlink {symlink_path} | sed 's/\$(.*):-//'" self.sanity_patterns = sn.all([ sn.assert_eq(self.exit_code, 0), sn.assert_found(f'\n/{self.symlink_to_host}\n', self.stdout),