diff --git a/.env_template b/.env_template index 5761baa26b..cd55bee2f5 100644 --- a/.env_template +++ b/.env_template @@ -2,8 +2,9 @@ # Copy this file to '.env' and add the required # credentials for simulation model database access -SIMTOOLS_CORSIKA_HE_INTERACTION=qgs3 +SIMTOOLS_CORSIKA_HE_INTERACTION=epos SIMTOOLS_CORSIKA_LE_INTERACTION=urqmd +SIMTOOLS_CORSIKA_INTERACTION_TABLE_PATH=/workdir/external/simpipe/simulation_software/corsika7-interaction-tables/interaction-tables/ SIMTOOLS_CORSIKA_PATH=/workdir/simulation_software/corsika7 SIMTOOLS_DB_API_AUTHENTICATION_DATABASE=admin SIMTOOLS_DB_API_PORT=27017 diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml index 6e3b2a4afe..5f63ab7200 100644 --- a/.github/workflows/CI-integrationtests.yml +++ b/.github/workflows/CI-integrationtests.yml @@ -1,6 +1,8 @@ --- name: CI-integrationtests +permissions: {} + on: workflow_dispatch: inputs: @@ -22,8 +24,6 @@ on: type: string default: '["7.0.0","6.0.2","5.0.0","6.0.2,6.1.1"]' secrets: - CLOUD_QGSJET3: - required: false DB_SERVER: required: false DB_API_USER: @@ -42,9 +42,6 @@ on: release: types: [published] -env: - CLOUD_URL: "https://syncandshare.desy.de/index.php/s/" - jobs: test_building: @@ -63,13 +60,41 @@ jobs: python -m pip install --upgrade pip build python -m build + prepare_interaction_tables: + name: download CORSIKA interaction tables + runs-on: ubuntu-latest + env: + IT_NAME: "corsika7-interaction-tables" + CORSIKA_TABLES: "v0.1.0" + + steps: + - name: Clone CORSIKA interaction tables (exclude QGSJet tables) + run: | + REPO_URL=https://gitlab.cta-observatory.org/cta-computing/dpps/simpipe/simulation_software/${IT_NAME} + export GIT_LFS_SKIP_SMUDGE=1 # disable automatic downloading of all LFS files + git clone --depth 1 --branch "$CORSIKA_TABLES" "$REPO_URL" "$IT_NAME" + git lfs install + (cd "$IT_NAME" && git lfs pull --exclude="interaction-tables/qgsdat-II-04,interaction-tables/qgsdat-III") + + - name: Upload CORSIKA interaction tables + uses: actions/upload-artifact@v4 + with: + name: corsika-interaction-tables + path: ${{ env.IT_NAME }} + if-no-files-found: error + retention-days: 30 + integration_tests: + needs: prepare_interaction_tables runs-on: ubuntu-latest permissions: contents: read container: image: ${{ inputs.container_image || 'ghcr.io/gammasim/simtools-dev:latest' }} options: --user 0 + env: + CONTAINER_IMAGE: ${{ inputs.container_image || 'ghcr.io/gammasim/simtools-dev:latest' }} + IT_NAME: "corsika7-interaction-tables" services: mongodb: @@ -124,10 +149,17 @@ jobs: echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=$SIMTOOLS_DB_SIMULATION_MODEL_VERSION" echo "SIMTOOLS_SIM_TELARRAY_PATH=$SIMTOOLS_SIM_TELARRAY_PATH" echo "SIMTOOLS_CORSIKA_PATH=$SIMTOOLS_CORSIKA_PATH" + echo "SIMTOOLS_CORSIKA_INTERACTION_TABLE_PATH=$SIMTOOLS_CORSIKA_INTERACTION_TABLE_PATH" echo "SIMTOOLS_CORSIKA_HE_INTERACTION=$SIMTOOLS_CORSIKA_HE_INTERACTION" echo "SIMTOOLS_CORSIKA_LE_INTERACTION=$SIMTOOLS_CORSIKA_LE_INTERACTION" } | tee .env >> "$GITHUB_ENV" + - name: Download CORSIKA interaction tables + uses: actions/download-artifact@v4 + with: + name: corsika-interaction-tables + path: ${{ env.SIMTOOLS_CORSIKA_INTERACTION_TABLE_PATH }}/.. + - name: Extend PATH (sim_telarray) run: | [ -n "$SIMTOOLS_SIM_TELARRAY_PATH" ] && echo "$SIMTOOLS_SIM_TELARRAY_PATH" >> "$GITHUB_PATH" @@ -173,16 +205,9 @@ jobs: --db_simulation_model_version ${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }} \ --branch "$SIMTOOLS_DB_SIMULATION_MODEL_BRANCH" - - name: Download QGSJet (prod images without tables) - if: contains(inputs.container_image, 'simtools-prod') - run: | - wget -nv -O - "${{ env.CLOUD_URL }}/${{ secrets.CLOUD_QGSJET3 }}/download" | \ - bunzip2 > "${SIMTOOLS_CORSIKA_PATH}/qgsdat-III" - - name: Integration tests shell: bash -l {0} run: | cat .env - simtools-print-version pytest --model_version=${{ matrix.model_version }} --color=yes --durations=20 \ -n 4 --dist loadscope --retries 2 --retry-delay 5 --no-cov tests/integration_tests/ diff --git a/.github/workflows/build-simtools-dev.yml b/.github/workflows/build-simtools-dev.yml index 7ea28598f4..2d418f091b 100644 --- a/.github/workflows/build-simtools-dev.yml +++ b/.github/workflows/build-simtools-dev.yml @@ -21,24 +21,8 @@ env: AVX_FLAG: "generic" jobs: - - download-qgsjet-tables: - runs-on: ubuntu-latest - steps: - - name: Download QGSJet - run: | - wget --no-verbose ${{ env.CLOUD_URL }}/${{ secrets.CLOUD_QGSJET3 }}/download -O qgsdat-III.bz2 - - name: Upload QGSJet files - uses: actions/upload-artifact@v5 - with: - name: upload-qgsjet-tables - path: | - qgsdat-III.bz2 - retention-days: 1 - build-simtools-dev: runs-on: ubuntu-latest - needs: [download-qgsjet-tables] permissions: contents: read packages: write @@ -49,12 +33,6 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 - - name: Download QGSJet tables - uses: actions/download-artifact@v7 - with: - name: upload-qgsjet-tables - path: . - - name: Set build branch run: | if [[ "${{ github.event_name }}" == 'pull_request' ]]; then diff --git a/docker/Dockerfile-corsika7 b/docker/Dockerfile-corsika7 index e859cf51cc..01ef58bda4 100644 --- a/docker/Dockerfile-corsika7 +++ b/docker/Dockerfile-corsika7 @@ -121,7 +121,8 @@ RUN ls ../config_*.h && for config_file in ../config_*.h; do \ mkdir -p ${CORSIKA_RUN_DIR} && \ corsika_name=$(basename "$config_file" .h | sed 's/^config_/corsika_/'); \ mv -v -f "./run/corsika${CORSIKA_VERSION}Linux_"* "${CORSIKA_RUN_DIR}/${corsika_name}" && \ - cp -v -r "./run/"* ${CORSIKA_RUN_DIR}/ && cp -v -r ./epos ${CORSIKA_RUN_DIR}/; \ + # NUCNUCS table required to be in same directory as executables + cp -v -f ./run/NUCNUCCS ${CORSIKA_RUN_DIR}/; \ done # Generate build_opts.yml with build information diff --git a/docker/Dockerfile-simtools-dev b/docker/Dockerfile-simtools-dev index c46f328c18..c4e80124ee 100644 --- a/docker/Dockerfile-simtools-dev +++ b/docker/Dockerfile-simtools-dev @@ -2,11 +2,11 @@ # # Uses CORSIKA (no vector optimization) and sim_telarray pre-built images. # -# Minimal CORSIKA version is 7.8x (otherwise QGSJet-II tables need to be added) -# -# All dependencies for simtools are installed, but not simtools itself. +# CORSIKA and sim_telarray are installed, but not simtools itself. # It is expected that simtools is installed in an external directory # mounted in the container. +# CORSIKA interaction tables are expected to be cloned into an external directory +# and mounted to the container (set SIMTOOLS_CORSIKA_INTERACTION_TABLE_PATH accordingly). # # hadolint global ignore=DL3013,DL3041 # - DL3013, DL3041: ignore warnings about using latest @@ -34,13 +34,6 @@ RUN microdnf update -y && microdnf install -y \ microdnf clean all && \ ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python -# Add QGSJet tables downloaded externally (large files!) -WORKDIR /workdir/simulation_software/corsika7 -COPY qgsdat-III.bz2 . -RUN if [ -f qgsdat-III.bz2 ]; then \ - bzip2 -dq qgsdat-III.bz2; \ - fi - # Install simtools (main branch) WORKDIR /workdir RUN wget --quiet https://raw.githubusercontent.com/gammasim/simtools/main/pyproject.toml && \ diff --git a/docs/Makefile b/docs/Makefile index 19a8719648..a1a26f9b72 100755 --- a/docs/Makefile +++ b/docs/Makefile @@ -3,7 +3,7 @@ # Check all https links # SPHINXOPTS = -W -v --keep-going -n --color -b linkcheck - SPHINXOPTS = -W -v --keep-going -n --color +SPHINXOPTS = -W -v --keep-going -n --color SPHINXBUILD = sphinx-build SPHINXPROJ = simtools SOURCEDIR = source diff --git a/docs/changes/1987.api.md b/docs/changes/1987.api.md new file mode 100644 index 0000000000..01d4b9070f --- /dev/null +++ b/docs/changes/1987.api.md @@ -0,0 +1,10 @@ +CORSIKA interactions tables are not part of the CORSIKA7 container anymore and also not added to the simtools-dev container. +CORSIKA tables are now available from the new [CTAO gitlab repository corsika7-interaction-tables](https://gitlab.cta-observatory.org/cta-computing/dpps/simpipe/simulation_software/corsika7-interaction-tables). This includes QGSJet-II, QGSJet-III, EPOS, EGS4, etc tables. + +To install and use the interaction tables for both production and development: + +- clone that repository into a separate directory from simtools (recommend the example such that it is reachable from inside the simtools-dev container by `/workdir/external/simpipe/simulation_software/corsika7-interaction-tables/interaction-tables/` +- ensure [Git LFS](https://git-lfs.com/) is installed and, after cloning, run `git lfs install` (once per system) and `git lfs pull` in the cloned repository so that the actual interaction table files (and not only pointer files) are downloaded +- Add a new env variable called `SIMTOOLS_CORSIKA_INTERACTION_TABLE_PATH` pointing to that path (see example in `.env_table`) + +Integration tests are cloning this repository and making the interaction tables available during testing. diff --git a/docs/changes/1987.feature.md b/docs/changes/1987.feature.md new file mode 100644 index 0000000000..96c4dc8b6e --- /dev/null +++ b/docs/changes/1987.feature.md @@ -0,0 +1 @@ +Change of the default interaction model from qgs3 to epos (this allows to avoid to download the Gigabyte large QGSJet tables). diff --git a/docs/changes/1988.maintenance.md b/docs/changes/1988.maintenance.md new file mode 100644 index 0000000000..f2f34e36c9 --- /dev/null +++ b/docs/changes/1988.maintenance.md @@ -0,0 +1 @@ +Solve sphinx errors after update to sphinx 9.10. diff --git a/docs/source/conf.py b/docs/source/conf.py index ca00f9094a..6156278fe6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -48,7 +48,7 @@ def get_python_version_from_pyproject(): # -- Project information ----------------------------------------------------- project = "simtools" -copyright = "2024-2025, gammasim-tools, simtools developers" # noqa A001 +copyright = "2024-2026, gammasim-tools, simtools developers" # noqa A001 author = get_authors_from_citation_file() python_min_requires, python_requires = get_python_version_from_pyproject() @@ -81,6 +81,13 @@ def get_python_version_from_pyproject(): "sphinx_design", ] +nitpicky = True + +nitpick_ignore = { + ("py:class", "numpy.float64"), + ("py:class", "numpy.uint32"), +} + # Display todos by setting to True todo_include_todos = True diff --git a/src/simtools/corsika/corsika_config.py b/src/simtools/corsika/corsika_config.py index 6064c05a97..38bdd8e9d2 100644 --- a/src/simtools/corsika/corsika_config.py +++ b/src/simtools/corsika/corsika_config.py @@ -53,7 +53,7 @@ def __init__(self, array_model, run_number, label=None): self.io_handler = io_handler.IOHandler() self.array_model = array_model self.corsika_exec = settings.config.corsika_exe - self.interaction_table_path = settings.config.corsika_path + self.interaction_table_path = settings.config.corsika_interaction_table_path self.config = self._fill_corsika_configuration(settings.config.args) self._initialize_from_config(settings.config.args) @@ -443,11 +443,13 @@ def _corsika_configuration_interaction_flags(self, parameters_from_db): def _epos_flags(self): """EPOS interaction model flags.""" epos_par = {} - epos_path = Path(self.interaction_table_path) / "epos" + epos_path = Path(self.interaction_table_path) epos_par["EPOPAR fname pathnx"] = [f"{epos_path}/"] - for epos_file in ["inics", "iniev", "inirj", "initl", "check"]: + for epos_file in ["inics", "iniev", "inirj", "initl"]: epos_par[f"EPOPAR fname {epos_file}"] = [str(epos_path / f"epos.{epos_file}")] epos_par["EPOPAR fname hpf"] = [str(epos_path / "urqmd34/tables.dat")] + for dummy_output in ["check", "histo", "data", "copy"]: + epos_par[f"EPOPAR fname {dummy_output}"] = ["none"] return epos_par diff --git a/src/simtools/db/mongo_db.py b/src/simtools/db/mongo_db.py index 830fe3fb82..86dc02dd54 100644 --- a/src/simtools/db/mongo_db.py +++ b/src/simtools/db/mongo_db.py @@ -4,7 +4,7 @@ import logging import re from pathlib import Path -from threading import Lock +from threading import Lock as _Lock import gridfs import jsonschema @@ -126,7 +126,7 @@ class MongoDBHandler: # pylint: disable=unsubscriptable-object """ db_client: MongoClient = None - _lock = Lock() + _lock = _Lock() _logger = logging.getLogger(__name__) def __init__(self, db_config=None): diff --git a/src/simtools/settings.py b/src/simtools/settings.py index bf86eac06e..e526ccb249 100644 --- a/src/simtools/settings.py +++ b/src/simtools/settings.py @@ -16,6 +16,7 @@ def __init__(self): self._sim_telarray_path = None self._sim_telarray_exe = None self._corsika_path = None + self._corsika_interaction_table_path = None self._corsika_exe = None self.user = os.getenv("USER", "unknown") self.hostname = socket.gethostname() @@ -54,6 +55,12 @@ def load(self, args=None, db_config=None): else os.getenv("SIMTOOLS_CORSIKA_PATH") ) + self._corsika_interaction_table_path = ( + args.get("corsika_interaction_table_path") + if args is not None and "corsika_interaction_table_path" in args + else os.getenv("SIMTOOLS_CORSIKA_INTERACTION_TABLE_PATH") + ) + self._corsika_exe = self._get_corsika_exec() if self._corsika_path is not None else None def _get_corsika_exec(self): @@ -121,6 +128,15 @@ def corsika_path(self): """Path to the CORSIKA installation directory.""" return Path(self._corsika_path) if self._corsika_path is not None else None + @property + def corsika_interaction_table_path(self): + """Path to the CORSIKA interaction table directory.""" + return ( + Path(self._corsika_interaction_table_path) + if self._corsika_interaction_table_path is not None + else self.corsika_path + ) + @property def corsika_exe(self): """Path to the CORSIKA executable.""" diff --git a/tests/unit_tests/corsika/test_corsika_config.py b/tests/unit_tests/corsika/test_corsika_config.py index 625ee83bcf..b1e3fbdf7b 100644 --- a/tests/unit_tests/corsika/test_corsika_config.py +++ b/tests/unit_tests/corsika/test_corsika_config.py @@ -253,7 +253,8 @@ def test_corsika_configuration_interaction_flags( assert isinstance(parameters, dict) assert "ECUTS" in parameters assert parameters["MAXPRT"] == ["10"] - assert len(parameters) == 9 + # number of parameters depend on HE interaction model (qgs3 or epos) + assert len(parameters) == 9 or len(parameters) == 19 def test_input_config_first_interaction_height(corsika_config_mock_array_model): @@ -1184,12 +1185,12 @@ def test_epos_flags(corsika_config_mock_array_model, mocker): assert isinstance(epos_flags, dict) assert "EPOPAR fname pathnx" in epos_flags - assert epos_flags["EPOPAR fname pathnx"] == ["/path/to/corsika/epos/epos/"] + assert epos_flags["EPOPAR fname pathnx"] == ["/path/to/corsika/epos/"] - for epos_file in ["inics", "iniev", "inirj", "initl", "check"]: + for epos_file in ["inics", "iniev", "inirj", "initl"]: key = f"EPOPAR fname {epos_file}" assert key in epos_flags - assert epos_flags[key] == [f"/path/to/corsika/epos/epos/epos.{epos_file}"] + assert epos_flags[key] == [f"/path/to/corsika/epos/epos.{epos_file}"] def test_epos_flags_with_different_paths(corsika_config_mock_array_model, mocker): @@ -1207,9 +1208,9 @@ def test_epos_flags_with_different_paths(corsika_config_mock_array_model, mocker epos_flags = corsika_config_mock_array_model._epos_flags() - assert epos_flags["EPOPAR fname pathnx"] == ["/custom/tables/epos/"] - assert epos_flags["EPOPAR fname inics"] == ["/custom/tables/epos/epos.inics"] - assert epos_flags["EPOPAR fname check"] == ["/custom/tables/epos/epos.check"] + assert epos_flags["EPOPAR fname pathnx"] == ["/custom/tables/"] + assert epos_flags["EPOPAR fname inics"] == ["/custom/tables/epos.inics"] + assert epos_flags["EPOPAR fname check"] == ["none"] def test_corsika_configuration_interaction_flags_with_epos(