Change of handling of CORSIKA interaction tables.

.env_template

@@ -2,8 +2,9 @@
 # Copy this file to '.env' and add the required
 # credentials for simulation model database access
 
-SIMTOOLS_CORSIKA_HE_INTERACTION=qgs3
+SIMTOOLS_CORSIKA_HE_INTERACTION=epos
 SIMTOOLS_CORSIKA_LE_INTERACTION=urqmd
+SIMTOOLS_CORSIKA_INTERACTION_TABLE_PATH=/workdir/external/simpipe/simulation_software/corsika7-interaction-tables/interaction-tables/
 SIMTOOLS_CORSIKA_PATH=/workdir/simulation_software/corsika7
 SIMTOOLS_DB_API_AUTHENTICATION_DATABASE=admin
 SIMTOOLS_DB_API_PORT=27017

.github/workflows/CI-integrationtests.yml

@@ -1,6 +1,8 @@
 ---
 name: CI-integrationtests
 
+permissions: {}
+
 on:
   workflow_dispatch:
     inputs:
@@ -22,8 +24,6 @@ on:
         type: string
         default: '["7.0.0","6.0.2","5.0.0","6.0.2,6.1.1"]'
     secrets:
-      CLOUD_QGSJET3:
-        required: false
       DB_SERVER:
         required: false
       DB_API_USER:
@@ -42,9 +42,6 @@ on:
   release:
     types: [published]
 
-env:
-  CLOUD_URL: "https://syncandshare.desy.de/index.php/s/"
-
 jobs:
 
   test_building:
@@ -63,13 +60,41 @@ jobs:
           python -m pip install --upgrade pip build
           python -m build
 
+  prepare_interaction_tables:
+    name: download CORSIKA interaction tables
+    runs-on: ubuntu-latest
+    env:
+      IT_NAME: "corsika7-interaction-tables"
+      CORSIKA_TABLES: "v0.1.0"
+
+    steps:
+      - name: Clone CORSIKA interaction tables (exclude QGSJet tables)
+        run: |
+          REPO_URL=https://gitlab.cta-observatory.org/cta-computing/dpps/simpipe/simulation_software/${IT_NAME}
+          export GIT_LFS_SKIP_SMUDGE=1  # disable automatic downloading of all LFS files
+          git clone --depth 1 --branch "$CORSIKA_TABLES" "$REPO_URL" "$IT_NAME"
+          git lfs install
+          (cd "$IT_NAME" && git lfs pull --exclude="interaction-tables/qgsdat-II-04,interaction-tables/qgsdat-III")
+
+      - name: Upload CORSIKA interaction tables
+        uses: actions/upload-artifact@v4
+        with:
+          name: corsika-interaction-tables
+          path: ${{ env.IT_NAME }}
+          if-no-files-found: error
+          retention-days: 30
+
   integration_tests:
+    needs: prepare_interaction_tables
     runs-on: ubuntu-latest
     permissions:
       contents: read
     container:
       image: ${{ inputs.container_image || 'ghcr.io/gammasim/simtools-dev:latest' }}
       options: --user 0
+    env:
+      CONTAINER_IMAGE: ${{ inputs.container_image || 'ghcr.io/gammasim/simtools-dev:latest' }}
+      IT_NAME: "corsika7-interaction-tables"
 
     services:
       mongodb:
@@ -124,10 +149,17 @@ jobs:
             echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=$SIMTOOLS_DB_SIMULATION_MODEL_VERSION"
             echo "SIMTOOLS_SIM_TELARRAY_PATH=$SIMTOOLS_SIM_TELARRAY_PATH"
             echo "SIMTOOLS_CORSIKA_PATH=$SIMTOOLS_CORSIKA_PATH"
+            echo "SIMTOOLS_CORSIKA_INTERACTION_TABLE_PATH=$SIMTOOLS_CORSIKA_INTERACTION_TABLE_PATH"
             echo "SIMTOOLS_CORSIKA_HE_INTERACTION=$SIMTOOLS_CORSIKA_HE_INTERACTION"
             echo "SIMTOOLS_CORSIKA_LE_INTERACTION=$SIMTOOLS_CORSIKA_LE_INTERACTION"
           } | tee .env >> "$GITHUB_ENV"
 
+      - name: Download CORSIKA interaction tables
+        uses: actions/download-artifact@v4
+        with:
+          name: corsika-interaction-tables
+          path: ${{ env.SIMTOOLS_CORSIKA_INTERACTION_TABLE_PATH }}/..
+
       - name: Extend PATH (sim_telarray)
         run: |
           [ -n "$SIMTOOLS_SIM_TELARRAY_PATH" ] && echo "$SIMTOOLS_SIM_TELARRAY_PATH" >> "$GITHUB_PATH"
@@ -173,16 +205,9 @@ jobs:
             --db_simulation_model_version ${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }} \
             --branch "$SIMTOOLS_DB_SIMULATION_MODEL_BRANCH"
 
-      - name: Download QGSJet (prod images without tables)
-        if: contains(inputs.container_image, 'simtools-prod')
-        run: |
-          wget -nv -O - "${{ env.CLOUD_URL }}/${{ secrets.CLOUD_QGSJET3 }}/download" | \
-            bunzip2 > "${SIMTOOLS_CORSIKA_PATH}/qgsdat-III"
-
       - name: Integration tests
         shell: bash -l {0}
         run: |
           cat .env
-          simtools-print-version
           pytest --model_version=${{ matrix.model_version }} --color=yes --durations=20 \
             -n 4 --dist loadscope --retries 2 --retry-delay 5 --no-cov tests/integration_tests/

.github/workflows/build-simtools-dev.yml

@@ -21,24 +21,8 @@ env:
   AVX_FLAG: "generic"
 
 jobs:
-
-  download-qgsjet-tables:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Download QGSJet
-        run: |
-          wget --no-verbose ${{ env.CLOUD_URL }}/${{ secrets.CLOUD_QGSJET3 }}/download -O qgsdat-III.bz2
-      - name: Upload QGSJet files
-        uses: actions/upload-artifact@v5
-        with:
-          name: upload-qgsjet-tables
-          path: |
-            qgsdat-III.bz2
-          retention-days: 1
-
   build-simtools-dev:
     runs-on: ubuntu-latest
-    needs: [download-qgsjet-tables]
     permissions:
       contents: read
       packages: write
@@ -49,12 +33,6 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v6
 
-      - name: Download QGSJet tables
-        uses: actions/download-artifact@v7
-        with:
-          name: upload-qgsjet-tables
-          path: .
-
       - name: Set build branch
         run: |
           if [[ "${{ github.event_name }}" == 'pull_request' ]]; then

docker/Dockerfile-corsika7

@@ -121,7 +121,8 @@ RUN ls ../config_*.h && for config_file in ../config_*.h; do \
         mkdir -p ${CORSIKA_RUN_DIR} && \
         corsika_name=$(basename "$config_file" .h | sed 's/^config_/corsika_/'); \
         mv -v -f "./run/corsika${CORSIKA_VERSION}Linux_"* "${CORSIKA_RUN_DIR}/${corsika_name}" && \
-        cp -v -r "./run/"* ${CORSIKA_RUN_DIR}/ && cp -v -r ./epos ${CORSIKA_RUN_DIR}/; \
+        # NUCNUCS table required to be in same directory as executables
+        cp -v -f ./run/NUCNUCCS ${CORSIKA_RUN_DIR}/; \
     done
 
 # Generate build_opts.yml with build information

docker/Dockerfile-simtools-dev

@@ -2,11 +2,11 @@
 #
 # Uses CORSIKA (no vector optimization) and sim_telarray pre-built images.
 #
-# Minimal CORSIKA version is 7.8x (otherwise QGSJet-II tables need to be added)
-#
-# All dependencies for simtools are installed, but not simtools itself.
+# CORSIKA and sim_telarray are installed, but not simtools itself.
 # It is expected that simtools is installed in an external directory
 # mounted in the container.
+# CORSIKA interaction tables are expected to be cloned into an external directory
+# and mounted to the container (set SIMTOOLS_CORSIKA_INTERACTION_TABLE_PATH accordingly).
 #
 # hadolint global ignore=DL3013,DL3041
 # - DL3013, DL3041: ignore warnings about using latest
@@ -34,13 +34,6 @@ RUN microdnf update -y && microdnf install -y \
     microdnf clean all && \
     ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python
 
-# Add QGSJet tables downloaded externally (large files!)
-WORKDIR /workdir/simulation_software/corsika7
-COPY qgsdat-III.bz2 .
-RUN if [ -f qgsdat-III.bz2 ]; then \
-        bzip2 -dq qgsdat-III.bz2; \
-    fi
-
 # Install simtools (main branch)
 WORKDIR /workdir
 RUN wget --quiet https://raw.githubusercontent.com/gammasim/simtools/main/pyproject.toml && \

docs/Makefile

@@ -3,7 +3,7 @@
 
 # Check all https links
 # SPHINXOPTS    = -W -v --keep-going -n --color -b linkcheck
- SPHINXOPTS    = -W -v --keep-going -n --color
+SPHINXOPTS    = -W -v --keep-going -n --color
 SPHINXBUILD   = sphinx-build
 SPHINXPROJ    = simtools
 SOURCEDIR     = source

docs/changes/1987.api.md

@@ -0,0 +1,10 @@
+CORSIKA interactions tables are not part of the CORSIKA7 container anymore and also not added to the simtools-dev container.
+CORSIKA tables are now available from the new [CTAO gitlab repository corsika7-interaction-tables](https://gitlab.cta-observatory.org/cta-computing/dpps/simpipe/simulation_software/corsika7-interaction-tables). This includes QGSJet-II, QGSJet-III, EPOS, EGS4, etc tables.
+
+To install and use the interaction tables for both production and development:
+
+- clone that repository into a separate directory from simtools (recommend the example such that it is reachable from inside the simtools-dev container by `/workdir/external/simpipe/simulation_software/corsika7-interaction-tables/interaction-tables/`
+- ensure [Git LFS](https://git-lfs.com/) is installed and, after cloning, run `git lfs install` (once per system) and `git lfs pull` in the cloned repository so that the actual interaction table files (and not only pointer files) are downloaded
+- Add a new env variable called `SIMTOOLS_CORSIKA_INTERACTION_TABLE_PATH` pointing to that path (see example in `.env_table`)
+
+Integration tests are cloning this repository and making the interaction tables available during testing.

docs/changes/1987.feature.md

@@ -0,0 +1 @@
+Change of the default interaction model from qgs3 to epos (this allows to avoid to download the Gigabyte large QGSJet tables).

docs/changes/1988.maintenance.md

@@ -0,0 +1 @@
+Solve sphinx errors after update to sphinx 9.10.

docs/source/conf.py

@@ -48,7 +48,7 @@ def get_python_version_from_pyproject():
 # -- Project information -----------------------------------------------------
 
 project = "simtools"
-copyright = "2024-2025, gammasim-tools, simtools developers"  # noqa A001
+copyright = "2024-2026, gammasim-tools, simtools developers"  # noqa A001
 author = get_authors_from_citation_file()
 
 python_min_requires, python_requires = get_python_version_from_pyproject()
@@ -81,6 +81,13 @@ def get_python_version_from_pyproject():
     "sphinx_design",
 ]
 
+nitpicky = True
+
+nitpick_ignore = {
+    ("py:class", "numpy.float64"),
+    ("py:class", "numpy.uint32"),
+}
+
 # Display todos by setting to True
 todo_include_todos = True
 

src/simtools/corsika/corsika_config.py

@@ -53,7 +53,7 @@ def __init__(self, array_model, run_number, label=None):
         self.io_handler = io_handler.IOHandler()
         self.array_model = array_model
         self.corsika_exec = settings.config.corsika_exe
-        self.interaction_table_path = settings.config.corsika_path
+        self.interaction_table_path = settings.config.corsika_interaction_table_path
         self.config = self._fill_corsika_configuration(settings.config.args)
         self._initialize_from_config(settings.config.args)
 
@@ -443,11 +443,13 @@ def _corsika_configuration_interaction_flags(self, parameters_from_db):
     def _epos_flags(self):
         """EPOS interaction model flags."""
         epos_par = {}
-        epos_path = Path(self.interaction_table_path) / "epos"
+        epos_path = Path(self.interaction_table_path)
         epos_par["EPOPAR fname pathnx"] = [f"{epos_path}/"]
-        for epos_file in ["inics", "iniev", "inirj", "initl", "check"]:
+        for epos_file in ["inics", "iniev", "inirj", "initl"]:
             epos_par[f"EPOPAR fname {epos_file}"] = [str(epos_path / f"epos.{epos_file}")]
         epos_par["EPOPAR fname hpf"] = [str(epos_path / "urqmd34/tables.dat")]
+        for dummy_output in ["check", "histo", "data", "copy"]:
+            epos_par[f"EPOPAR fname {dummy_output}"] = ["none"]
 
         return epos_par
 

src/simtools/db/mongo_db.py

@@ -4,7 +4,7 @@
 import logging
 import re
 from pathlib import Path
-from threading import Lock
+from threading import Lock as _Lock
 
 import gridfs
 import jsonschema
@@ -126,7 +126,7 @@ class MongoDBHandler:  # pylint: disable=unsubscriptable-object
     """
 
     db_client: MongoClient = None
-    _lock = Lock()
+    _lock = _Lock()
     _logger = logging.getLogger(__name__)
 
     def __init__(self, db_config=None):

src/simtools/settings.py

@@ -16,6 +16,7 @@ def __init__(self):
         self._sim_telarray_path = None
         self._sim_telarray_exe = None
         self._corsika_path = None
+        self._corsika_interaction_table_path = None
         self._corsika_exe = None
         self.user = os.getenv("USER", "unknown")
         self.hostname = socket.gethostname()
@@ -54,6 +55,12 @@ def load(self, args=None, db_config=None):
             else os.getenv("SIMTOOLS_CORSIKA_PATH")
         )
 
+        self._corsika_interaction_table_path = (
+            args.get("corsika_interaction_table_path")
+            if args is not None and "corsika_interaction_table_path" in args
+            else os.getenv("SIMTOOLS_CORSIKA_INTERACTION_TABLE_PATH")
+        )
+
         self._corsika_exe = self._get_corsika_exec() if self._corsika_path is not None else None
 
     def _get_corsika_exec(self):
@@ -121,6 +128,15 @@ def corsika_path(self):
         """Path to the CORSIKA installation directory."""
         return Path(self._corsika_path) if self._corsika_path is not None else None
 
+    @property
+    def corsika_interaction_table_path(self):
+        """Path to the CORSIKA interaction table directory."""
+        return (
+            Path(self._corsika_interaction_table_path)
+            if self._corsika_interaction_table_path is not None
+            else self.corsika_path
+        )
+
     @property
     def corsika_exe(self):
         """Path to the CORSIKA executable."""

tests/unit_tests/corsika/test_corsika_config.py

@@ -253,7 +253,8 @@ def test_corsika_configuration_interaction_flags(
     assert isinstance(parameters, dict)
     assert "ECUTS" in parameters
     assert parameters["MAXPRT"] == ["10"]
-    assert len(parameters) == 9
+    # number of parameters depend on HE interaction model (qgs3 or epos)
+    assert len(parameters) == 9 or len(parameters) == 19
 
 
 def test_input_config_first_interaction_height(corsika_config_mock_array_model):
@@ -1184,12 +1185,12 @@ def test_epos_flags(corsika_config_mock_array_model, mocker):
 
     assert isinstance(epos_flags, dict)
     assert "EPOPAR fname pathnx" in epos_flags
-    assert epos_flags["EPOPAR fname pathnx"] == ["/path/to/corsika/epos/epos/"]
+    assert epos_flags["EPOPAR fname pathnx"] == ["/path/to/corsika/epos/"]
 
-    for epos_file in ["inics", "iniev", "inirj", "initl", "check"]:
+    for epos_file in ["inics", "iniev", "inirj", "initl"]:
         key = f"EPOPAR fname {epos_file}"
         assert key in epos_flags
-        assert epos_flags[key] == [f"/path/to/corsika/epos/epos/epos.{epos_file}"]
+        assert epos_flags[key] == [f"/path/to/corsika/epos/epos.{epos_file}"]
 
 
 def test_epos_flags_with_different_paths(corsika_config_mock_array_model, mocker):
@@ -1207,9 +1208,9 @@ def test_epos_flags_with_different_paths(corsika_config_mock_array_model, mocker
 
     epos_flags = corsika_config_mock_array_model._epos_flags()
 
-    assert epos_flags["EPOPAR fname pathnx"] == ["/custom/tables/epos/"]
-    assert epos_flags["EPOPAR fname inics"] == ["/custom/tables/epos/epos.inics"]
-    assert epos_flags["EPOPAR fname check"] == ["/custom/tables/epos/epos.check"]
+    assert epos_flags["EPOPAR fname pathnx"] == ["/custom/tables/"]
+    assert epos_flags["EPOPAR fname inics"] == ["/custom/tables/epos.inics"]
+    assert epos_flags["EPOPAR fname check"] == ["none"]
 
 
 def test_corsika_configuration_interaction_flags_with_epos(