diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..3ba3d5da --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + # Enable version updates for Github Actions + - package-ecosystem: "github-actions" + # Look for `/.github/workflows` and `/action.yml` or `.yaml` + directory: "/" + # Check for updates once a week + schedule: + interval: "weekly" + diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 715b12f0..771659bf 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -10,12 +10,12 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 # Fetch full history for git describe to work - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.11' cache: 'pip' @@ -84,7 +84,7 @@ jobs: - name: Upload documentation artifact if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: documentation path: docs/build/html/ diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 4430d290..e74324d3 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -86,7 +86,7 @@ jobs: cibw_archs: arm64 AL_BACKEND_HDF5: AL_BACKEND_HDF5=ON AL_BACKEND_MDSPLUS: AL_BACKEND_MDSPLUS=OFF - AL_BACKEND_UDA: AL_BACKEND_UDA=OFF + AL_BACKEND_UDA: AL_BACKEND_UDA=ON UDA_REF: "2.9.3" - os: macos-14 @@ -97,7 +97,7 @@ jobs: cibw_archs: arm64 AL_BACKEND_HDF5: AL_BACKEND_HDF5=ON AL_BACKEND_MDSPLUS: AL_BACKEND_MDSPLUS=OFF - AL_BACKEND_UDA: AL_BACKEND_UDA=OFF + AL_BACKEND_UDA: AL_BACKEND_UDA=ON UDA_REF: "2.9.3" - os: macos-14 @@ -108,7 +108,7 @@ jobs: cibw_archs: arm64 AL_BACKEND_HDF5: AL_BACKEND_HDF5=ON AL_BACKEND_MDSPLUS: AL_BACKEND_MDSPLUS=OFF - AL_BACKEND_UDA: AL_BACKEND_UDA=OFF + AL_BACKEND_UDA: AL_BACKEND_UDA=ON UDA_REF: "2.9.3" - os: macos-14 @@ -119,7 +119,7 @@ jobs: cibw_archs: arm64 AL_BACKEND_HDF5: AL_BACKEND_HDF5=ON AL_BACKEND_MDSPLUS: AL_BACKEND_MDSPLUS=OFF - AL_BACKEND_UDA: AL_BACKEND_UDA=OFF + AL_BACKEND_UDA: AL_BACKEND_UDA=ON UDA_REF: "2.9.3" - os: macos-14 @@ -130,7 +130,7 @@ jobs: cibw_archs: arm64 AL_BACKEND_HDF5: AL_BACKEND_HDF5=ON AL_BACKEND_MDSPLUS: AL_BACKEND_MDSPLUS=OFF - AL_BACKEND_UDA: AL_BACKEND_UDA=OFF + AL_BACKEND_UDA: AL_BACKEND_UDA=ON UDA_REF: "2.9.3" - os: windows-2022 @@ -199,7 +199,7 @@ jobs: FMT_REF: "11.1.4" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 # unshallow for setuptools-scm.get_version to discover tags @@ -213,7 +213,7 @@ jobs: # core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); # core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '') - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: # python installation used when to start cibuildwheel python-version: "3.12" @@ -226,7 +226,7 @@ jobs: - name: Restore cibuildwheel cache id: cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~\AppData\Local\pypa\cibuildwheel\Cache @@ -238,7 +238,7 @@ jobs: - name: Cache restore windows deps if: startsWith(matrix.os, 'windows') id: cache-win-deps - uses: actions/cache/restore@v4 + uses: actions/cache/restore@v5 env: cache-name: win-deps with: @@ -274,14 +274,14 @@ jobs: # the cache redundantly doesn't compare to the time it takes to rebuild vcpkg packages. if: ${{ always() && startsWith(matrix.os, 'windows') }} id: cache-win-deps-save - uses: actions/cache/save@v4 + uses: actions/cache/save@v5 env: cache-name: win-deps with: path: C:\vcpkg\installed key: ${{ matrix.os }}-${{ env.cache-name }}-a - - uses: pypa/cibuildwheel@v3.1.0 + - uses: pypa/cibuildwheel@v3.4.1 if: startsWith(matrix.os, 'ubuntu-') env: CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} @@ -334,7 +334,7 @@ jobs: output-dir: wheelhouse config-file: "{package}/pyproject.toml" - - uses: pypa/cibuildwheel@v3.1.0 + - uses: pypa/cibuildwheel@v3.4.1 if: startsWith(matrix.os, 'macos-') env: CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} @@ -347,10 +347,15 @@ jobs: cmake.define.${{ matrix.AL_BACKEND_MDSPLUS }} cmake.define.${{ matrix.AL_BACKEND_UDA }} - # Dependency installationinto /tmp + # Dependency installation into /tmp/imas-core-install CIBW_BEFORE_ALL_MACOS: > brew update >&2; - brew install cmake pkg-config boost hdf5 libomp ninja fmt spdlog libxml2 openssl capnp libmemcached >&2; + brew install cmake pkg-config boost hdf5 libomp ninja fmt spdlog libxml2 openssl libmemcached >&2; + git clone -b master https://github.com/capnproto/capnproto.git >&2 && + cd capnproto >&2; + cmake -B build . \ + -DCMAKE_INSTALL_PREFIX=/tmp/imas-core-install >&2 && + cmake --build build --target install -j >&2; git clone --depth 1 --branch ${{ matrix.UDA_REF }} https://github.com/ukaea/UDA.git >&2 && cd UDA >&2; cmake -G Ninja -B build . \ @@ -359,21 +364,21 @@ jobs: -DCLIENT_ONLY=ON \ -DENABLE_CAPNP=ON \ -DMACOSX_DEPLOYMENT_TARGET=14.0 \ - -DCMAKE_INSTALL_PREFIX=/tmp/uda-install >&2 && + -DCMAKE_INSTALL_PREFIX=/tmp/imas-core-install >&2 && cmake --build build --target install -j >&2; # Where to find the dependencies CIBW_ENVIRONMENT_MACOS: > MACOSX_DEPLOYMENT_TARGET=14.0 - CMAKE_PREFIX_PATH="/tmp/uda-install:/opt/homebrew:/usr/local" - PKG_CONFIG_PATH="/tmp/uda-install/lib/pkgconfig:/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig" + CMAKE_PREFIX_PATH="/tmp/imas-core-install:/opt/homebrew:/usr/local" + PKG_CONFIG_PATH="/tmp/imas-core-install/lib/pkgconfig:/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig" with: package-dir: . output-dir: wheelhouse config-file: "{package}/pyproject.toml" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: cibw-wheels-cp${{ matrix.python }}-${{ matrix.platform_id }} path: wheelhouse/*.whl @@ -392,7 +397,7 @@ jobs: name: testpypi url: https://test.pypi.org/p/imas-core steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: pattern: cibw-wheels-* merge-multiple: true # avoid artifact subdirs below wheelhouse @@ -419,7 +424,7 @@ jobs: name: pypi url: https://pypi.org/p/imas-core steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: pattern: cibw-wheels-* merge-multiple: true # avoid artifact subdirs below wheelhouse diff --git a/CMakeLists.txt b/CMakeLists.txt index e35f6e6e..9f14738a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,9 +23,13 @@ option(AL_BACKEND_MDSPLUS "Build the MDSplus backend" OFF) option(AL_BACKEND_HDF5 "Build the HDF5 backend" ON) option(AL_BACKEND_UDA "Build the UDA backend" OFF) option(AL_BACKEND_UDAFAT "Build the UDA backend (use FAT UDA)" OFF) -include(CMakeDependentOption) -cmake_dependent_option(AL_BUILD_MDSPLUS_MODELS "Build MDSplus models" ON -AL_BACKEND_MDSPLUS OFF) +# AL_BUILD_MDSPLUS_MODELS defaults to ON when AL_BACKEND_MDSPLUS is ON, otherwise OFF. +# This can be overridde by user by setting AL_BUILD_MDSPLUS_MODELS explicitly to ON or OFF. +if(AL_BACKEND_MDSPLUS) + option(AL_BUILD_MDSPLUS_MODELS "Build MDSplus models" ON) +else() + option(AL_BUILD_MDSPLUS_MODELS "Build MDSplus models" OFF) +endif() # Configuration options for python bindings # ############################################################################## @@ -73,13 +77,10 @@ if(NOT PROJECT_VERSION_TWEAK EQUAL 0) message("Building a development version of the Access Layer core") endif() -if(APPLE) +if(APPLE AND CMAKE_OSX_ARCHITECTURES MATCHES "arm64") # Disable MDSPlus: message(STATUS "Disabling MDSPlus backend on macOS") set(AL_BACKEND_MDSPLUS OFF CACHE BOOL "MDSPlus backend" FORCE) - # Disable UDA: - message(STATUS "Disabling UDA backend on macOS") - set(AL_BACKEND_UDA OFF CACHE BOOL "UDA backend" FORCE) endif() @@ -126,6 +127,9 @@ if(WIN32) find_package(dlfcn-win32 CONFIG REQUIRED) endif() +# build AL_CORE library only if backend is enabled +if(AL_BACKEND_HDF5 OR AL_BACKEND_MDSPLUS OR AL_BACKEND_UDA OR AL_BACKEND_UDAFAT OR AL_PYTHON_BINDINGS) + # Core dependencies set(Boost_USE_MULTITHREADED FALSE) find_package( @@ -242,6 +246,8 @@ if(AL_PYTHON_BINDINGS) include(skbuild.cmake) endif() +endif() # AL core library (AL_BACKEND_HDF5 OR AL_BACKEND_MDSPLUS OR AL_BACKEND_UDA OR AL_BACKEND_UDAFAT OR AL_PYTHON_BINDINGS) + # MDSplus models # ############################################################################## diff --git a/README.md b/README.md index d487ef9b..70a0480b 100644 --- a/README.md +++ b/README.md @@ -17,45 +17,35 @@ This repository contains the **Lowlevel components of the Access Layer**: - **MDS+ model logic** for creating the models required by the MDS+ backend -## Quick Installation - -Install IMAS-Core with a single command: - -```bash -pip install imas-core -python -c "import imas_core" -``` - -That's it! No need to compile or configure anything. - -## Features - -- ✅ **Easy to Install** - Single `pip install` command -- ✅ **Multiple Formats** - HDF5, MDSplus, UDA, in-memory, and more -- ✅ **Cross-Platform** - Works on Linux, macOS, and Windows -- ✅ **IMAS Standard** - Access standardized fusion data structures -- ✅ **Read & Write** - Both data access and creation supported - ## Installation Options -### For Python Users (Recommended) +### For Python Users ```bash -# Simple install from PyPI for Python applications +# Install from PyPI pip install imas-core # Verify installation python -c "import imas; print(imas.__version__)" ``` - ### For Developers -See [Building from Source](docs/source/user_guide/installation.rst) for detailed build instructions. +To build IMAS-Core from source: + +```bash +git clone https://github.com/iterorganization/IMAS-Core.git +cd IMAS-Core +cmake -Bbuild -GNinja -DAL_PYTHON_BINDINGS=ON -DCMAKE_INSTALL_PREFIX="$(pwd)/test-install" +cmake --build build --target install +``` + +See [Developer Guide](docs/source/developers/index.rst) for build instructions. + ## Using IMAS-Core with High-Level Languages -When IMAS-Core is built and installed via CMake, it creates a complete runtime environment with: +When IMAS-Core is built and installed via CMake, it installs: - **C/C++ Libraries** (`libal.so`) with full headers - **Python Bindings** (`imas_core` Python package) @@ -74,79 +64,45 @@ export HDF5_USE_FILE_LOCKING=FALSE export PYTHONPATH="/path/to/install/lib/pythonX.X/site-packages:$PYTHONPATH" ``` -Then use IMAS-Core from your preferred language: +Then use IMAS-Core from the required language: - **Python**: `import imas` (see examples above) - **C/C++**: Link against `libal.so` with provided headers - **Fortran**: Use pkg-config to get compiler flags - **Java**: Use `imas.jar` in your classpath - **MATLAB**: Add MEX directory to MATLAB path +See [Building from Source](docs/source/user_guide/installation.rst) for detailed build instructions. ## Documentation -- **[User Guide](docs/source/user_guide/index.rst)** - Complete user documentation +- **[User Guide](docs/source/user_guide/index.rst)** - User documentation - **[Installation Guide](docs/source/user_guide/installation.rst)** - Installation instructions - **[Backends Guide](docs/source/user_guide/backends_guide.rst)** - Available data backends - **[URIs Guide](docs/source/user_guide/uris_guide.rst)** - Data entry URI documentation - **[Configuration](docs/source/user_guide/configuration.rst)** - Configuration options - **[FAQ](docs/source/faq.rst)** - Frequently asked questions -- **[Troubleshooting](docs/source/troubleshooting.rst)** - Common issues & solutions - -## System Requirements +- **[Troubleshooting](docs/source/troubleshooting.rst)** - Troubleshooting -- **Python**: 3.8 or newer -- **OS**: Linux, macOS, or Windows -- **pip**: 19.0 or newer ## Available Backends -IMAS-Core supports multiple data storage formats: +IMAS-Core supports these storage backends: | Backend | Use Case | Remote | File-based | |---------|----------|--------|-----------| | **HDF5** | Default, local storage | No | Yes | | **MDSplus** | ITER experiments | Yes | No | | **UDA** | Distributed access | Yes | No | -| **Memory** | Testing, IPC | No | No | +| **Memory** | Testing | No | No | | **FlexBuffers** | Message passing | No | Yes | | **ASCII** | Debugging | No | Yes | -## Troubleshooting - -**Can't find file?** -```python -# Check file path -import os -print(os.path.exists('/path/to/file.h5')) -``` -**Need help?** +**Help** - See [Troubleshooting Guide](docs/source/troubleshooting.rst) - Check [FAQ](docs/source/faq.rst) - Open an [Issue on GitHub](https://github.com/iterorganization/IMAS-Core/issues) -## What's Included? - -IMAS-Core provides: - -- **Python API** - Full Python bindings with NumPy support -- **Multiple Backends** - HDF5, MDSplus, UDA, and more -- **Data Creation** - Create and populate IMAS IDS structures -- **Data Access** - Read from multiple sources transparently -- **Standard Format** - IMAS standardized data structures - -## For Developers - -To build IMAS-Core from source: - -```bash -git clone https://github.com/iterorganization/IMAS-Core.git -cd IMAS-Core -cmake -Bbuild -GNinja -DAL_PYTHON_BINDINGS=ON -DCMAKE_INSTALL_PREFIX="$(pwd)/test-install" -cmake --build build --target install -``` - -See [Developer Guide](docs/source/developers/index.rst) for detailed instructions. ## Links @@ -165,4 +121,3 @@ IMAS-Core is released under the [LGPL-3.0 License](LICENSE.txt) - **Email**: imas-support@iter.org - **Documentation**: https://imas-core.readthedocs.io/ - **Issues**: https://github.com/iterorganization/IMAS-Core/issues - diff --git a/common/cmake/ALBuildDataDictionary.cmake b/common/cmake/ALBuildDataDictionary.cmake index 7961b623..a14854f2 100644 --- a/common/cmake/ALBuildDataDictionary.cmake +++ b/common/cmake/ALBuildDataDictionary.cmake @@ -38,63 +38,119 @@ else() endif() if( NOT AL_DOWNLOAD_DEPENDENCIES AND NOT AL_DEVELOPMENT_LAYOUT ) - # The DD easybuild module should be loaded, use that module: - # Create Python venv first and install imas_data_dictionary + # Check if imas_data_dictionary is already available in the current Python environment + execute_process( + COMMAND ${PYTHON_EXECUTABLE} -c "import imas_data_dictionary" + RESULT_VARIABLE _IDD_IMPORT_RESULT + OUTPUT_QUIET + ERROR_QUIET + ) + if(NOT EXISTS "${_VENV_PYTHON}") - execute_process( - COMMAND ${PYTHON_EXECUTABLE} -m venv dd_build_env - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - RESULT_VARIABLE _VENV_EXITCODE - OUTPUT_VARIABLE _VENV_OUTPUT - ERROR_VARIABLE _VENV_ERROR - ) - + if(_IDD_IMPORT_RESULT) + # if not available in the current environment create a dd_build_env and pip-install it + execute_process( + COMMAND ${PYTHON_EXECUTABLE} -m venv dd_build_env + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + RESULT_VARIABLE _VENV_EXITCODE + OUTPUT_VARIABLE _VENV_OUTPUT + ERROR_VARIABLE _VENV_ERROR + ) + else() + # IMAS-Data-Dictioanry is already available, just create environment for saxonche by pulling site packages + execute_process( + COMMAND ${PYTHON_EXECUTABLE} -m venv --system-site-packages dd_build_env + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + RESULT_VARIABLE _VENV_EXITCODE + OUTPUT_VARIABLE _VENV_OUTPUT + ERROR_VARIABLE _VENV_ERROR + ) + endif() + if(_VENV_EXITCODE) message(STATUS "venv stdout: ${_VENV_OUTPUT}") message(STATUS "venv stderr: ${_VENV_ERROR}") message(FATAL_ERROR "Failed to create venv (exit code: ${_VENV_EXITCODE}). Ensure Python has venv module installed: python -m venv --help") endif() - - if(DEFINED DD_VERSION) + + if(_IDD_IMPORT_RESULT) + # pip-install imas_data_dictionary only when not already present. + if(DEFINED DD_VERSION AND DD_VERSION MATCHES "^[0-9]") + message(STATUS "Installing imas_data_dictionary==${DD_VERSION} from PyPI") + execute_process( + COMMAND ${_VENV_PIP} install imas_data_dictionary==${DD_VERSION} + RESULT_VARIABLE _PIP_EXITCODE + OUTPUT_VARIABLE _PIP_OUTPUT + ERROR_VARIABLE _PIP_ERROR + ) + else() + if(DEFINED DD_VERSION AND NOT DD_VERSION STREQUAL "") + message(WARNING + "DD_VERSION='${DD_VERSION}' looks like a git ref which can be used when AL_DOWNLOAD_DEPENDECY=OFF. " + "Installing the latest imas_data_dictionary from PyPI. " + "Pass a numeric version (e.g. -DDD_VERSION=4.1.0) to use a specific release.") + else() + message(WARNING + "DD_VERSION is not set. Installing the latest imas_data_dictionary from PyPI. " + "Pass -DDD_VERSION= to use a specific release.") + endif() + execute_process( + COMMAND ${_VENV_PIP} install imas_data_dictionary + RESULT_VARIABLE _PIP_EXITCODE + OUTPUT_VARIABLE _PIP_OUTPUT + ERROR_VARIABLE _PIP_ERROR + ) + endif() + + if(_PIP_EXITCODE) + message(STATUS "imas_data_dictionary pip output: ${_PIP_OUTPUT}") + message(STATUS "imas_data_dictionary pip error: ${_PIP_ERROR}") + message(FATAL_ERROR "Failed to install imas_data_dictionary dependency (exit code: ${_PIP_EXITCODE}). Check network connectivity and Python wheel compatibility.") + endif() + + # Report which version was actually installed execute_process( - COMMAND ${_VENV_PIP} install imas_data_dictionary==${DD_VERSION} - RESULT_VARIABLE _PIP_EXITCODE - OUTPUT_VARIABLE _PIP_OUTPUT - ERROR_VARIABLE _PIP_ERROR + COMMAND ${_VENV_PYTHON} -c + "import importlib.metadata; print(importlib.metadata.version('imas_data_dictionary'))" + OUTPUT_VARIABLE _IDD_INSTALLED_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET ) - else() + if(_IDD_INSTALLED_VERSION) + message(STATUS "Using imas_data_dictionary version ${_IDD_INSTALLED_VERSION}") + endif() + endif() + + execute_process( + COMMAND ${_VENV_PYTHON} -c "import saxonche" + RESULT_VARIABLE _SAXONCHE_CHECK + OUTPUT_QUIET + ERROR_QUIET + ) + if(_SAXONCHE_CHECK) execute_process( - COMMAND ${_VENV_PIP} install imas_data_dictionary + COMMAND ${_VENV_PYTHON} -m pip install saxonche RESULT_VARIABLE _PIP_EXITCODE OUTPUT_VARIABLE _PIP_OUTPUT ERROR_VARIABLE _PIP_ERROR ) - endif() - - if(_PIP_EXITCODE) - message(STATUS "imas_data_dictionary pip output: ${_PIP_OUTPUT}") - message(STATUS "imas_data_dictionary pip error: ${_PIP_ERROR}") - message(FATAL_ERROR "Failed to install imas_data_dictionary dependency (exit code: ${_PIP_EXITCODE}). Check network connectivity and Python wheel compatibility.") - endif() - - execute_process( - COMMAND ${_VENV_PIP} install saxonche - RESULT_VARIABLE _PIP_EXITCODE - OUTPUT_VARIABLE _PIP_OUTPUT - ERROR_VARIABLE _PIP_ERROR - ) - - if(_PIP_EXITCODE) - message(STATUS "saxonche pip output: ${_PIP_OUTPUT}") - message(STATUS "saxonche pip error: ${_PIP_ERROR}") - message(FATAL_ERROR "Failed to install saxonche dependency (exit code: ${_PIP_EXITCODE}). Check network connectivity and Python wheel compatibility.") + + if(_PIP_EXITCODE) + message(STATUS "saxonche pip output: ${_PIP_OUTPUT}") + message(STATUS "saxonche pip error: ${_PIP_ERROR}") + message(FATAL_ERROR "Failed to install saxonche dependency (exit code: ${_PIP_EXITCODE}). Check network connectivity and Python wheel compatibility.") + endif() endif() endif() # Set up idsinfo command path -if(WIN32) - set(_IDSINFO_COMMAND "${CMAKE_CURRENT_BINARY_DIR}/dd_build_env/Scripts/idsinfo.exe") +if(_IDD_IMPORT_RESULT) + if(WIN32) + set(_IDSINFO_COMMAND "${CMAKE_CURRENT_BINARY_DIR}/dd_build_env/Scripts/idsinfo.exe") + else() + set(_IDSINFO_COMMAND "${CMAKE_CURRENT_BINARY_DIR}/dd_build_env/bin/idsinfo") + endif() else() - set(_IDSINFO_COMMAND "${CMAKE_CURRENT_BINARY_DIR}/dd_build_env/bin/idsinfo") + find_program(_IDSINFO_COMMAND NAMES idsinfo REQUIRED) endif() # Use idsinfo idspath command from venv to get the path to IDSDef.xml or data_dictionary.xml diff --git a/common/doc_common/building_installing.rst b/common/doc_common/building_installing.rst index a5571fb3..6394abeb 100644 --- a/common/doc_common/building_installing.rst +++ b/common/doc_common/building_installing.rst @@ -227,9 +227,10 @@ Configuration options - ``AL_BACKEND_MDSPLUS``, allowed values ``ON`` or ``OFF`` *(default)*. Enable/disable the MDSplus backend. - - ``AL_BUILD_MDSPLUS_MODELS``, allowed values ``ON`` *(default)* or ``OFF``, - only available when the MDSplus backend is enabled. Enable building MDSplus - models for the selected Data Dictionary version. + - ``AL_BUILD_MDSPLUS_MODELS``, allowed values ``ON`` or ``OFF``. + Enable building MDSplus models for the selected Data Dictionary version. + Defaults to ``ON`` when ``AL_BACKEND_MDSPLUS`` is enabled, ``OFF`` otherwise. + Can be overridden by user to explicitly enable or disable building MDSplus models. - ``AL_BACKEND_UDA``, allowed values ``ON`` or ``OFF`` *(default)*. Enable/disable the UDA backend. diff --git a/docs/source/user_guide/backends_guide.rst b/docs/source/user_guide/backends_guide.rst index f918a7ea..f40dec61 100644 --- a/docs/source/user_guide/backends_guide.rst +++ b/docs/source/user_guide/backends_guide.rst @@ -52,6 +52,11 @@ This backend imposes some limitations on the data that can be stored, see This backend has been around and stable for a longer time, so most older IMAS data is stored in this format. +To map an existing MDSplus ``imasdb`` from the Access Layer 4 layout to the +Access Layer 5 layout, use ``mdsplusIMASDB4to5``. For example, +``mdsplusIMASDB4to5 --path $HOME/public --database ITER --dry-run`` shows the +directory and link changes without modifying the data. + .. _uda backend: diff --git a/docs/source/user_guide/installation.rst b/docs/source/user_guide/installation.rst index dc2f77bd..9fb4c633 100644 --- a/docs/source/user_guide/installation.rst +++ b/docs/source/user_guide/installation.rst @@ -219,9 +219,10 @@ Configuration options - ``AL_BACKEND_MDSPLUS``, allowed values ``ON`` or ``OFF`` *(default)*. Enable/disable the MDSplus backend. - - ``AL_BUILD_MDSPLUS_MODELS``, allowed values ``ON`` *(default)* or ``OFF``, - only available when the MDSplus backend is enabled. Enable building MDSplus - models for the selected Data Dictionary version. + - ``AL_BUILD_MDSPLUS_MODELS``, allowed values ``ON`` or ``OFF``. + Enable building MDSplus models for the selected Data Dictionary version. + Defaults to ``ON`` when ``AL_BACKEND_MDSPLUS`` is enabled, ``OFF`` otherwise. + Can be overridden by user to explicitly enable or disable building MDSplus models. - ``AL_BACKEND_UDA``, allowed values ``ON`` or ``OFF`` *(default)*. Enable/disable the UDA backend. @@ -240,8 +241,30 @@ Configuration options ONLY the documentation will be built (needs ``AL_HLI_DOCS=ON``). Regardless of other configuration options, nothing else will be built. - ``AL_PYTHON_BINDINGS``, allowed values ``ON`` *(default when building the Python - API)* or ``OFF`` *(default when not building the Python API)*. When enabled, this - builds the Access Layer Python lowlevel bindings. + API)* or ``OFF`` *(default when not building the Python API)*. This CMake option + controls whether and how the `imas_core` Python package is built when installing + through CMake. + + | Value | When to use | + |---|---| + | `OFF` | Default for CMake builds. Build and install only the native IMAS-Core library, without Python bindings. | + | `ON` | Build the `imas_core` wheel using pip's normal build isolation. Build dependencies will be installed by pip. | + | `no-build-isolation` | Build the `imas_core` wheel using the current Python environment. Use this when build dependencies like `numpy` are already installed. | + | `editable` or `e` | Development mode. Install `imas_core` with `pip install --editable`, so Python changes can be used without reinstalling. | + + When CMake installs the Python bindings, it first builds a local wheel and then + installs that wheel with `pip install --no-index --no-deps --find-links + /dist`. + `--no-index` prevents pip from searching PyPI or another package + index, and + `--no-deps` prevents pip from installing or upgrading dependencies. + This keeps the CMake install step local and reproducible: only the wheel built + by this build is installed, and dependencies are expected to be provided by the + user's environment or system installation. + + When not using CMake to build the Python bindings, you can install the `imas_core` with `pip install .` + command without specifying above options. `imas_core` will install with pip's normal build isolation. + - **Dependency configuration options** diff --git a/include/al_backend.h b/include/al_backend.h index afa1cdc8..aec9860f 100644 --- a/include/al_backend.h +++ b/include/al_backend.h @@ -175,6 +175,17 @@ class IMAS_CORE_LIBRARY_API Backend virtual void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) = 0; + /** + Get a list of all Data Dictionary paths for which some data is filled. + This function is only implemented for tensorizing backends (i.e. the HDF5 backend). + @param[in] dataobjectname IDS name and occurrence + @param[in,out] path_list list of c-style strings (ending with a null byte) + @param[in,out] size specify the size of the array (number of elements) + @throw BackendException + */ + virtual void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) = 0; + + /** Returns true if the backend performs time data interpolation (e.g time slices operations or IMAS-3885 with data resampling), false otherwise. **/ diff --git a/include/al_lowlevel.h b/include/al_lowlevel.h index e2dea3a2..e3bd79be 100644 --- a/include/al_lowlevel.h +++ b/include/al_lowlevel.h @@ -490,6 +490,20 @@ extern "C" IMAS_CORE_LIBRARY_API al_status_t al_get_occurrences(int pctxID, const char* ids_name, int** occurrences_list, int* size); + /** + Get a list of Data Dictionary paths (without indices) containing filled data in the backend. + + Notes: + 1. This function is only implemented for tensorizing backends (i.e. the HDF5 backend). + 2. The paths may appear in any order. + 3. The caller is responsible for freeing the list and all strings in it. + + @param[in] dataobjectname IDS name and occurrence, e.g. "core_profiles", "equilibrium/1" + @param[in,out] path_list list of c-style strings (ending with a null byte) + @param[in,out] size specify the size of the array (number of elements) + */ + IMAS_CORE_LIBRARY_API al_status_t al_list_filled_paths(int pctxID, const char* dataobjectname, char*** path_list, int* size); + //IMAS_CORE_LIBRARY_API al_status_t al_close_pulse(int pctxID, int mode, const char *options); //HLI wrappers for plugins API diff --git a/include/al_utilities.h b/include/al_utilities.h new file mode 100644 index 00000000..ed17a4f5 --- /dev/null +++ b/include/al_utilities.h @@ -0,0 +1,24 @@ +#ifndef AL_UTILITIES_H +#define AL_UTILITIES_H + +/** + * General-purpose utility functions for the Access Layer. +*/ + +#include +#include + +namespace utilities { + +/** + * Convert a vector of strings into a C-style array of strings. + * + * @param paths the vector of strings to convert + * @param path_list a pointer to the C-style array to create + * @param size a pointer to an integer to store the size of the created array + */ +void copy_stringvector_to_c_list(const std::vector& paths, char*** path_list, int* size); + +} // namespace utilities + +#endif // AL_UTILITIES_H diff --git a/pyproject.toml b/pyproject.toml index 9a6d0ee5..e0ce230c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,9 +62,14 @@ before-build = "bash ./ci/wheels/cibw_before_build_win.sh" repair-wheel-command = "bash ./ci/wheels/repair_windows.sh {wheel} {dest_dir}" [tool.cibuildwheel.macos.environment] -CMAKE_PREFIX_PATH = "/opt/homebrew;/usr/local" -PKG_CONFIG_PATH = "/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig" +CMAKE_PREFIX_PATH = "/tmp/imas-core-install:/opt/homebrew:/usr/local" +PKG_CONFIG_PATH = "/tmp/imas-core-install/lib/pkgconfig:/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig" MACOSX_DEPLOYMENT_TARGET = "14.0" +AL_BACKEND_HDF5 = { env = "AL_BACKEND_HDF5", default = "ON" } +AL_BACKEND_UDA = { env = "AL_BACKEND_UDA", default = "ON" } +AL_BACKEND_MDSPLUS = { env = "AL_BACKEND_MDSPLUS", default = "OFF" } +AL_BUILD_MDSPLUS_MODELS = { env = "AL_BUILD_MDSPLUS_MODELS", default = "OFF" } +AL_PYTHON_BINDINGS = "ON" [tool.cibuildwheel.macos] archs = ["arm64"] diff --git a/python/imas_core/_al_lowlevel.pyx b/python/imas_core/_al_lowlevel.pyx index 562940d3..5eb16a80 100644 --- a/python/imas_core/_al_lowlevel.pyx +++ b/python/imas_core/_al_lowlevel.pyx @@ -1227,3 +1227,22 @@ def al_get_occurrences(ctx, ids_name): def get_al_version(): version = ll.getALVersion() return version.decode('UTF-8') + + +def al_list_filled_paths(ctx: int, dataobjectname: str): + cdef char** path_list + cdef int cSize + + al_status = ll.al_list_filled_paths(ctx, dataobjectname.encode(), &path_list, &cSize) + if al_status.code < 0: + raise get_proper_exception_class(f'Error while calling al_list_filled_paths: {al_status.message.decode()}', al_status.code) + + # Create python list of strings from the C-style path_list and clean up memory + result = [] + for i in range(cSize): + result.append(path_list[i].decode()) + free(path_list[i]) + if cSize > 0: + free(path_list) + + return al_status.code, result diff --git a/python/imas_core/al_lowlevel_interface.pxd b/python/imas_core/al_lowlevel_interface.pxd index b590795e..4ac9c78a 100644 --- a/python/imas_core/al_lowlevel_interface.pxd +++ b/python/imas_core/al_lowlevel_interface.pxd @@ -54,4 +54,6 @@ cdef extern from "al_lowlevel.h": al_status_t al_get_occurrences(int ctx, const char* ids_name, int **occurrences_list, int *size) + al_status_t al_list_filled_paths(int ctx, const char* dataobjectname, char*** path_list, int *size) + const char* getALVersion() diff --git a/skbuild.cmake b/skbuild.cmake index ab217342..facd0058 100644 --- a/skbuild.cmake +++ b/skbuild.cmake @@ -52,6 +52,7 @@ set_target_properties( install(CODE "execute_process(COMMAND ${Python_EXECUTABLE} -m pip install imas_core --no-index +--no-deps --prefix=${CMAKE_INSTALL_PREFIX} --find-links ${CMAKE_CURRENT_BINARY_DIR}/dist/ )" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2800528e..3aa7a997 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,6 +6,7 @@ target_sources( al PRIVATE al_context.cpp al_const.cpp al_exception.cpp + al_utilities.cpp no_backend.cpp memory_backend.cpp ascii_backend.cpp diff --git a/src/al_lowlevel.cpp b/src/al_lowlevel.cpp index fa7f42ed..93b80798 100644 --- a/src/al_lowlevel.cpp +++ b/src/al_lowlevel.cpp @@ -1775,6 +1775,30 @@ al_status_t al_get_occurrences(int pctxID, const char* ids_name, int** occurrenc return status; } +al_status_t al_list_filled_paths(int pctxID, const char* dataobjectname, char*** path_list, int* size) { + al_status_t status; + + status.code = 0; + try { + LLenv lle = Lowlevel::getLLenv(pctxID); + lle.backend->list_filled_paths(lle.context, dataobjectname, path_list, size); + } + catch (const ALBackendException& e) { + status.code = alerror::backend_err; + ALException::registerStatus(status.message, __func__, e); + } + catch (const ALLowlevelException& e) { + status.code = alerror::lowlevel_err; + ALException::registerStatus(status.message, __func__, e); + } + catch (const std::exception& e) { + status.code = alerror::unknown_err; + ALException::registerStatus(status.message, __func__, e); + } + + return status; +} + al_status_t al_setvalue_parameter_plugin(const char* parameter_name, int datatype, int dim, int *size, void *data, const char* pluginName) { al_status_t status; diff --git a/src/al_utilities.cpp b/src/al_utilities.cpp new file mode 100644 index 00000000..f2faa4c7 --- /dev/null +++ b/src/al_utilities.cpp @@ -0,0 +1,21 @@ +#include "al_utilities.h" + +#include +#include + +namespace utilities { + +void copy_stringvector_to_c_list(const std::vector& paths, char*** path_list, int* size) { + *size = static_cast(paths.size()); + + if (*size == 0) { + *path_list = nullptr; + return; + } + + *path_list = static_cast(malloc(*size * sizeof(char*))); + for (int i = 0; i < *size; ++i) { + (*path_list)[i] = strdup(paths[i].c_str()); + } +} +} // namespace utilities \ No newline at end of file diff --git a/src/ascii_backend.cpp b/src/ascii_backend.cpp index 40d943ed..2fa9be03 100644 --- a/src/ascii_backend.cpp +++ b/src/ascii_backend.cpp @@ -708,4 +708,7 @@ void AsciiBackend::get_occurrences(Context* ctx, const char* ids_name, int** oc (*occurrences_list)[i] = occurrences[i]; } +void AsciiBackend::list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) { + throw ALBackendException("list_filled_paths is not implemented in the ASCII Backend", LOG); +} diff --git a/src/ascii_backend.h b/src/ascii_backend.h index fe884e5c..8ccbd26d 100644 --- a/src/ascii_backend.h +++ b/src/ascii_backend.h @@ -91,6 +91,7 @@ class IMAS_CORE_LIBRARY_API AsciiBackend : public Backend std::pair getVersion(DataEntryContext *ctx) override; void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override; + void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override; bool supportsTimeDataInterpolation() override { return false; diff --git a/src/flexbuffers_backend.cpp b/src/flexbuffers_backend.cpp index 32f71811..75494803 100644 --- a/src/flexbuffers_backend.cpp +++ b/src/flexbuffers_backend.cpp @@ -398,6 +398,15 @@ void FlexbuffersBackend::get_occurrences( throw ALBackendException("get_occurrences is not implemented in the Serialize Backend", LOG); } +void FlexbuffersBackend::list_filled_paths( + Context* ctx, + const char* dataobjectname, + char*** path_list, + int* size +) { + throw ALBackendException("list_filled_paths is not implemented in the Serialize Backend", LOG); +} + void FlexbuffersBackend::_start_vector() { _vector_starts.push(_builder->StartVector()); } diff --git a/src/flexbuffers_backend.h b/src/flexbuffers_backend.h index 5517cb2e..7471be24 100644 --- a/src/flexbuffers_backend.h +++ b/src/flexbuffers_backend.h @@ -48,6 +48,7 @@ class IMAS_CORE_LIBRARY_API FlexbuffersBackend : public Backend void deleteData(OperationContext *ctx, std::string path) override; void beginArraystructAction(ArraystructContext *ctx, int *size) override; void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override; + void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override; // timerange (get_sample) API is not supported: bool supportsTimeDataInterpolation() override { return false; } bool supportsTimeRangeOperation() override { return false; } diff --git a/src/hdf5/hdf5_backend.cpp b/src/hdf5/hdf5_backend.cpp index 2e4112e7..cb95f539 100644 --- a/src/hdf5/hdf5_backend.cpp +++ b/src/hdf5/hdf5_backend.cpp @@ -162,3 +162,11 @@ void HDF5Backend::get_occurrences(Context* ctx, const char* ids_name, int** occ throw ALBackendException("HDF5Backend: master file not opened while calling HDF5Backend::get_occurrences()", LOG); hdf5Reader->get_occurrences(ids_name, occurrences_list, size, file_id); } + +void HDF5Backend::list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) +{ + if (file_id == -1) // master file not opened + throw ALBackendException("HDF5Backend: master file not opened while calling HDF5Backend::list_filled_paths()", LOG); + + hdf5Reader->list_filled_paths(dataobjectname, path_list, size, file_id, opened_IDS_files, files_directory, relative_file_path); +} diff --git a/src/hdf5/hdf5_backend.h b/src/hdf5/hdf5_backend.h index 26bdc001..96cc1e60 100644 --- a/src/hdf5/hdf5_backend.h +++ b/src/hdf5/hdf5_backend.h @@ -131,6 +131,7 @@ class HDF5Backend:public Backend { void beginAction(OperationContext * ctx) override; void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override; + void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override; bool supportsTimeDataInterpolation() override { return true; diff --git a/src/hdf5/hdf5_reader.cpp b/src/hdf5/hdf5_reader.cpp index 4dde86c7..a4a553b9 100644 --- a/src/hdf5/hdf5_reader.cpp +++ b/src/hdf5/hdf5_reader.cpp @@ -8,6 +8,7 @@ #include #include #include +#include "al_utilities.h" #define MAX_LENGTH 200 #define HOMOGENEOUS_TIME_FIELD_NAME "ids_properties&homogeneous_time" @@ -1568,6 +1569,45 @@ void HDF5Reader::get_occurrences(const char *ids_name, int **occurrences_list, i p[i] = occurrences[i]; } +void HDF5Reader::list_filled_paths(const char* dataobjectname, char*** path_list, int* size, hid_t file_id, std::unordered_map < std::string, hid_t > &opened_IDS_files, std::string & files_directory, std::string & relative_file_path) +{ + HDF5Utils hdf5_utils; + // Create temporary OperationContext to allow reusing existing logic from hdf5_utils: + OperationContext ctx(NULL, dataobjectname, "", READ_OP); + hid_t gid = -1; + hdf5_utils.open_IDS_group(&ctx, file_id, opened_IDS_files, files_directory, relative_file_path, &gid); + if (gid <= 0) { + // Group does not exist => nothing is filled + *size = 0; + return; + } + + // Create a vector of all link names in the hdf5 group + std::vector variables; + H5L_iterate_t iterate_callback = [](hid_t group_id, const char* cname, const H5L_info_t* info, void* op_data) -> herr_t { + std::string name(cname); + // Skip if name ends with _SHAPE + if (name.size() < 6 || name.substr(name.size() - 6) != "_SHAPE") + static_cast*>(op_data)->push_back(name); + return 0; // success + }; + herr_t status = H5Literate(gid, H5_INDEX_NAME, H5_ITER_NATIVE, NULL, iterate_callback, &variables); + if (status != 0) + throw ALBackendException("HDF5Backend: H5Literate has failed in HDF5Reader::list_filled_paths()", LOG); + + // Convert to DD path: remove AoS brackets ([]) and use / to separate paths + for (auto &str : variables) { + // Remove square brackets + str.erase(std::remove(str.begin(), str.end(), '['), str.end()); + str.erase(std::remove(str.begin(), str.end(), ']'), str.end()); + // Replace '&' with '/' + std::replace(str.begin(), str.end(), '&', '/'); + } + + // Create the C-style array: + utilities::copy_stringvector_to_c_list(variables, path_list, size); +} + herr_t HDF5Reader::iterate_callback(hid_t loc_id, const char *name, const H5L_info_t *info, void *callback_data) { std::vector *op = reinterpret_cast *>(callback_data); diff --git a/src/hdf5/hdf5_reader.h b/src/hdf5/hdf5_reader.h index 80ebeaba..a00874cd 100644 --- a/src/hdf5/hdf5_reader.h +++ b/src/hdf5/hdf5_reader.h @@ -89,6 +89,7 @@ class HDF5Reader { virtual int read_ND_Data(Context * ctx, std::string & att_name, std::string & timebasename, int datatype, void **data, int *dim, int *size); virtual void beginReadArraystructAction(ArraystructContext * ctx, int *size); virtual void get_occurrences(const char* ids_name, int** occurrences_list, int* size, hid_t master_file_id); + void list_filled_paths(const char* dataobjectname, char*** path_list, int* size, hid_t file_id, std::unordered_map < std::string, hid_t > &opened_IDS_files, std::string & files_directory, std::string & relative_file_path); void open_IDS_group(OperationContext * ctx, hid_t file_id, std::unordered_map < std::string, hid_t > &opened_IDS_files, std::string & files_directory, std::string & relative_file_path); void close_file_handler(std::string external_link_name, std::unordered_map < std::string, hid_t > &opened_IDS_files); diff --git a/src/hdf5/hdf5_utils.cpp b/src/hdf5/hdf5_utils.cpp index c76352a9..f945ce62 100644 --- a/src/hdf5/hdf5_utils.cpp +++ b/src/hdf5/hdf5_utils.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include using namespace boost::filesystem; @@ -75,24 +76,33 @@ int throw ALBackendException(error_message, LOG); } - hid_t dtype_id = H5Tcopy (H5T_C_S1); - H5Tset_size(dtype_id, strlen(backend_version_attribute_name)); - - herr_t tset = H5Tset_cset(dtype_id, H5T_CSET_UTF8); - if (tset < 0) { - char error_message[100]; - sprintf(error_message, "Unable to set characters to UTF8 for: %s\n", backend_version_attribute_name); - throw ALBackendException(error_message); + hid_t attr_file_type = H5Aget_type(att_id); + if (attr_file_type < 0) { + H5Aclose(att_id); + char error_message[200]; + sprintf(error_message, "Unable to get type of attribute: %s\n", backend_version_attribute_name); + throw ALBackendException(error_message, LOG); } + size_t attr_size = H5Tget_size(attr_file_type); + H5T_cset_t attr_cset = H5Tget_cset(attr_file_type); + H5Tclose(attr_file_type); + + hid_t dtype_id = H5Tcopy(H5T_C_S1); + H5Tset_size(dtype_id, attr_size + 1); + H5Tset_strpad(dtype_id, H5T_STR_NULLTERM); + H5Tset_cset(dtype_id, attr_cset); - char version[10]; - herr_t status = H5Aread(att_id, dtype_id, version); + std::string version(attr_size + 1, '\0'); + herr_t status = H5Aread(att_id, dtype_id, version.data()); if (status < 0) { + H5Tclose(dtype_id); + H5Aclose(att_id); char error_message[200]; sprintf(error_message, "Unable to read attribute: %s\n", backend_version_attribute_name); throw ALBackendException(error_message, LOG); } - backend_version = std::string(version); + version[attr_size] = '\0'; + backend_version = version.c_str(); H5Tclose(dtype_id); H5Aclose(att_id); } else { @@ -344,8 +354,8 @@ void HDF5Utils::writeHeader(DataEntryContext * ctx, hid_t file_id, std::string & //write version to file hid_t dataspace_id = H5Screate(H5S_SCALAR); - hid_t dtype_id = H5Tcopy (H5T_C_S1); - H5Tset_size(dtype_id, strlen(backend_version_attribute_name)); + hid_t dtype_id = H5Tcopy(H5T_C_S1); + H5Tset_size(dtype_id, backend_version.length() + 1); herr_t tset = H5Tset_cset(dtype_id, H5T_CSET_UTF8); if (tset < 0) { char error_message[200]; diff --git a/src/mdsplus/mdsplus_backend.cpp b/src/mdsplus/mdsplus_backend.cpp index a0c73f1b..fa1fa713 100644 --- a/src/mdsplus/mdsplus_backend.cpp +++ b/src/mdsplus/mdsplus_backend.cpp @@ -4496,7 +4496,12 @@ std::string MDSplusBackend::getTimedNode(ArraystructContext *ctx, std::string fu } catch(MDSplus::MdsException &exc) { if (mode!=alconst::force_open_pulse) { resetIdsPath(szTree); - throw ALBackendException(exc.what()+mdsplusBaseStr,LOG); + std::string hint; + const char *modelsPath = getenv("MDSPLUS_MODELS_PATH"); + if (!modelsPath || !*modelsPath) { + hint = " -- MDSPLUS_MODELS_PATH is not set; set it to the MDSplus models directory (e.g. export MDSPLUS_MODELS_PATH=/path/to/models/mdsplus)"; + } + throw ALBackendException(exc.what()+mdsplusBaseStr+hint,LOG); } } case alconst::create_pulse: @@ -4509,6 +4514,17 @@ std::string MDSplusBackend::getTimedNode(ArraystructContext *ctx, std::string fu } catch (const std::exception& exc) { throw ALBackendException("Unable to create data-entry directory: "+mdsplusBaseStr,LOG); } + { + const char *modelsPath = getenv("MDSPLUS_MODELS_PATH"); + if (!modelsPath || !*modelsPath) { + resetIdsPath(szTree); + throw ALBackendException( + std::string("MDSPLUS_MODELS_PATH is not set. Set it to the MDSplus models directory " + "(e.g. export MDSPLUS_MODELS_PATH=/path/to/models/mdsplus) " + "before creating a pulse file."), + LOG); + } + } try { MDSplus::Tree *modelTree = new MDSplus::Tree(szTree, -1, DEF_READONLYMODE); modelTree->createPulse(shotNum); @@ -4517,7 +4533,12 @@ std::string MDSplusBackend::getTimedNode(ArraystructContext *ctx, std::string fu saveVersion(tree); } catch(MDSplus::MdsException &exc) { resetIdsPath(szTree); - throw ALBackendException(exc.what()+mdsplusBaseStr,LOG); + std::string hint; + const char *modelsPath = getenv("MDSPLUS_MODELS_PATH"); + if (!modelsPath || !*modelsPath) { + hint = " -- MDSPLUS_MODELS_PATH is not set; set it to the MDSplus models directory (e.g. export MDSPLUS_MODELS_PATH=/path/to/models/mdsplus)"; + } + throw ALBackendException(exc.what()+mdsplusBaseStr+hint,LOG); } break; default: @@ -4932,6 +4953,10 @@ void MDSplusBackend::get_occurrences(Context* ctx, const char* ids_name, int** o *size = occurrences.size(); } +void MDSplusBackend::list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) { + throw ALBackendException("list_filled_paths is not implemented in the MDSplus Backend", LOG); +} + void MDSplusBackend::fullPath(Context *ctx, std::string &path) { if (ctx->getType() == CTX_PULSE_TYPE) { path = ""; diff --git a/src/mdsplus/mdsplus_backend.h b/src/mdsplus/mdsplus_backend.h index 75617518..a9dcb524 100644 --- a/src/mdsplus/mdsplus_backend.h +++ b/src/mdsplus/mdsplus_backend.h @@ -250,6 +250,7 @@ class IMAS_CORE_LIBRARY_API MDSplusBackend:public Backend std::pair getVersion(DataEntryContext *ctx) override; void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override; + void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override; bool supportsTimeDataInterpolation() { return true; diff --git a/src/memory_backend.cpp b/src/memory_backend.cpp index 9b380fb3..8009d20e 100644 --- a/src/memory_backend.cpp +++ b/src/memory_backend.cpp @@ -1148,7 +1148,10 @@ else throw ALBackendException(message, LOG); } - + void MemoryBackend::list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) { + std::string message("list_filled_paths() is not implemented in the MemoryBackend"); + throw ALBackendException(message, LOG); + } ////////////////////////////////////////////////////////////////////////// diff --git a/src/memory_backend.h b/src/memory_backend.h index e74bcb92..c0d22ec9 100644 --- a/src/memory_backend.h +++ b/src/memory_backend.h @@ -673,6 +673,7 @@ class IMAS_CORE_LIBRARY_API MemoryBackend:public Backend ALData *getAlSlice(ArraystructContext *ctx, ALData &inData, double time, std::vector timebaseV); void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override; + void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override; bool supportsTimeDataInterpolation() override { return false; diff --git a/src/no_backend.cpp b/src/no_backend.cpp index 2b3807cb..222f21c2 100644 --- a/src/no_backend.cpp +++ b/src/no_backend.cpp @@ -85,3 +85,11 @@ void NoBackend::get_occurrences(Context* ctx, const char* ids_name, int** occurr *size = 0; } +void NoBackend::list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) +{ + if (verbose) { + std::string message("list_filled_paths() is not implemented in the NoBackend"); + throw ALBackendException(message, LOG); + } + *size = 0; +} diff --git a/src/no_backend.h b/src/no_backend.h index 8b46e6b2..2162632a 100644 --- a/src/no_backend.h +++ b/src/no_backend.h @@ -64,6 +64,8 @@ class IMAS_CORE_LIBRARY_API NoBackend : public Backend void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override; + void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override; + bool supportsTimeDataInterpolation() override { return false; } diff --git a/src/uda/uda_backend.cpp b/src/uda/uda_backend.cpp index b1ccb81b..1bb0a868 100644 --- a/src/uda/uda_backend.cpp +++ b/src/uda/uda_backend.cpp @@ -17,6 +17,7 @@ #include #include +#include "al_utilities.h" using namespace semver::literals; @@ -1258,6 +1259,54 @@ std::vector read_occurrences(NodeReader *node) { return occurrences; } +std::vector read_filled_paths(NodeReader *node) { + std::vector paths; + + // Read the shape + std::vector shape_vec(1); + uda_capnp_read_shape(node, shape_vec.data()); + const size_t total_bytes = std::accumulate(shape_vec.begin(), shape_vec.end(), 1, std::multiplies()); + + // For empty buffers, return empty list + if (total_bytes == 0) { + return paths; + } + + const char *name = uda_capnp_read_name(node); + if (name != nullptr && std::string(name) != "filled_paths") { + throw imas::uda::CacheException("Invalid node: " + std::string(name)); + } + + bool eos = uda_capnp_read_is_eos(node); + if (!eos) { + throw imas::uda::CacheException("UDA backend does not currently handle streamed data"); + } + + size_t num_slices = uda_capnp_read_num_slices(node); + if (num_slices != 1) { + throw imas::uda::CacheException("Incorrect number of slices for filled_paths node"); + } + + // Read the data slice + size_t slice_size = uda_capnp_read_slice_size(node, 0); + if (slice_size != total_bytes) { + throw imas::uda::CacheException("Slice size does not match total bytes for filled_paths"); + } + + std::vector buffer(total_bytes); + uda_capnp_read_data(node, 0, buffer.data()); + + const char* ptr = buffer.data(); + const char* end = ptr + total_bytes; + + while (ptr < end && *ptr != '\0') { + std::string path(ptr); + paths.push_back(path); + ptr += strlen(ptr) + 1; + } + + return paths; +} } // anon namespace void UDABackend::get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) { @@ -1283,6 +1332,7 @@ void UDABackend::get_occurrences(Context* ctx, const char* ids_name, int** occur ss << plugin_ << "::getOccurrences(" << "uri='" << uri << "'" + << ", mode='" << imas::uda::convert_imas_to_uda(OPEN_PULSE) << "'" << ", ids='" << ids_name << "'" << ")"; @@ -1310,3 +1360,69 @@ void UDABackend::get_occurrences(Context* ctx, const char* ids_name, int** occur } } +void UDABackend::list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) { + + *size = 0; + *path_list = nullptr; + + if (access_local_) { + return local_backend_->list_filled_paths(ctx, dataobjectname, path_list, size); + } + + if (verbose_) { + std::cout << "UDABackend list_filled_paths\n"; + } + + auto query = ctx->getURI().query; + std::string backend = get_backend(query); + if (backend != "hdf5") { + throw ALException("UDABackend only supports HDF5 backend for list_filled_paths API", LOG); + } + + query.remove("backend"); + query.remove("cache_mode"); + query.remove("verbose"); + std::string dd_version = query.get("dd_version").value_or(dd_version_); + query.set("dd_version", dd_version); + std::string uri = "imas:" + backend + "?" + query.to_string(); + + std::stringstream ss; + + ss << plugin_ + << "::listFilledPaths(" + << "uri='" << uri << "'" + << ", mode='" << imas::uda::convert_imas_to_uda(OPEN_PULSE) << "'" + << ", ids='" << dataobjectname << "'" + << ")"; + + const std::string directive = ss.str(); + + if (verbose_) { + std::cout << "UDABackend request: " << directive << "\n"; + } + + try { + const uda::Result& result = uda_client_.get(directive, ""); + + if (result.errorCode() == 0 && result.uda_type() == UDA_TYPE_CAPNP) { + const char* data = result.raw_data(); + const size_t result_size = result.size(); + + // If buffer is empty, return empty path list + if (result_size == 0) { + return; + } + + const auto tree = uda_capnp_deserialise(data, result_size); + const auto root = uda_capnp_read_root(tree); + + auto paths = read_filled_paths(root); + // Allocate and copy paths to C list + utilities::copy_stringvector_to_c_list(paths, path_list, size); + + uda_capnp_free_tree_reader(tree); + } + } catch (const uda::UDAException& ex) { + throw ALException(ex.what(), LOG); + } +} diff --git a/src/uda/uda_backend.h b/src/uda/uda_backend.h index 0869aa3c..115e8d5b 100644 --- a/src/uda/uda_backend.h +++ b/src/uda/uda_backend.h @@ -183,6 +183,7 @@ class IMAS_CORE_LIBRARY_API UDABackend : public Backend std::pair getVersion(DataEntryContext *ctx) override; void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override; + void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override; bool supportsTimeDataInterpolation() override;