From 6f5e0ed278330dfa17b5b4a955200504d2d848f0 Mon Sep 17 00:00:00 2001 From: Nghia Dam Date: Tue, 6 Oct 2026 12:59:10 -0400 Subject: [PATCH 1/5] launch_conformance_tests.sh: select the SSH key by name Add --key-name for the key file name. --key-path is now the directory that holds it (default: ~/.ssh). Update the README to match, and to explain that REFERENCE_APPLICATION selects the reference application from the community submodule. Change-Id: Ic2b9afd60437562f3a6c8c95b066731c17654532 Signed-off-by: Nghia Dam --- README.md | 9 ++++++--- launch_conformance_tests.sh | 27 +++++++++++++++++++++------ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index b4a30245..351e637d 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,8 @@ Optional options: * `--user `: The username for the target device (default: root) * `--password `: Password fallback used only when SSH key authentication fails -* `--key-path `: Optional SSH private key path. Without it, standard OpenSSH key locations and agents are used +* `--key-name `: Optional SSH private key file name. Without it, standard OpenSSH key locations and agents are used +* `--key-path `: Directory holding the key given by `--key-name` (default: `~/.ssh`) * `--no-reports` : Do not generate test reports (only run tests and display results) * `--no-reboot`: Do not reset the board, skipping the tests that require a reboot * `--baudrate `: The baudrate for the serial port of the board (default: 115200) @@ -91,8 +92,10 @@ when testing another application layout. It is independent of `LXC_PATH`. Each container resource setting is a separate absolute target-side path. The defaults locate the reference application's resources under `LXC_PATH`. Override the settings individually when testing another layout. -The `geisa-simple` reference application is pinned in the -`src/GEISA-LEE-tests/src/community` submodule. +The reference application sources are pinned in the +`src/GEISA-LEE-tests/src/community` submodule. `REFERENCE_APPLICATION` +selects one, which is built and installed with the manifest kept in its +folder. The SSH launcher transfers only the Cukinia runner and the LEE test runtime files. Reference application sources remain on the host. When reference diff --git a/launch_conformance_tests.sh b/launch_conformance_tests.sh index 9c2c1495..8c1283dc 100755 --- a/launch_conformance_tests.sh +++ b/launch_conformance_tests.sh @@ -52,7 +52,8 @@ or Optional options: --user Specify the username for the target device (default: root) --password Specify the password for the target device (default: empty) - --key-path Specify the SSH private key path (optional) + --key-name Specify the SSH private key file name to use (optional, default: ssh default keys) + --key-path Specify the directory holding the SSH key (default: ~/.ssh, requires --key-name) --clean-up Clean selected test artifacts from the board and exit --no-reports Do not generate test reports (only run tests and display results) --no-reboot Do not reset the board, skipping the tests that require a reboot @@ -116,6 +117,14 @@ while [[ "$#" -gt 0 ]]; do fi shift 2 ;; + --key-name) + SSH_KEY_NAME="$2" + if [[ -z "${SSH_KEY_NAME}" ]]; then + echo -e "${RED}Error:${ENDCOLOR} SSH key name cannot be empty" + usage + fi + shift 2 + ;; --key-path) SSH_KEY_PATH="$2" if [[ -z "${SSH_KEY_PATH}" ]]; then @@ -202,13 +211,19 @@ while [[ "$#" -gt 0 ]]; do esac done -if [[ -n "${SSH_KEY_PATH:-}" ]]; then - if [[ ! -f "${SSH_KEY_PATH}" ]]; then - echo -e "${RED}Error:${ENDCOLOR} SSH key not found: ${SSH_KEY_PATH}" +if [[ -n "${SSH_KEY_PATH:-}" && -z "${SSH_KEY_NAME:-}" ]]; then + echo -e "${RED}Error:${ENDCOLOR} --key-path requires --key-name" + exit 1 +fi + +if [[ -n "${SSH_KEY_NAME:-}" ]]; then + ssh_key="${SSH_KEY_PATH:-${HOME}/.ssh}/${SSH_KEY_NAME}" + if [[ ! -f "${ssh_key}" ]]; then + echo -e "${RED}Error:${ENDCOLOR} SSH key not found: ${ssh_key}" exit 1 fi - CONFORMANCE_SSH_ARGS="${CONFORMANCE_SSH_ARGS:-} -i ${SSH_KEY_PATH}" - CONFORMANCE_SCP_ARGS="${CONFORMANCE_SCP_ARGS:-} -i ${SSH_KEY_PATH}" + CONFORMANCE_SSH_ARGS="${CONFORMANCE_SSH_ARGS:-} -i ${ssh_key}" + CONFORMANCE_SCP_ARGS="${CONFORMANCE_SCP_ARGS:-} -i ${ssh_key}" fi BOARD_USER=${BOARD_USER:-root} From c8545f0696ece43640dca560199252fc2b5a905b Mon Sep 17 00:00:00 2001 From: Nghia Dam Date: Tue, 6 Oct 2026 12:59:49 -0400 Subject: [PATCH 2/5] launch_glee_conformance_tests_ssh.sh: send the cleanup configuration The final cleanup read the configuration from the board, and failed when it was missing, for example after a reboot. Send it from the host. Also print the persistence marker before the reboot tests. Change-Id: I6911e39f69042656e2016d5bc10dbfa12c10e10a Signed-off-by: Nghia Dam --- src/launch_glee_conformance_tests_ssh.sh | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/launch_glee_conformance_tests_ssh.sh b/src/launch_glee_conformance_tests_ssh.sh index cfb553e4..fa4928bb 100755 --- a/src/launch_glee_conformance_tests_ssh.sh +++ b/src/launch_glee_conformance_tests_ssh.sh @@ -289,7 +289,7 @@ prepare_glee_reboot_with_ssh() { marker_hex=$(od -An -N16 -tx1 /dev/urandom) || return 1 marker=".geisa-${marker_hex// /}" echo "" - echo "Seeding persistent storage before board reset" + echo "Preparing board for LEE reboot tests with marker: ${marker}" SSH "set -a; . /tmp/conformance_tests/GEISA-LEE-tests/tests_configuration.conf; \ . /tmp/conformance_tests/GEISA-LEE-tests/user_configuration.conf; set +a; \ LEE_TESTS_DIR=/tmp/conformance_tests/GEISA-LEE-tests; \ @@ -385,6 +385,15 @@ cleanup_glee_ssh() { local app_name cleanup_input app_dir status=0 app_dir="${LXC_PATH}/${REFERENCE_APPLICATION}" + # The cleanup scripts load the local test configuration, since the board + # copy may be gone. + print_cleanup_configuration() { + echo "set -a" + cat "${topdir}/src/GEISA-LEE-tests/tests_configuration.conf" + cat "${topdir}/src/GEISA-LEE-tests/user_configuration.conf" + echo "set +a" + } + print_cleanup_paths() { local resolver label path printf ' Application directory: %s\n' "${app_dir}" @@ -412,10 +421,7 @@ cleanup_glee_ssh() { for app_name in ${APPLICATIONS}; do if [[ "${app_name}" == "${REFERENCE_APPLICATION}" ]]; then cleanup_input="$({ - echo "set -a" - cat "${topdir}/src/GEISA-LEE-tests/tests_configuration.conf" - cat "${topdir}/src/GEISA-LEE-tests/user_configuration.conf" - echo "set +a" + print_cleanup_configuration echo "export_application_layout '${app_name}'" echo "\${PRIVILEGED_COMMAND} python3 - uninstall '${app_name}' <<'PYTHON'" cat "${topdir}/src/GEISA-LEE-tests/manage_package.py" @@ -424,9 +430,11 @@ cleanup_glee_ssh() { SSH "sh -s" <<< "${cleanup_input}" || status=1 fi done - SSH "set -a; . /tmp/conformance_tests/GEISA-LEE-tests/tests_configuration.conf; \ - . /tmp/conformance_tests/GEISA-LEE-tests/user_configuration.conf; set +a; \ - \${PRIVILEGED_COMMAND} rm -rf /tmp/conformance_tests '${APPLICATION_MANIFEST_PATH}' '${LXC_PATH}'" || status=1 + cleanup_input="$({ + print_cleanup_configuration + echo "\${PRIVILEGED_COMMAND} rm -rf /tmp/conformance_tests '${APPLICATION_MANIFEST_PATH}' '${LXC_PATH}'" + })" + SSH "sh -s" <<< "${cleanup_input}" || status=1 if [[ ${status} -ne 0 ]]; then echo -e "${RED}Error:${ENDCOLOR} Failed to clean up LEE test artifacts on board" echo "Manual cleanup required: de-provision '${REFERENCE_APPLICATION}' and unmount its filesystems, then remove '${app_dir}' and '/tmp/conformance_tests' on the board." From 8a3f21798814edb3defc7fc992f43830ccc10103 Mon Sep 17 00:00:00 2001 From: Nghia Dam Date: Wed, 30 Sep 2026 10:14:53 -0400 Subject: [PATCH 3/5] GEISA-LEE-tests: use geisa-test as the reference application geisa-test supersedes geisa-simple as the reference application. Unlike geisa-simple, it can load the CPU, allocate memory, write to storage and open network connections on demand, which the tests need to check that the platform enforces the manifest limits. Change-Id: Ibaaf6b38b91e1a32037d717d20478c879f86eb91 Signed-off-by: Nghia Dam --- README.md | 7 +-- src/GEISA-LEE-tests/geisa-app.Dockerfile | 39 ++++++++++++++ .../geisa-reference.Dockerfile | 22 -------- src/GEISA-LEE-tests/manage_package.py | 19 ++++--- src/GEISA-LEE-tests/src/community | 2 +- .../tests.d/build-reference-application.sh | 52 ++++++++++++++++--- .../tests.d/check-container.sh | 6 ++- .../tests.d/check-lxc-container.py | 2 +- src/GEISA-LEE-tests/tests_configuration.conf | 16 ++++-- src/GEISA-LEE-tests/user_configuration.conf | 18 +++++-- src/launch_glee_conformance_tests_ssh.sh | 20 ++++--- 11 files changed, 145 insertions(+), 58 deletions(-) create mode 100644 src/GEISA-LEE-tests/geisa-app.Dockerfile delete mode 100644 src/GEISA-LEE-tests/geisa-reference.Dockerfile diff --git a/README.md b/README.md index 351e637d..19da0c4d 100644 --- a/README.md +++ b/README.md @@ -94,12 +94,13 @@ defaults locate the reference application's resources under `LXC_PATH`. Override the settings individually when testing another layout. The reference application sources are pinned in the `src/GEISA-LEE-tests/src/community` submodule. `REFERENCE_APPLICATION` -selects one, which is built and installed with the manifest kept in its -folder. +selects one. Its own `scripts/package.py` packages it as an appoverlay `.tgz` +with the manifest kept in its folder, and it is installed under the name that +manifest gives. The SSH launcher transfers only the Cukinia runner and the LEE test runtime files. Reference application sources remain on the host. When reference -provisioning is selected, only the built application image and its complete +provisioning is selected, only the application package and its complete manifest are transferred to the board. GADM test options (optional): diff --git a/src/GEISA-LEE-tests/geisa-app.Dockerfile b/src/GEISA-LEE-tests/geisa-app.Dockerfile new file mode 100644 index 00000000..08cc4cb5 --- /dev/null +++ b/src/GEISA-LEE-tests/geisa-app.Dockerfile @@ -0,0 +1,39 @@ +# Copyright (C) 2026 Southern California Edison + +FROM debian:bookworm AS builder + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + git \ + libmosquitto-dev \ + pkg-config \ + protobuf-compiler \ + python3 \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /src +COPY . . +# A reference application builds build/ or build/, +# where SOURCE_NAME is its folder name. It is shipped as /, next to +# /toolchain, which gives the toolchain id and version for its manifest. +ARG APP_BINARY=geisa-app +ARG SOURCE_NAME=geisa-app +# geisa-test only: its probe target is capped by the #define MAX_PROBE_TARGET_BYTES +# (64 MiB) that -D cannot override, so the cap is rewritten in place before the build. +# The probe smoke test hardcodes the default cap, so make test is skipped in that case. +ARG PROBE_TARGET_BYTES= +RUN if [ -n "${PROBE_TARGET_BYTES}" ]; then \ + sed -i "s/^#define MAX_PROBE_TARGET_BYTES .*/#define MAX_PROBE_TARGET_BYTES ${PROBE_TARGET_BYTES}/" src/geisa-test.c && \ + grep -q "^#define MAX_PROBE_TARGET_BYTES ${PROBE_TARGET_BYTES}$" src/geisa-test.c; \ + fi && \ + make setup-dev && \ + make && \ + if [ -z "${PROBE_TARGET_BYTES}" ]; then make test; fi && \ + if [ ! -f "build/${APP_BINARY}" ]; then cp "build/${SOURCE_NAME}" "build/${APP_BINARY}"; fi && \ + printf '%s\t%s\n' "$(cc -dumpmachine)" "$(cc -dumpfullversion)" > build/toolchain + +FROM scratch +ARG APP_BINARY=geisa-app +COPY --from=builder /src/build/${APP_BINARY} /${APP_BINARY} +COPY --from=builder /src/build/toolchain /toolchain diff --git a/src/GEISA-LEE-tests/geisa-reference.Dockerfile b/src/GEISA-LEE-tests/geisa-reference.Dockerfile deleted file mode 100644 index df180a2b..00000000 --- a/src/GEISA-LEE-tests/geisa-reference.Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (C) 2026 Southern California Edison - -FROM debian:bookworm AS builder - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - build-essential \ - git \ - libmosquitto-dev \ - pkg-config \ - protobuf-compiler \ - python3 \ - python3-venv \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /src -COPY . . -RUN make setup-dev && \ - make && \ - make test - -FROM scratch -COPY --from=builder /src/build/geisa-app /geisa-app diff --git a/src/GEISA-LEE-tests/manage_package.py b/src/GEISA-LEE-tests/manage_package.py index e11b0964..e77f6777 100644 --- a/src/GEISA-LEE-tests/manage_package.py +++ b/src/GEISA-LEE-tests/manage_package.py @@ -13,6 +13,7 @@ import shutil import subprocess import sys +import tarfile from pathlib import Path @@ -410,7 +411,11 @@ def mount_application(metadata: dict) -> None: if not mounted(item["base"]): run("mount", "-t", IMAGE_FSTYPE, "-o", "loop,ro", metadata["base_image"], str(item["base"])) if not mounted(item["application"]): - run("mount", "-t", IMAGE_FSTYPE, "-o", "loop,ro", str(item["image"]), str(item["application"])) + # The appoverlay package is unpacked into its own read-only layer. + run("mount", "-t", "tmpfs", "-o", "mode=0755", "tmpfs", str(item["application"])) + with tarfile.open(item["image"], "r:gz") as package: + package.extractall(item["application"], filter="data") + run("mount", "-o", "remount,ro", str(item["application"])) if not mounted(item["rootfs"]): (item["upper"] / NONPERSISTENT_MOUNT).mkdir(parents=True, exist_ok=True) (item["upper"] / PERSISTENT_MOUNT).mkdir(parents=True, exist_ok=True) @@ -638,15 +643,15 @@ def uninstall(name: str) -> None: def install(image: Path, manifest_path: Path, application_name: str | None = None) -> None: - """Provision an application image, manifest, runtime configuration, and storage. + """Provision an application package, manifest, runtime configuration, and storage. Args: - image: Application filesystem image to install. + image: Application appoverlay package (.tgz) to install. manifest_path: Manifest JSON file that defines the application. application_name: Optional installed name that overrides the manifest name. """ if not image.is_file() or not manifest_path.is_file(): - raise ValueError("application image or manifest is missing") + raise ValueError("application package or manifest is missing") metadata = read_manifest(manifest_path, application_name) metadata["base_image"] = str(find_base_image()) root = app_directory(metadata["name"]) @@ -671,7 +676,7 @@ def install(image: Path, manifest_path: Path, application_name: str | None = Non write_lxc_config(metadata, root) try: mount_application(metadata) - except (OSError, RuntimeError, subprocess.CalledProcessError): + except (OSError, RuntimeError, subprocess.CalledProcessError, tarfile.TarError): try: unmount_application(metadata) finally: @@ -732,7 +737,9 @@ def main() -> int: deactivate(arguments.name) elif arguments.operation == "uninstall": uninstall(arguments.name) - except (OSError, ValueError, RuntimeError, subprocess.CalledProcessError) as error: + except ( + OSError, ValueError, RuntimeError, subprocess.CalledProcessError, tarfile.TarError + ) as error: print(f"manage_package: {error}", file=sys.stderr) return 1 return 0 diff --git a/src/GEISA-LEE-tests/src/community b/src/GEISA-LEE-tests/src/community index d65304a2..0c9e8afe 160000 --- a/src/GEISA-LEE-tests/src/community +++ b/src/GEISA-LEE-tests/src/community @@ -1 +1 @@ -Subproject commit d65304a243a742df44dcd0760059bb50aa303937 +Subproject commit 0c9e8afe2ea7b34cf0f06e6ddbe91f32289ec670 diff --git a/src/GEISA-LEE-tests/tests.d/build-reference-application.sh b/src/GEISA-LEE-tests/tests.d/build-reference-application.sh index ead9ceda..4be290dc 100644 --- a/src/GEISA-LEE-tests/tests.d/build-reference-application.sh +++ b/src/GEISA-LEE-tests/tests.d/build-reference-application.sh @@ -43,6 +43,12 @@ esac echo "Application directory not found: ${APPLICATION_DIR}" >&2 exit 2 } +for script in package.py generate-manifest.py; do + [[ -f "${APPLICATION_DIR}/scripts/${script}" ]] || { + echo "Packaging script not found: ${APPLICATION_DIR}/scripts/${script}" >&2 + exit 2 + } +done MANIFEST="$(find "${APPLICATION_DIR}" -maxdepth 1 -type f -name '*-manifest.json' -print -quit)" [[ -n "${MANIFEST}" ]] || { @@ -52,17 +58,32 @@ MANIFEST="$(find "${APPLICATION_DIR}" -maxdepth 1 -type f -name '*-manifest.json MANIFEST_METADATA="$( python3 - "${MANIFEST}" <<'PY' import json +from pathlib import Path +import shlex import sys with open(sys.argv[1], encoding="utf-8") as stream: manifest = json.load(stream)["geisa-application-manifest"]["manifest"] -print(manifest["name"], manifest["app-version"], sep="\t") +start = shlex.split(manifest["default-launch-strategy"]["start-string"]) +resources = manifest["resources"] +# Largest byte limit the probes must be able to exceed, plus 25% leeway, kept +# within the int range geisa-test stores it in. +limit_kib = max(resources["app-ram"], resources["storage-persistent"], + resources["storage-nonpersistent"]) +probe_target_bytes = min(limit_kib * 1024 * 5 // 4, 2**31 - 1) +print(manifest["name"], manifest["app-version"], Path(start[0]).name, + probe_target_bytes, sep="\t") PY )" || { echo "Failed to read application manifest: ${MANIFEST}" >&2 exit 2 } -IFS=$'\t' read -r APP_NAME APP_VERSION <<< "${MANIFEST_METADATA}" +IFS=$'\t' read -r APP_NAME APP_VERSION APP_BINARY PROBE_TARGET_BYTES <<< "${MANIFEST_METADATA}" + +# Only geisa-test has a compiled-in probe cap to lift, see geisa-app.Dockerfile. +if [[ "${APP_NAME}" != geisa-test ]]; then + PROBE_TARGET_BYTES= +fi [[ "${APP_NAME}" =~ ^[A-Za-z0-9_.-]+$ ]] || { echo "Invalid application name in manifest: ${APP_NAME}" >&2 @@ -72,10 +93,14 @@ IFS=$'\t' read -r APP_NAME APP_VERSION <<< "${MANIFEST_METADATA}" echo "Invalid application version in manifest: ${APP_VERSION}" >&2 exit 2 } +[[ "${APP_BINARY}" =~ ^[A-Za-z0-9_.-]+$ ]] || { + echo "Invalid application binary in manifest: ${APP_BINARY}" >&2 + exit 2 +} APP_OUTPUT_DIR="${OUTPUT_DIR}/${APP_NAME}" -STAGE_DIR="${APP_OUTPUT_DIR}/rootfs" -IMAGE="${APP_OUTPUT_DIR}/${APP_NAME}-${APP_VERSION}.squashfs" +STAGE_DIR="${APP_OUTPUT_DIR}/build" +IMAGE="${APP_OUTPUT_DIR}/${APP_NAME}-${APP_VERSION}-appoverlay.tgz" OUTPUT_MANIFEST="${APP_OUTPUT_DIR}/manifest.json" rm -rf "${APP_OUTPUT_DIR}" @@ -83,13 +108,24 @@ mkdir -p "${STAGE_DIR}" "${CONTAINER_CLI}" build \ --platform "linux/${PLATFORM_ARCH}" \ - --file "${SCRIPT_DIR}/../geisa-reference.Dockerfile" \ + --build-arg "APP_BINARY=${APP_BINARY}" \ + --build-arg "SOURCE_NAME=$(basename "${APPLICATION_DIR}")" \ + --build-arg "PROBE_TARGET_BYTES=${PROBE_TARGET_BYTES}" \ + --file "${SCRIPT_DIR}/../geisa-app.Dockerfile" \ --output "type=local,dest=${STAGE_DIR}" \ "${APPLICATION_DIR}" >&2 -chmod 0755 "${STAGE_DIR}/geisa-app" -cp "${MANIFEST}" "${OUTPUT_MANIFEST}" -mksquashfs "${STAGE_DIR}" "${IMAGE}" -all-root -noappend -quiet >&2 +chmod 0755 "${STAGE_DIR}/${APP_BINARY}" +IFS=$'\t' read -r TOOLCHAIN_ID TOOLCHAIN_VERSION < "${STAGE_DIR}/toolchain" +TOOLCHAIN_ID="${TOOLCHAIN_ID}" TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}" \ + python3 "${APPLICATION_DIR}/scripts/package.py" \ + --executable "${STAGE_DIR}/${APP_BINARY}" \ + --member-name "${APP_BINARY}" \ + --artifact "${IMAGE}" \ + --manifest-template "${MANIFEST}" \ + --manifest-output "${OUTPUT_MANIFEST}" \ + --manifest-generator "${APPLICATION_DIR}/scripts/generate-manifest.py" >&2 +chmod 0644 "${IMAGE}" "${OUTPUT_MANIFEST}" rm -rf "${STAGE_DIR}" printf '%s\t%s\t%s\n' "${APP_NAME}" "${IMAGE}" "${OUTPUT_MANIFEST}" diff --git a/src/GEISA-LEE-tests/tests.d/check-container.sh b/src/GEISA-LEE-tests/tests.d/check-container.sh index 2b7c807a..296baf09 100644 --- a/src/GEISA-LEE-tests/tests.d/check-container.sh +++ b/src/GEISA-LEE-tests/tests.d/check-container.sh @@ -14,6 +14,10 @@ REFERENCE_LXC_CPU_WEIGHT="${REFERENCE_LXC_CPU_WEIGHT:-}" REBOOT_PERSISTENCE_MARKER="${REBOOT_PERSISTENCE_MARKER:-}" PRIVILEGED_COMMAND="${PRIVILEGED_COMMAND:-}" +# The reference application is installed under the name its manifest gives. +REFERENCE_APPLICATION_NAME="$(manifest_application_name "${APPLICATION_MANIFEST_PATH}" 2>/dev/null)" +APPLICATIONS="${APPLICATIONS:-${REFERENCE_APPLICATION_NAME}}" + # Mount points and journaling filesystem expectations are mandated by the GEISA # specification and are not configurable. CONTAINER_PERSISTENT_MOUNT="/home/geisa" @@ -275,7 +279,7 @@ manage_package() { local app="$1" shift ( - export APPLICATIONS LXC_PATH REFERENCE_APPLICATION + export APPLICATIONS LXC_PATH export CONTAINER_BASE_IMAGE CONTAINER_BASE_IMAGE_DIR CONTAINER_BASE_IMAGE_GLOB export CONTAINER_IMAGE_FSTYPE CONTAINER_PERSISTENT_FSTYPE export LXC_NETWORK_BRIDGE LXC_NETWORK_GATEWAY LXC_NETWORK_PREFIX diff --git a/src/GEISA-LEE-tests/tests.d/check-lxc-container.py b/src/GEISA-LEE-tests/tests.d/check-lxc-container.py index 4fd02fcb..0b7ee7d6 100644 --- a/src/GEISA-LEE-tests/tests.d/check-lxc-container.py +++ b/src/GEISA-LEE-tests/tests.d/check-lxc-container.py @@ -13,7 +13,7 @@ import sys LXC_PATH = Path(os.environ.get("LXC_PATH", "/platform/conformance/lxc")) -REFERENCE_APPLICATION = os.environ.get("REFERENCE_APPLICATION", "geisa-simple") +REFERENCE_APPLICATION = os.environ.get("REFERENCE_APPLICATION") MANIFEST_PATH = os.environ.get("APPLICATION_MANIFEST_PATH") STORAGE_FIELDS = ("persistent", "nonpersistent") diff --git a/src/GEISA-LEE-tests/tests_configuration.conf b/src/GEISA-LEE-tests/tests_configuration.conf index 99b9190b..0935baec 100644 --- a/src/GEISA-LEE-tests/tests_configuration.conf +++ b/src/GEISA-LEE-tests/tests_configuration.conf @@ -3,11 +3,21 @@ # Conformance tests installation CONFIGURATION_FILE=true -# Reference application -REFERENCE_APPLICATION="geisa-simple" +# Reference application. REFERENCE_APPLICATION selects a folder under +# community/applications, which is built and installed with the manifest kept +# in that folder, under the name that manifest gives. +REFERENCE_APPLICATION="geisa-test" + +# Print the application name given by an application manifest. +manifest_application_name() { + python3 -c 'import json, sys +with open(sys.argv[1], encoding="utf-8") as stream: + print(json.load(stream)["geisa-application-manifest"]["manifest"]["name"])' "$1" +} # Resolve the host-side LXC resources of one application and export them for -# the reference provisioner, which reads its configuration from the environment. +# the reference provisioner, which is configured through the environment. +# The layout functions are defined in user_configuration.conf. export_application_layout() { BASE_LAYER_PATH="$(layout_base_layer_path "$1")" APPLICATION_LAYER_PATH="$(layout_application_layer_path "$1")" diff --git a/src/GEISA-LEE-tests/user_configuration.conf b/src/GEISA-LEE-tests/user_configuration.conf index 8e55e7ba..d805c067 100644 --- a/src/GEISA-LEE-tests/user_configuration.conf +++ b/src/GEISA-LEE-tests/user_configuration.conf @@ -5,9 +5,9 @@ # ================================== Overall =================================== -# Applications under test. Separated by spaces defaults to the reference -# application when unset or empty. -APPLICATIONS="${APPLICATIONS:-${REFERENCE_APPLICATION}}" +# Applications under test, separated by spaces. When unset or empty, this is +# the reference application, named after its manifest. +APPLICATIONS="${APPLICATIONS:-}" # Command prefix for the host and container inspection that requires platform # privilege, such as lxc-attach, lxc-info and reads of the host /proc, cgroup @@ -38,7 +38,7 @@ layout_configuration_layer_path() { printf '%s\n' "${LXC_PATH}/$1/configuration" layout_upper_path() { printf '%s\n' "${LXC_PATH}/$1/upper"; } layout_work_path() { printf '%s\n' "${LXC_PATH}/$1/work"; } layout_rootfs_path() { printf '%s\n' "${LXC_PATH}/$1/rootfs"; } -layout_application_image_path() { printf '%s\n' "${LXC_PATH}/$1/packages/application.squashfs"; } +layout_application_image_path() { printf '%s\n' "${LXC_PATH}/$1/packages/application.tgz"; } layout_persistent_image_path() { printf '%s\n' "${LXC_PATH}/$1/persistent.img"; } # Reference application resource policy used when a value is absent from its manifest. @@ -51,12 +51,20 @@ REFERENCE_LXC_MEMORY_KIB="1" REFERENCE_LXC_PERSISTENT_KIB="0" REFERENCE_LXC_NONPERSISTENT_KIB="0" -# Reference application (e.g: geisa-app) network setup. +# Reference application network setup. # Re-configure these values to match the network configuration of your LXC bridge used for GEISA applications. LXC_NETWORK_BRIDGE="lxcbr0" LXC_NETWORK_GATEWAY="10.0.3.1" LXC_NETWORK_PREFIX="24" +# MQTT broker admin access. manage_package.py uses it to give each installed +# application a broker client through the mosquitto dynamic security plugin, +# limited to the API topics that its manifest api-access requests. +MQTT_BROKER_HOST="127.0.0.1" +MQTT_BROKER_PORT="1883" +MQTT_ADMIN_USER="admin" +MQTT_ADMIN_PASSWORD="admin" + # Base image supplied to applications. Set CONTAINER_BASE_IMAGE to select # one file, or leave it empty to use the newest matching file. CONTAINER_BASE_IMAGE="" diff --git a/src/launch_glee_conformance_tests_ssh.sh b/src/launch_glee_conformance_tests_ssh.sh index fa4928bb..070fac4d 100755 --- a/src/launch_glee_conformance_tests_ssh.sh +++ b/src/launch_glee_conformance_tests_ssh.sh @@ -10,12 +10,16 @@ RED="\e[31m" ENDCOLOR="\e[0m" declare CONFORMANCE_SSH_ARGS declare CONFORMANCE_SCP_ARGS -APPLICATIONS="${APPLICATIONS:-}" GLEE_TESTS="${GLEE_TESTS:-}" REFERENCE_APPLICATION="${REFERENCE_APPLICATION:-}" NO_REBOOT="${NO_REBOOT:-false}" REBOOT_PERSISTENCE_MARKER="${REBOOT_PERSISTENCE_MARKER:-}" +# The reference application is installed under the name its manifest gives. +REFERENCE_APPLICATION_DIR="${TOPDIR:-}/src/GEISA-LEE-tests/src/community/applications/${REFERENCE_APPLICATION}" +REFERENCE_APPLICATION_NAME="$(manifest_application_name "${REFERENCE_APPLICATION_DIR}/${REFERENCE_APPLICATION}-manifest.json")" +APPLICATIONS="${APPLICATIONS:-${REFERENCE_APPLICATION_NAME}}" + SSH() { local status @@ -64,7 +68,7 @@ provision_reference_application() { return 1 fi - source_dir="${topdir}/src/GEISA-LEE-tests/src/community/applications/${app_name}" + source_dir="${REFERENCE_APPLICATION_DIR}" if [[ ! -d "${source_dir}" ]]; then echo -e "${RED}Error:${ENDCOLOR} Reference application source not found: ${source_dir}" return 1 @@ -176,7 +180,7 @@ provision_glee_apps_with_ssh() { SSH "mkdir -p /tmp/conformance_tests/glee-apps" for app_name in ${APPLICATIONS}; do - if [[ "${app_name}" == "${REFERENCE_APPLICATION}" ]]; then + if [[ "${app_name}" == "${REFERENCE_APPLICATION_NAME}" ]]; then provision_reference_application "${board_ip}" "${board_user}" "${board_password}" "${topdir}" "${target_arch}" "${app_name}" || exit 1 fi done @@ -383,7 +387,7 @@ cleanup_glee_ssh() { local board_password="$3" local topdir="$4" local app_name cleanup_input app_dir status=0 - app_dir="${LXC_PATH}/${REFERENCE_APPLICATION}" + app_dir="${LXC_PATH}/${REFERENCE_APPLICATION_NAME}" # The cleanup scripts load the local test configuration, since the board # copy may be gone. @@ -401,7 +405,7 @@ cleanup_glee_ssh() { for resolver in layout_base_layer_path layout_application_layer_path \ layout_configuration_layer_path layout_upper_path layout_work_path \ layout_rootfs_path layout_application_image_path layout_persistent_image_path; do - path="$("${resolver}" "${REFERENCE_APPLICATION}")" + path="$("${resolver}" "${REFERENCE_APPLICATION_NAME}")" label="${resolver#layout_}" label="${label//_/ }" printf ' %s: %s\n' "${label^}" "${path}" @@ -414,12 +418,12 @@ cleanup_glee_ssh() { print_cleanup_paths if ! wait_for_board_up "${board_ip}"; then echo "Warning: Board at ${board_ip} is not reachable. Cleanup failed." >&2 - echo "Manual cleanup required: de-provision '${REFERENCE_APPLICATION}' and unmount its filesystems, then remove '${app_dir}' and '/tmp/conformance_tests' on the board." >&2 + echo "Manual cleanup required: de-provision '${REFERENCE_APPLICATION_NAME}' and unmount its filesystems, then remove '${app_dir}' and '/tmp/conformance_tests' on the board." >&2 print_cleanup_paths >&2 return 1 fi for app_name in ${APPLICATIONS}; do - if [[ "${app_name}" == "${REFERENCE_APPLICATION}" ]]; then + if [[ "${app_name}" == "${REFERENCE_APPLICATION_NAME}" ]]; then cleanup_input="$({ print_cleanup_configuration echo "export_application_layout '${app_name}'" @@ -437,7 +441,7 @@ cleanup_glee_ssh() { SSH "sh -s" <<< "${cleanup_input}" || status=1 if [[ ${status} -ne 0 ]]; then echo -e "${RED}Error:${ENDCOLOR} Failed to clean up LEE test artifacts on board" - echo "Manual cleanup required: de-provision '${REFERENCE_APPLICATION}' and unmount its filesystems, then remove '${app_dir}' and '/tmp/conformance_tests' on the board." + echo "Manual cleanup required: de-provision '${REFERENCE_APPLICATION_NAME}' and unmount its filesystems, then remove '${app_dir}' and '/tmp/conformance_tests' on the board." print_cleanup_paths return 1 fi From 807d7561efd0d6986483351b30ccc0b86b4d7b5b Mon Sep 17 00:00:00 2001 From: Nghia Dam Date: Tue, 6 Oct 2026 13:00:17 -0400 Subject: [PATCH 4/5] GEISA-LEE-tests: exercise the app isolation limits with geisa-test The app isolation tests only read the configured limits. When geisa-test is the reference application, they now also run it past each limit and check that the platform enforces it: - APP-SHALL-05, CPU percent: a full CPU load is throttled to the quota. - APP-SHALL-07, memory: allocating past the limit gets the app killed. - APP-SHALL-08 and 09, persistent and nonpersistent storage: writing past the limit fails. - APP-SHALL-10, standard sockets: the app can connect to each outbound TCP port that its manifest grants. - APP-SHALL-11, ACL: a port that is not granted cannot be reached. - APP-SHALL-12, volume limit: a granted port cannot be reached once the daily volume is used. Change-Id: I779edf86955238f7e9e4a3a0abb222795f1e5030 Signed-off-by: Nghia Dam --- src/GEISA-LEE-tests/manage_package.py | 306 +++++++--- .../tests.d/check-container.sh | 84 ++- .../tests.d/check-lxc-container.py | 568 ++++++++++++++++-- 3 files changed, 770 insertions(+), 188 deletions(-) diff --git a/src/GEISA-LEE-tests/manage_package.py b/src/GEISA-LEE-tests/manage_package.py index e77f6777..54f03fb4 100644 --- a/src/GEISA-LEE-tests/manage_package.py +++ b/src/GEISA-LEE-tests/manage_package.py @@ -30,13 +30,61 @@ ROOTFS_PATH = os.environ["ROOTFS_PATH"] APPLICATION_IMAGE_PATH = os.environ["APPLICATION_IMAGE_PATH"] PERSISTENT_IMAGE_PATH = os.environ["PERSISTENT_IMAGE_PATH"] +MQTT_BROKER_HOST = os.environ["MQTT_BROKER_HOST"] +MQTT_BROKER_PORT = os.environ["MQTT_BROKER_PORT"] +MQTT_ADMIN_USER = os.environ["MQTT_ADMIN_USER"] +MQTT_ADMIN_PASSWORD = os.environ["MQTT_ADMIN_PASSWORD"] PERSISTENT_MOUNT = "home/geisa" NONPERSISTENT_MOUNT = "tmp" NETWORK_POLICY_TABLE = "geisa_conformance" NETWORK_POLICY_QUOTA = "network_volume" +MQTT_PASSWORD = "geisa-conformance" +# Marks a broker client that install created, so uninstall only removes those. +BROKER_CLIENT_MARKER = "broker-client" +# API topics that every application may use. "{}" stands for its userid. +BROKER_BASE_ACLS = ( + ("publishClientSend", "geisa/api/platform/discovery/req/{}"), + ("publishClientSend", "geisa/api/app/manifest/req/{}"), + ("publishClientSend", "geisa/api/app/platform/status/{}"), + ("subscribeLiteral", "geisa/api/platform/discovery/rsp/{}"), + ("subscribeLiteral", "geisa/api/app/manifest/rsp/{}"), + ("subscribeLiteral", "geisa/api/platform/app/status/{}"), + ("subscribeLiteral", "geisa/api/platform/status"), +) +# API topics granted by each api-access entry of the manifest. +BROKER_API_ACLS = { + "messaging": ( + ("publishClientSend", "geisa/api/message/upstream/req/{}"), + ("publishClientSend", "geisa/api/message/downstream/rsp/{}"), + ("subscribeLiteral", "geisa/api/message/upstream/rsp/{}"), + ("subscribeLiteral", "geisa/api/message/downstream/req/{}"), + ), + "actuator-status": ( + ("publishClientSend", "geisa/api/actuator/get/req/{}"), + ("subscribeLiteral", "geisa/api/actuator/get/rsp/{}"), + ), + "actuator-control": ( + ("publishClientSend", "geisa/api/actuator/set/req/{}"), + ("subscribeLiteral", "geisa/api/actuator/set/rsp/{}"), + ), + "instantaneous": ( + ("subscribeLiteral", "geisa/api/instantaneous/data"), + ), + "sensor": ( + ("subscribeLiteral", "geisa/api/sensor"), + ("publishClientSend", "geisa/api/sensor-req/{}"), + ("subscribeLiteral", "geisa/api/sensor-rsp/{}"), + ), + "waveform": ( + ("publishClientSend", "geisa/api/waveform/req/{}"), + ("subscribeLiteral", "geisa/api/waveform/rsp/{}"), + ), +} NAME_PATTERN = re.compile(r"^[A-Za-z0-9_.-]+$") +# transport:address:port notation of the manifest schema. +PERMISSION_PATTERN = re.compile(r"(tcp:|udp:|)([0-9a-f:.]+|\[[0-9a-f:.]+\]|):([0-9]+)", re.IGNORECASE) CPUSET_PATTERN = re.compile(r"^[0-9]+(?:-[0-9]+)?(?:,[0-9]+(?:-[0-9]+)?)*$") def environment_integer(name: str) -> int: @@ -86,46 +134,68 @@ def conformance_network_address(app_name: str, gateway: str, prefix: int) -> str return str(address) -def manifest_network_policy(communication: dict) -> dict: - """Translate manifest communication permissions into an enforceable network policy. +def parse_network_permission(section: str, direction: str, permission: object) -> dict: + """Parse one transport:address:port permission string of the manifest. Args: - communication: Manifest communication section to validate and interpret. + section: Manifest network section holding the permission. + direction: "inbound" or "outbound". + permission: Permission string to parse. """ - # Network access is denied by default, following the principle of least privilege. - denied = { - "network_access": False, - "network_transport": None, - "network_port": 0, - "network_daily_volume_bytes": 0, - } - if not isinstance(communication, dict) or communication.get("HAN") is not True: - return denied - local = communication.get("local") - if not isinstance(local, dict): - return denied - outbound = local.get("outbound") - if not isinstance(outbound, list) or not outbound: - return denied - permission = outbound[0] if not isinstance(permission, str): - raise ValueError("local outbound network permission must be a string") - match = re.fullmatch(r"(?:(tcp|udp):)?.*:([0-9]+)", permission, re.IGNORECASE) + raise ValueError(f"{section} {direction} network permission must be a string") + match = PERMISSION_PATTERN.fullmatch(permission) if not match: - raise ValueError(f"invalid local outbound network permission: {permission}") - transport_group = match.group(1) - port_group = match.group(2) - transport = transport_group.lower() if transport_group else None # None here (when network access is allowed) means both TCP and UDP are allowed - port = int(port_group) - daily_volume = int(local["daily-volume"]) + raise ValueError(f"invalid {section} {direction} network permission: {permission}") + transport, address, port = match.groups() + address = address.strip("[]") + if address: + address = str(ipaddress.ip_address(address)) return { - "network_access": True, - "network_transport": transport, - "network_port": port, - "network_daily_volume_bytes": daily_volume, + # No transport means both TCP and UDP, no address means any address. + "transport": transport.rstrip(":").lower() or None, + "address": address, + "port": int(port), } +def manifest_network_sections(communication: dict) -> list[dict]: + """Translate communication permissions in the manifest into enforceable network sections. + + Args: + communication: Manifest communication section to validate and interpret. + """ + sections = [] + if not isinstance(communication, dict): + return sections + granted = [] + if communication.get("HAN") is True: + granted.append("local") + if communication.get("FAN") is True: + granted.extend(("operator", "internet")) + for section in granted: + requirements = communication.get(section) + if not isinstance(requirements, dict): + continue + permissions = { + direction: [ + parse_network_permission(section, direction, permission) + for permission in requirements.get(direction, []) + ] + for direction in ("inbound", "outbound") + } + if not permissions["inbound"] and not permissions["outbound"]: + continue + sections.append( + { + "section": section, + "daily_volume_bytes": int(requirements["daily-volume"]), + **permissions, + } + ) + return sections + + def run(*command: str, check: bool = True, capture: bool = False) -> subprocess.CompletedProcess[str]: """Run a system command with consistent error handling and optional output capture. @@ -204,6 +274,7 @@ def read_manifest(path: Path, application_name: str | None = None) -> dict: "REFERENCE_LXC_NONPERSISTENT_KIB", ), "communication": communication, + "network_sections": manifest_network_sections(communication), "api_access": manifest.get("api-access", {}), "network_bridge": os.environ["LXC_NETWORK_BRIDGE"], "network_gateway": network_gateway, @@ -212,7 +283,6 @@ def read_manifest(path: Path, application_name: str | None = None) -> dict: name, network_gateway, network_prefix ), } - values.update(manifest_network_policy(communication)) if "threads" in resources: values["threads"] = int(resources["threads"]) except (KeyError, TypeError, ValueError) as error: @@ -235,7 +305,6 @@ def read_manifest(path: Path, application_name: str | None = None) -> dict: "cpu_weight", "persistent_kib", "nonpersistent_kib", - "network_daily_volume_bytes", ): if values[field] < 0: raise ValueError(f"{field} cannot be negative") @@ -243,9 +312,12 @@ def read_manifest(path: Path, application_name: str | None = None) -> dict: raise ValueError("memory and CPU percentage constraints must be greater than zero") if not 1 <= values["cpu_weight"] <= 10000: raise ValueError("cpu_weight must be between 1 and 10000") - if values["network_access"]: - if not 1 <= values["network_port"] <= 65535: - raise ValueError("network_port must be between 1 and 65535") + for section in values["network_sections"]: + if section["daily_volume_bytes"] < 0: + raise ValueError(f"{section['section']} daily-volume cannot be negative") + for permission in section["inbound"] + section["outbound"]: + if not 1 <= permission["port"] <= 65535: + raise ValueError("network port must be between 1 and 65535") for field in ("cpuset_cpus", "cpuset_mems"): if not CPUSET_PATTERN.fullmatch(values[field]): raise ValueError(f"invalid {field}: {values[field]}") @@ -370,21 +442,14 @@ def write_lxc_config(metadata: dict, root: Path) -> None: "lxc.environment = USER=geisa", "lxc.environment = PATH=/usr/local/bin:/usr/bin:/bin", f"lxc.mount.entry = {log_socket} dev/log none bind,optional,create=file 0 0", + # Every application gets a network interface to reach the broker. + "lxc.net.0.type = veth", + f"lxc.net.0.link = {metadata['network_bridge']}", + "lxc.net.0.flags = up", + "lxc.net.0.name = eth0", + f"lxc.net.0.ipv4.address = {metadata['network_address']}/{metadata['network_prefix']}", + f"lxc.net.0.ipv4.gateway = {metadata['network_gateway']}", ] - if metadata["network_access"]: - lines.extend( - ( - "lxc.net.0.type = veth", - f"lxc.net.0.link = {metadata['network_bridge']}", - "lxc.net.0.flags = up", - "lxc.net.0.name = eth0", - ( - "lxc.net.0.ipv4.address = " - f"{metadata['network_address']}/{metadata['network_prefix']}" - ), - f"lxc.net.0.ipv4.gateway = {metadata['network_gateway']}", - ) - ) lines.extend( ( f"lxc.cgroup2.cpuset.mems = {metadata['cpuset_mems']}", @@ -521,33 +586,48 @@ def network_policy_rules(metadata: dict) -> str: metadata: Validated application metadata constructed from its manifest. """ gateway = metadata["network_gateway"] - port = metadata["network_port"] - volume = metadata["network_daily_volume_bytes"] - transport = metadata["network_transport"] - protocols = ("tcp", "udp") if transport is None else (transport,) - output_lines = [] - input_lines = [] - for protocol in protocols: - output_lines.append( - f" ip daddr {gateway} {protocol} dport {port} " - f"quota name \"{NETWORK_POLICY_QUOTA}\" drop" - ) - output_lines.append( - f" ip daddr {gateway} {protocol} dport {port} accept" - ) - input_lines.append( - f" ip saddr {gateway} {protocol} sport {port} " - f"quota name \"{NETWORK_POLICY_QUOTA}\" drop" - ) - input_lines.append( - f" ip saddr {gateway} {protocol} sport {port} " - f"ct state established accept" - ) + # The broker is always reachable and its traffic does not count in any daily volume. + output_lines = [ + " oifname \"lo\" accept", + f" ip daddr {gateway} tcp dport {MQTT_BROKER_PORT} accept", + ] + input_lines = [ + " iifname \"lo\" accept", + f" ip saddr {gateway} tcp sport {MQTT_BROKER_PORT} ct state established accept", + ] + quota_lines = [] + for section in metadata["network_sections"]: + quota = f"{NETWORK_POLICY_QUOTA}_{section['section']}" + quota_lines.append(f" quota {quota} {{ over {section['daily_volume_bytes']} bytes }}") + for direction in ("outbound", "inbound"): + for permission in section[direction]: + transport = permission["transport"] + protocols = ("tcp", "udp") if transport is None else (transport,) + address = permission["address"] + family = "ip6" if ":" in address else "ip" + to_remote = f"{family} daddr {address} " if address else "" + from_remote = f"{family} saddr {address} " if address else "" + for protocol in protocols: + # Outbound: the port is the remote one and the application + # opens the connection. Inbound: the port is the + # application's own and the remote side opens it. + if direction == "outbound": + opened = f"{to_remote}{protocol} dport {permission['port']}" + replied = f"{from_remote}{protocol} sport {permission['port']}" + opening, replying = output_lines, input_lines + else: + opened = f"{from_remote}{protocol} dport {permission['port']}" + replied = f"{to_remote}{protocol} sport {permission['port']}" + opening, replying = input_lines, output_lines + opening.append(f" {opened} quota name \"{quota}\" drop") + opening.append(f" {opened} accept") + replying.append(f" {replied} quota name \"{quota}\" drop") + replying.append(f" {replied} ct state established accept") output_rules = "\n".join(output_lines) input_rules = "\n".join(input_lines) + quotas = "".join(f"{line}\n" for line in quota_lines) return f"""table inet {NETWORK_POLICY_TABLE} {{ - quota {NETWORK_POLICY_QUOTA} {{ over {volume} bytes }} - chain output {{ +{quotas} chain output {{ type filter hook output priority 0; policy drop; {output_rules} }} @@ -586,7 +666,7 @@ def install_network_policy(metadata: dict) -> None: metadata: Validated application metadata constructed from its manifest. """ name = metadata["name"] - if not network_quota_supported(name): + if metadata["network_sections"] and not network_quota_supported(name): raise RuntimeError( "the kernel lacks nftables quota support (CONFIG_NFT_QUOTA)" ) @@ -608,6 +688,71 @@ def install_network_policy(metadata: dict) -> None: ) +def broker_dynsec(*arguments: str, check: bool = True) -> str: + """Run a dynamic security command on the MQTT broker as its admin. + + mosquitto_ctrl exits with 0 even when the command fails, so its output is + checked instead. + + Args: + arguments: dynsec command and its arguments. + check: Whether an error reported by the broker raises an exception. + """ + output = subprocess.run( + ( + "mosquitto_ctrl", "-h", MQTT_BROKER_HOST, "-p", MQTT_BROKER_PORT, + "-u", MQTT_ADMIN_USER, "-P", MQTT_ADMIN_PASSWORD, "dynsec", *arguments, + ), + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ).stdout + errors = [line for line in output.splitlines() if "Error:" in line] + if check and errors: + raise RuntimeError(f"broker {errors[0]}") + return output + + +def register_broker_client(metadata: dict, root: Path) -> None: + """Give an application a broker client with the credentials of its mqtt.conf. + + Its role only allows the topics that its manifest api-access requests. + A client that the platform already provides is left as it is. + + Args: + metadata: Validated application metadata constructed from its manifest. + root: Application installation directory. + """ + name = metadata["name"] + if "Error:" not in broker_dynsec("getClient", name, check=False): + return + (root / BROKER_CLIENT_MARKER).touch() + role = f"{name}-topics" + acls = list(BROKER_BASE_ACLS) + for api, granted in metadata["api_access"].items(): + if granted is True: + acls.extend(BROKER_API_ACLS.get(api, ())) + broker_dynsec("createClient", name, "-p", MQTT_PASSWORD) + broker_dynsec("createRole", role) + for acl_type, topic in acls: + broker_dynsec("addRoleACL", role, acl_type, topic.format(name), "allow") + broker_dynsec("addClientRole", name, role) + + +def unregister_broker_client(name: str, root: Path) -> None: + """Remove the broker client and role that install created, if any. + + Args: + name: Installed application name, also its broker userid. + root: Application installation directory. + """ + if not (root / BROKER_CLIENT_MARKER).is_file(): + return + broker_dynsec("deleteClient", name, check=False) + broker_dynsec("deleteRole", f"{name}-topics", check=False) + + def deactivate(name: str) -> None: """Stop an application, if running, and detach its mounted filesystems. @@ -633,6 +778,7 @@ def uninstall(name: str) -> None: name: Installed application name. """ root = app_directory(name) + unregister_broker_client(name, root) if not (root / "manifest.json").is_file(): shutil.rmtree(root, ignore_errors=True) return @@ -663,8 +809,9 @@ def install(image: Path, manifest_path: Path, application_name: str | None = Non shutil.copy2(image, item["image"]) shutil.copy2(manifest_path, root / "manifest.json") (item["configuration"] / "etc/geisa/mqtt.conf").write_text( - f"HOST=127.0.0.1\nPORT=1883\nUSERID={metadata['name']}\n" - "PASSWORD=geisa-conformance\n", + f"HOST={metadata['network_gateway']}\n" + f"PORT={MQTT_BROKER_PORT}\nUSERID={metadata['name']}\n" + f"PASSWORD={MQTT_PASSWORD}\n", encoding="utf-8", ) write_launcher(item["configuration"], metadata) @@ -676,8 +823,10 @@ def install(image: Path, manifest_path: Path, application_name: str | None = Non write_lxc_config(metadata, root) try: mount_application(metadata) + register_broker_client(metadata, root) except (OSError, RuntimeError, subprocess.CalledProcessError, tarfile.TarError): try: + unregister_broker_client(metadata["name"], root) unmount_application(metadata) finally: shutil.rmtree(root, ignore_errors=True) @@ -707,8 +856,7 @@ def activate(name: str) -> None: "-t", str(metadata["start_timeout"]), ) - if metadata["network_access"]: - install_network_policy(metadata) + install_network_policy(metadata) def parse_arguments() -> argparse.Namespace: diff --git a/src/GEISA-LEE-tests/tests.d/check-container.sh b/src/GEISA-LEE-tests/tests.d/check-container.sh index 296baf09..89ba5e24 100644 --- a/src/GEISA-LEE-tests/tests.d/check-container.sh +++ b/src/GEISA-LEE-tests/tests.d/check-container.sh @@ -274,6 +274,9 @@ EOF ####################################### # Run manage_package.py for one application, with its resolved host layout. +# +# The first argument names the application whose resources are provisioned. +# The remaining arguments are the manage_package.py command line. ####################################### manage_package() { local app="$1" @@ -286,6 +289,7 @@ manage_package() { export REFERENCE_LXC_CPUSET_CPUS REFERENCE_LXC_CPUSET_MEMS export REFERENCE_LXC_CPU_PERCENT REFERENCE_LXC_CPU_WEIGHT REFERENCE_LXC_MEMORY_KIB export REFERENCE_LXC_PERSISTENT_KIB REFERENCE_LXC_NONPERSISTENT_KIB + export MQTT_BROKER_HOST MQTT_BROKER_PORT MQTT_ADMIN_USER MQTT_ADMIN_PASSWORD export_application_layout "${app}" host_exec python3 "${LEE_TESTS_DIR}/manage_package.py" "$@" ) >/dev/null 2>&1 @@ -376,6 +380,8 @@ check_container_namespaces_are_isolated() { ####################################### # Check one cgroup constraint for every configured application. +# +# Exercising a limit may stop the application, so its state is restored after. ####################################### check_all_container_cgroup_constraints() { local constraint="$1" app pid cgroup cgroup_entries @@ -384,7 +390,7 @@ check_all_container_cgroup_constraints() { cgroup_entries="$(host_exec cat "/proc/${pid}/cgroup")" || return 1 cgroup="$(awk -F: '$1 == "0" && $2 == "" { print $3; exit }' <<< "${cgroup_entries}")" test -n "${cgroup}" && test "${cgroup}" != / || return 1 - container_backend cgroup \ + keep_container_states container_backend cgroup \ "${constraint}" "${pid}" "${app}" \ "${REFERENCE_LXC_CPUSET_CPUS}" \ "${REFERENCE_LXC_CPU_WEIGHT}" || return 1 @@ -416,37 +422,6 @@ check_containers_are_isolated() { return 0 } -####################################### -# Read every configured storage limit from the deployment manifest. -# -# The limits are collected in a single manifest access and kept in memory, so -# later lookups do not read the manifest again. -####################################### -collect_application_limits() { - local limits - if [[ -n "${STORAGE_LIMITS:-}" ]]; then - return 0 - fi - limits="$(container_backend storage-limits)" || return 1 - STORAGE_LIMITS=$'\n'"${limits}"$'\n' - return 0 -} - -####################################### -# Return a collected storage limit for an application. -####################################### -get_application_limit() { - local app="$1" field="$2" key limit - key=$'\n'"${app}|${field}|" - case "${STORAGE_LIMITS:-}" in - *"${key}"*) : ;; - *) return 1 ;; - esac - limit="${STORAGE_LIMITS##*"${key}"}" - limit="${limit%%$'\n'*}" - printf '%s\n' "${limit}" -} - ####################################### # Check whether an application's container root mount is read-only. ####################################### @@ -565,39 +540,43 @@ get_container_mount_size() { ####################################### check_container_read_write_size() { local app limit size - collect_application_limits || return 1 collect_container_mount_sizes || return 1 for app in ${APPLICATIONS}; do if is_container_root_read_only "${app}"; then continue fi - limit="$(get_application_limit "${app}" nonpersistent)" || return 1 + limit="$(container_backend storage-limit "${app}" nonpersistent)" || return 1 size="$(get_container_mount_size "${app}" /)" || return 1 test "${size}" -le "${limit}" || return 1 done return 0 } +####################################### +# Check with geisa-test that the reference application cannot write past a +# storage limit, then restore the application state. +####################################### +exercise_container_storage_limit() { + keep_container_states container_backend storage-enforcement "$1" "$2" +} + ####################################### # Check a container-visible mount size against its manifest limit. ####################################### check_container_mount_limit() { local path="$1" field="$2" app limit size - collect_application_limits || return 1 collect_container_mount_sizes || return 1 for app in ${APPLICATIONS}; do - limit="$(get_application_limit "${app}" "${field}")" || return 1 + limit="$(container_backend storage-limit "${app}" "${field}")" || return 1 size="$(get_container_mount_size "${app}" "${path}")" || return 1 test "${size}" -le "${limit}" || return 1 done return 0 } -# TODO: ensure threshold is really enforced after workload done (and check if is -# actually throttled) -# test workload will be given with geisa-test ####################################### -# Check host-side storage mounts against their manifest limits. +# Check host-side storage mounts against their manifest limits, then check with +# geisa-test that the reference application cannot write past them. # # This is separate from check_container_mount_limit(), which checks the path # from inside the application container. @@ -610,10 +589,9 @@ check_container_storage_limit() { nonpersistent) mount_path="${CONTAINER_NONPERSISTENT_MOUNT}" ;; *) return 2 ;; esac - collect_application_limits || return 1 for app in ${APPLICATIONS}; do mount_point="$(layout_rootfs_path "${app}")${mount_path}" - declared_kib="$(get_application_limit "${app}" "${storage_kind}")" || return 1 + declared_kib="$(container_backend storage-limit "${app}" "${storage_kind}")" || return 1 case "${declared_kib}" in *[!0-9]*|'') return 1 ;; *) : ;; @@ -635,6 +613,7 @@ check_container_storage_limit() { *) : ;; esac test "${actual_kib}" -le "${declared_kib}" || return 1 + exercise_container_storage_limit "${app}" "${storage_kind}" || return 1 done return 0 } @@ -714,24 +693,21 @@ check_container_image_layer() { # Check that declared network access exposes standard sockets. ####################################### check_container_network_standard_sockets() { - container_backend network standard-sockets - # TODO: Add geisa-test traffic assertions for allowed socket connectivity + keep_container_states container_backend network standard-sockets } ####################################### # Check the per-application network access-control policy. ####################################### check_container_network_acl() { - container_backend network acl - # TODO: Add geisa-test traffic assertions for denied undeclared network access + keep_container_states container_backend network acl } ####################################### # Check the per-application network volume limit policy. ####################################### check_container_network_volume_limit() { - container_backend network volume-limit - # TODO: Add geisa-test traffic assertions for quota exhaustion. + keep_container_states container_backend network volume-limit } ####################################### @@ -785,6 +761,18 @@ restore_container_states() { rm -f "${CONTAINER_STATE_FILE}" } +####################################### +# Run a command that may stop or replace the applications, then return each +# application to its previous state. +####################################### +keep_container_states() { + local status=0 + save_container_states || return 1 + "$@" || status=1 + restore_container_states + return "${status}" +} + ####################################### # Verify persistence of /home/geisa across an application restart. ####################################### diff --git a/src/GEISA-LEE-tests/tests.d/check-lxc-container.py b/src/GEISA-LEE-tests/tests.d/check-lxc-container.py index 0b7ee7d6..a3bc238e 100644 --- a/src/GEISA-LEE-tests/tests.d/check-lxc-container.py +++ b/src/GEISA-LEE-tests/tests.d/check-lxc-container.py @@ -8,7 +8,9 @@ import os from pathlib import Path import re +import select import shlex +import socket import subprocess import sys @@ -16,6 +18,23 @@ REFERENCE_APPLICATION = os.environ.get("REFERENCE_APPLICATION") MANIFEST_PATH = os.environ.get("APPLICATION_MANIFEST_PATH") STORAGE_FIELDS = ("persistent", "nonpersistent") +# Container mount point and geisa-test probe for each storage field. +STORAGE_PROBES = { + "persistent": ("/home/geisa", "persistent_storage", "GEISA_TEST_PERSISTENT_STORAGE_DIR"), + "nonpersistent": ("/tmp", "transient_storage", "GEISA_TEST_TRANSIENT_STORAGE_DIR"), +} +# geisa-test needs a few seconds to connect and complete discovery before a +# probe starts. It keeps running after the probe, so it is stopped after that. +PROBE_STARTUP_S = 5 +PROBE_CPU_DURATION_MS = 5000 +PROBE_MEMORY_DURATION_MS = 3000 +# Storage probes ignore the duration and write until they reach their target. +# It only sets how long the writes may take before the probe is stopped. +PROBE_STORAGE_DURATION_MS = 3000 +# Bytes sent past the daily volume, so that the quota is surely used up. +VOLUME_MARGIN = 4096 +# transport:address:port notation of the manifest schema. +PERMISSION_PATTERN = re.compile(r"(tcp:|udp:|)([0-9a-f:.]+|\[[0-9a-f:.]+\]|):([0-9]+)", re.IGNORECASE) def is_reference(app_name: str) -> bool: @@ -27,6 +46,39 @@ def is_reference(app_name: str) -> bool: return app_name == REFERENCE_APPLICATION +def exercise_limit(app_name: str, limit: str, exercise, *arguments) -> None: + """Exercise a limit of an application with geisa-test, then stop its container. + + Only the reference application is exercised, and only when it is geisa-test. + Otherwise, warn that the exercise is skipped. The probes replace the running + application, so the caller activates it again with manage_package.py, which + also restores its network policy. + + Args: + app_name: Installed application name. + limit: Name of the limit, for the warning. + exercise: Function that runs the probes, given the application name first. + arguments: Other arguments of the exercise. + """ + if not is_reference(app_name): + return + if REFERENCE_APPLICATION != "geisa-test": + print( + f"check-lxc-container: warning: the {limit} limit is not exercised, which is " + "only supported with geisa-test as the reference application for now", + file=sys.stderr, + ) + return + try: + exercise(app_name, *arguments) + finally: + subprocess.run( + ("lxc-stop", "-P", str(LXC_PATH), "-n", app_name, "-k"), + check=True, + capture_output=True, + ) + + # --------------------------------------------------------------------------- # # Application manifest # --------------------------------------------------------------------------- # @@ -70,29 +122,28 @@ def storage_limit(app_name: str, field: str) -> object: return limit -def launch_command(app_name: str) -> str: - """Read the entry point an application declares in its manifest. +def start_string(app_name: str) -> str: + """Read the exact start command an application declares in its manifest. Args: app_name: Installed application name. """ launch = read_manifest(app_name)["default-launch-strategy"] - words = shlex.split(launch.get("start-string", "/sbin/init")) + return launch.get("start-string", "/sbin/init") + + +def launch_command(app_name: str) -> str: + """Read the entry point of the start command an application declares in its manifest. + + Args: + app_name: Installed application name. + """ + words = shlex.split(start_string(app_name)) if not words: raise ValueError("application start-string cannot be empty") return words[0] -def storage_limits() -> list[str]: - """Return every configured application's storage limits as text records.""" - records = [ - f"{app_name}|{field}|{storage_limit(app_name, field)}" - for app_name in configured_applications() - for field in STORAGE_FIELDS - ] - return records - - # --------------------------------------------------------------------------- # # cgroup v2 constraints # --------------------------------------------------------------------------- # @@ -164,7 +215,6 @@ def check_cpu_count(pid: int, policy: dict) -> None: raise RuntimeError(f"effective CPU set is {sorted(effective)}, expected {sorted(expected)}") -# TODO: ensure threshold is really enforced after workload done (and check if is actually throttled) def check_cpu_percent(pid: int, policy: dict) -> None: """Check the cgroup CPU quota against the application policy. @@ -198,7 +248,7 @@ def check_cpu_priority(pid: int, policy: dict) -> None: if expected is not None and actual != expected: raise RuntimeError(f"CPU priority is {actual}, expected {expected}") -# TODO: ensure memory threshold is really enforced after workload done (and check if is actually throttled) + def check_memory(pid: int, policy: dict) -> None: """Check the cgroup memory limit against the application policy. @@ -224,56 +274,305 @@ def check_memory(pid: int, policy: dict) -> None: } +# --------------------------------------------------------------------------- # +# cgroup v2 enforcement with the geisa-test reference application +# --------------------------------------------------------------------------- # + + +def read_counters(path: Path) -> dict[str, int]: + """Read a cgroup file made of name and value pairs, such as cpu.stat. + + Args: + path: cgroup file to read. + """ + lines = path.read_text(encoding="utf-8").splitlines() + return {name: int(value) for name, value in (line.split() for line in lines)} + + +def geisa_test_command(app_name: str, seconds: int, **settings: object) -> tuple[str, ...]: + """Build the command that replaces the running reference application with geisa-test. + + The running instance is stopped, then the exact manifest start-string is run + again from the container root, like the platform does, with the geisa-test + settings in its environment. After the given time geisa-test gets SIGTERM + by name, so that it prints its summary whatever shell wraps the start-string. + + Args: + app_name: Installed application name. + seconds: Time before the application is stopped. + settings: geisa-test environment variables. + """ + entry_point = shlex.quote(Path(launch_command(app_name)).name) + variables = " ".join(f"{name}={shlex.quote(str(value))}" for name, value in settings.items()) + script = ( + f"killall -q {entry_point}; cd / && " + f"env {variables} /bin/sh -c {shlex.quote(start_string(app_name))} & app=$!; " + f"{{ sleep {seconds}; killall -q -TERM {entry_point}; }} & timer=$!; " + 'wait "$app"; kill "$timer" 2>/dev/null' + ) + return ("lxc-attach", "-P", str(LXC_PATH), "-n", app_name, "--", "/bin/sh", "-c", script) + + +def run_geisa_test(app_name: str, seconds: int, **settings: object) -> str: + """Replace the running reference application with a geisa-test run. + + Args: + app_name: Installed application name. + seconds: Time before the application is stopped. + settings: geisa-test environment variables. + + Returns: + The geisa-test standard output. + """ + result = subprocess.run( + geisa_test_command(app_name, seconds, **settings), + check=False, + text=True, + capture_output=True, + ) + return result.stdout + + +def geisa_test_summary(output: str) -> dict | None: + """Find the summary in the geisa-test output. + + Args: + output: geisa-test standard output. + + Returns: + The geisa-test summary, or None when the application did not exit cleanly. + """ + for line in output.splitlines(): + if line.startswith('{"geisa-test-summary"'): + return json.loads(line)["geisa-test-summary"] + return None + + +def run_probe(app_name: str, test: str, duration_ms: int, **settings: object) -> dict | None: + """Replace the running reference application with a geisa-test probe run. + + The probe starts after discovery and runs for the given duration. + + Args: + app_name: Installed application name. + test: geisa-test probe name, such as cpu or memory. + duration_ms: Probe duration in milliseconds. + settings: Extra geisa-test environment variables. + + Returns: + The geisa-test summary, or None when the application did not exit cleanly. + """ + output = run_geisa_test( + app_name, + duration_ms // 1000 + PROBE_STARTUP_S, + GEISA_TEST_APP_MODE="test", + GEISA_TEST_APP_TEST=test, + GEISA_TEST_APP_TEST_DURATION_MS=duration_ms, + **settings, + ) + return geisa_test_summary(output) + + +def exercise_cpu_percent(app_name: str, pid: int, policy: dict) -> None: + """Check that a full CPU load is throttled down to the CPU quota. + + Worst case with a 100 ms period and a 20% quota (Q = 20 ms per period): + the probe starts at t=80 and stops at t=320, so elapsed_ms = 240. + It runs 80-100, 100-120, 200-220 and 300-320, so 80 ms of CPU, while 20% of 240 is only 48. + The first and last periods each add up to Q, so the bound is 20% of elapsed_ms + 2 * Q = 88. + + Args: + app_name: Installed application name. + pid: Container init process ID. + policy: Expected cgroup values for the application. + """ + stat = cgroup_path(pid) / "cpu.stat" + period_ms = int((cgroup_path(pid) / "cpu.max").read_text(encoding="utf-8").split()[1]) / 1000 + throttled = read_counters(stat)["nr_throttled"] + summary = run_probe(app_name, "cpu", PROBE_CPU_DURATION_MS) + if summary is None or summary["probe"]["result"] != "completed": + raise RuntimeError("geisa-test CPU probe did not complete") + probe = summary["probe"] + # The quota is granted once per cpu.max period. The probe might not start or + # end exactly on a period boundary, so it may get a full quota in each partial + # period at both ends. + allowed_ms = policy["cpu_percent"] * (probe["elapsed_ms"] + 2 * period_ms) / 100 + if probe["cpu_time_ms"] > allowed_ms: + usage = probe["cpu_time_ms"] * 100 / probe["elapsed_ms"] + raise RuntimeError( + f"CPU probe used {usage:.1f}% of a CPU, above the {policy['cpu_percent']}% quota" + ) + if read_counters(stat)["nr_throttled"] <= throttled: + raise RuntimeError("CPU probe was never throttled") + + +def exercise_memory(app_name: str, pid: int, policy: dict) -> None: + """Check that allocating past the memory limit gets the application killed. + + Args: + app_name: Installed application name. + pid: Container init process ID. + policy: Expected cgroup values for the application. + """ + events = cgroup_path(pid) / "memory.events" + killed = read_counters(events)["oom_kill"] + # Allocating app-ram is enough to go over the limit, because the process + # already uses memory for its code and buffers. + target = policy["memory_kib"] * 1024 + summary = run_probe( + app_name, "memory", PROBE_MEMORY_DURATION_MS, GEISA_TEST_APP_TEST_TARGET_BYTES=target + ) + if summary is not None: + if summary["probe"]["target_bytes"] != target: + raise RuntimeError(f"geisa-test cannot allocate {target} bytes, the manifest app-ram") + raise RuntimeError( + f"memory probe touched {summary['probe']['touched_bytes']} bytes " + f"with a limit of {policy['memory_kib']} KiB and was not stopped" + ) + if read_counters(events)["oom_kill"] <= killed: + raise RuntimeError("memory probe was not stopped by the cgroup OOM killer") + + +CGROUP_EXERCISES = { + "cpu-percent": exercise_cpu_percent, + "memory": exercise_memory, +} + + +def exercise_storage(app_name: str, field: str) -> None: + """Check that writing past a storage limit fails once the limit is reached. + + Args: + app_name: Installed application name. + field: Storage field suffix, such as persistent or nonpersistent. + """ + mount, test, directory_variable = STORAGE_PROBES[field] + limit = int(storage_limit(app_name, field)) * 1024 + # One MiB past the limit, so the write cannot fit even with no filesystem overhead. + target = limit + 1024 * 1024 + summary = run_probe( + app_name, + test, + PROBE_STORAGE_DURATION_MS, + GEISA_TEST_APP_TEST_TARGET_BYTES=target, + **{directory_variable: mount}, + ) + if summary is None: + raise RuntimeError(f"geisa-test {test} probe did not report a result") + probe = summary["probe"] + if probe["target_bytes"] != target: + raise RuntimeError(f"geisa-test cannot write {target} bytes, past the manifest limit") + if probe["result"] == "completed": + raise RuntimeError( + f"{test} probe wrote {probe['touched_bytes']} bytes in {mount} " + f"with a limit of {limit} bytes and was not stopped" + ) + # Some data must have been written, or the probe failed for another reason. + if not 0 < probe["touched_bytes"] <= limit: + raise RuntimeError( + f"{test} probe wrote {probe['touched_bytes']} bytes in {mount} " + f"with a limit of {limit} bytes ({probe['error']})" + ) + + +def check_storage_enforcement(arguments: argparse.Namespace) -> None: + """Exercise a storage limit of the reference application. + + Args: + arguments: Parsed command-line arguments for the storage-enforcement command. + """ + # A limit of zero means a read-only mount, which the configured-value check + # already covers. + if int(storage_limit(arguments.application, arguments.field)) == 0: + return + exercise_limit( + arguments.application, f"{arguments.field} storage", exercise_storage, arguments.field + ) + + def check_cgroup(arguments: argparse.Namespace) -> None: """Run the selected cgroup check for an application. Args: arguments: Parsed command-line arguments for the cgroup command. """ + # Are the cgroup v2 limits present and configured as expected? resources = read_manifest(arguments.application)["resources"] policy = { "memory_kib": int(resources["app-ram"]), "cpu_percent": int(resources["app-cpu"]) if "app-cpu" in resources else None, } + # app-cpu is optional: without it there is no percentage to check or exercise. + if arguments.constraint == "cpu-percent" and policy["cpu_percent"] is None: + return if is_reference(arguments.application): policy["cpuset_cpus"] = parse_cpu_list(arguments.reference_cpuset) policy["cpu_weight"] = int(arguments.reference_cpu_weight) CGROUP_CHECKS[arguments.constraint](arguments.pid, policy) + # Are the cgroup v2 limits enforced by the platform as expected? + # Note: enforcement checks are currently only available for the geisa-test reference application. + exercise = CGROUP_EXERCISES.get(arguments.constraint) + if exercise: + exercise_limit(arguments.application, arguments.constraint, exercise, arguments.pid, policy) + # --------------------------------------------------------------------------- # # Network policy # --------------------------------------------------------------------------- # -# Note: Stage 1 validates the nftables policy installed for each -# application's deployed manifest. Stage 2 will use geisa-test to attempt -# allowed and disallowed traffic and to intentionally violate the configured quota. -# TODO: Implement the geisa-test enforcement stage. def network_policy(app_name: str) -> dict | None: """Read direct network permissions from an application manifest. + Returns one entry per granted section (local under HAN, operator and + internet under FAN) with its inbound and outbound permissions and daily + volume, or None when no section grants anything. + Args: app_name: Installed application name. """ manifest = read_manifest(app_name) communication = manifest.get("communication", {}) - local = communication.get("local") if communication.get("HAN") is True else None - if not isinstance(local, dict) or not local.get("outbound"): - return None - permission = local["outbound"][0] - match = re.fullmatch(r"(?:(tcp|udp):)?.*:([0-9]+)", permission, re.IGNORECASE) - if not match: - raise ValueError(f"invalid local outbound network permission: {permission}") - transport_group = match.group(1) - port_group = match.group(2) - transport = transport_group.lower() if transport_group else None - port = int(port_group) - policy = { - "transport": transport, - "port": port, - "daily_volume": int(local["daily-volume"]), - } - return policy + granted = [] + if communication.get("HAN") is True: + granted.append("local") + if communication.get("FAN") is True: + granted.extend(("operator", "internet")) + sections = {} + for section in granted: + requirements = communication.get(section) + if not isinstance(requirements, dict): + continue + permissions = {"inbound": [], "outbound": []} + for direction in permissions: + for permission in requirements.get(direction, []): + match = PERMISSION_PATTERN.fullmatch(permission) + if not match: + raise ValueError(f"invalid {section} {direction} network permission: {permission}") + transport, address, port = match.groups() + permissions[direction].append( + { + "transport": transport.rstrip(":").lower() or None, + "address": address.strip("[]"), + "port": int(port), + } + ) + if not permissions["inbound"] and not permissions["outbound"]: + continue + permissions["daily_volume"] = int(requirements["daily-volume"]) + sections[section] = permissions + return sections or None + + +def policy_permissions(policy: dict): + """Yield the inbound and outbound permissions of every section. + + Args: + policy: Parsed network policy. + """ + for section in policy.values(): + yield from section["inbound"] + yield from section["outbound"] def init_pid(app_name: str) -> int: @@ -381,19 +680,19 @@ def granted_ports(ruleset: list) -> set: return ports -def expected_protocols(policy: dict) -> tuple: - """Return the protocols allowed by a network policy. +def expected_protocols(permission: dict) -> tuple: + """Return the protocols allowed by a network permission. Args: - policy: Parsed application network policy. + permission: Parsed network permission. """ - transport = policy["transport"] + transport = permission["transport"] protocols = ("tcp", "udp") if transport is None else (transport,) return protocols def check_standard_sockets(app_name: str) -> None: - """Check that the manifest port is allowed for its expected protocols. + """Check that every inbound and outbound manifest port is allowed for its protocols. Args: app_name: Installed application name. @@ -402,15 +701,16 @@ def check_standard_sockets(app_name: str) -> None: if policy is None: return ruleset = network_ruleset(app_name) - for protocol in expected_protocols(policy): - if not rule_grants(ruleset, protocol, policy["port"]): - raise RuntimeError( - f"no rule grants socket connectivity to {protocol} port {policy['port']}" - ) + for permission in policy_permissions(policy): + for protocol in expected_protocols(permission): + if not rule_grants(ruleset, protocol, permission["port"]): + raise RuntimeError( + f"no rule grants socket connectivity to {protocol} port {permission['port']}" + ) def check_acl(app_name: str) -> None: - """Check that network hooks deny traffic outside the manifest policy. + """Check that network hooks deny traffic outside the manifest policy and the MQTT port. Args: app_name: Installed application name. @@ -421,12 +721,17 @@ def check_acl(app_name: str) -> None: ruleset = network_ruleset(app_name) for hook in ("output", "input"): if hook_policy(ruleset, hook) != "drop": - raise RuntimeError(f"the {hook} hook is not deny-by-default") - for port in granted_ports(ruleset): - if port != policy["port"]: raise RuntimeError( - f"port {port} is granted but is absent from the manifest's network policy" + f"the {hook} hook does not deny traffic by default " + "(principle of least privilege)" ) + allowed = {permission["port"] for permission in policy_permissions(policy)} + allowed.add(mqtt_address(app_name)[1]) + extra = granted_ports(ruleset) - allowed + if extra: + raise RuntimeError( + f"ports {sorted(extra)} are granted but are absent from the manifest's network policy" + ) def volume_quotas(ruleset: list) -> list: @@ -447,7 +752,7 @@ def volume_quotas(ruleset: list) -> list: def check_volume_limit(app_name: str) -> None: - """Check that the manifest daily volume matches an nftables quota. + """Check that the daily volume of every manifest section matches an nftables quota. Args: app_name: Installed application name. @@ -459,11 +764,12 @@ def check_volume_limit(app_name: str) -> None: limits = volume_quotas(ruleset) if not limits: raise RuntimeError("no volume quota defined") - if policy["daily_volume"] not in limits: - raise RuntimeError( - "no volume quota matches the manifest limit of " - f"{policy['daily_volume']} bytes (found {sorted(limits)})" - ) + for name, section in policy.items(): + if section["daily_volume"] not in limits: + raise RuntimeError( + f"no volume quota matches the {name} daily volume of " + f"{section['daily_volume']} bytes (found {sorted(limits)})" + ) NETWORK_CHECKS = { @@ -473,6 +779,141 @@ def check_volume_limit(app_name: str) -> None: } +def mqtt_address(app_name: str) -> tuple[str, int]: + """Read the MQTT host and port that the platform gives to an application. + + Args: + app_name: Installed application name. + """ + path = Path(f"/proc/{init_pid(app_name)}/root/etc/geisa/mqtt.conf") + settings = {} + for line in path.read_text(encoding="utf-8").splitlines(): + name, _, value = line.partition("=") + settings[name.strip()] = value.strip() + if "HOST" not in settings or "PORT" not in settings: + raise RuntimeError(f"{path} has no HOST or PORT") + return settings["HOST"], int(settings["PORT"]) + + +def granted_host_ports(app_name: str) -> tuple[str, dict[int, int]]: + """Return the MQTT host and the outbound TCP ports that the manifest grants on it. + + geisa-test only opens TCP connections to its MQTT host, but + GEISA_TEST_APP_PORT changes the port. The test listens on that host as a + stand-in for any peer the application may reach, so only these ports can + be tried. The MQTT port itself is left out, since the broker uses it. Each + port maps to the daily volume of the section that grants it. + + Args: + app_name: Installed application name. + """ + host, mqtt_port = mqtt_address(app_name) + ports = {} + for section in network_policy(app_name).values(): + for permission in section["outbound"]: + if "tcp" in expected_protocols(permission) and permission["address"] in ("", host): + ports[permission["port"]] = section["daily_volume"] + ports.pop(mqtt_port, None) + if not ports: + raise RuntimeError(f"the manifest grants no outbound TCP port on {host} to try") + return host, ports + + +def connects(app_name: str, host: str, listener: socket.socket) -> bool: + """Run geisa-test on the port of a listener and tell whether it got through. + + Args: + app_name: Installed application name. + host: MQTT host of the application, where the listener listens. + listener: Listening socket. + """ + port = listener.getsockname()[1] + output = run_geisa_test(app_name, PROBE_STARTUP_S, GEISA_TEST_APP_PORT=port) + if f"host={host} port={port} " not in output: + raise RuntimeError(f"geisa-test did not try to connect to {host}:{port}") + return bool(select.select([listener], [], [], 0)[0]) + + +def exercise_standard_sockets(app_name: str) -> None: + """Check that the application reaches each TCP port that its manifest grants on the MQTT host. + + Args: + app_name: Installed application name. + """ + host, ports = granted_host_ports(app_name) + for port in sorted(ports): + # A listener on the granted port sees the connection that the policy lets through. + with socket.create_server((host, port)) as listener: + if not connects(app_name, host, listener): + raise RuntimeError(f"the application could not reach {host}:{port}, which its manifest grants") + + +def exercise_acl(app_name: str) -> None: + """Check that the application cannot reach a port that its manifest does not grant. + + Args: + app_name: Installed application name. + """ + host = mqtt_address(app_name)[0] + # A listener on the MQTT host, on a free port that the manifest does not + # grant, sees any connection that the policy lets through. + with socket.create_server((host, 0)) as listener: + port = listener.getsockname()[1] + if connects(app_name, host, listener): + raise RuntimeError(f"the application reached {host}:{port}, which its manifest does not grant") + + +def exercise_volume_limit(app_name: str) -> None: + """Check that the application cannot reach a granted port once its daily volume is used. + + geisa-test connects to a granted port and the test sends it more than the + daily volume. Received bytes count in the volume like sent ones. Then + geisa-test must no longer reach that port. + + Args: + app_name: Installed application name. + """ + host, ports = granted_host_ports(app_name) + port = min(ports) + volume = ports[port] + # A CONNACK header that announces the whole length keeps the MQTT client of + # geisa-test reading until it has received everything. + length = volume + VOLUME_MARGIN + header = bytearray([0x20]) + remaining = length + while True: + remaining, digit = divmod(remaining, 128) + header.append(digit | (0x80 if remaining else 0)) + if not remaining: + break + with socket.create_server((host, port)) as listener: + command = geisa_test_command(app_name, PROBE_STARTUP_S, GEISA_TEST_APP_PORT=port) + with subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) as probe: + while not select.select([listener], [], [], 0.1)[0]: + if probe.poll() is not None: + raise RuntimeError(f"the application could not reach {host}:{port}, which its manifest grants") + connection, _ = listener.accept() + with connection: + connection.settimeout(PROBE_STARTUP_S) + connection.sendall(bytes(header) + bytes(length)) + probe.wait() + # A new listener, so that a reconnection made while the volume was not + # used up yet is not counted. + with socket.create_server((host, port)) as listener: + if connects(app_name, host, listener): + raise RuntimeError( + f"the application still reached {host}:{port} after receiving {length} bytes, " + f"more than its daily volume of {volume} bytes" + ) + + +NETWORK_EXERCISES = { + "standard-sockets": exercise_standard_sockets, + "acl": exercise_acl, + "volume-limit": exercise_volume_limit, +} + + def configured_applications() -> list[str]: """Read the configured application names from the environment.""" applications = os.environ.get("APPLICATIONS", "").split() @@ -502,6 +943,9 @@ def check_network(arguments: argparse.Namespace) -> None: raise RuntimeError("no application manifest grants direct network access") for app_name in applications: NETWORK_CHECKS[arguments.check](app_name) + exercise = NETWORK_EXERCISES.get(arguments.check) + if exercise: + exercise_limit(app_name, f"network {arguments.check}", exercise) # --------------------------------------------------------------------------- # @@ -525,9 +969,11 @@ def parse_arguments() -> argparse.Namespace: storage.add_argument("application") storage.add_argument("field") - commands.add_parser( - "storage-limits", help="print every application storage limit as application|field|value" + enforcement = commands.add_parser( + "storage-enforcement", help="exercise a storage limit of the reference application" ) + enforcement.add_argument("application") + enforcement.add_argument("field", choices=STORAGE_FIELDS) launch = commands.add_parser( "launch-command", help="print the entry point declared in an application manifest" @@ -556,8 +1002,8 @@ def main() -> int: check_network(arguments) elif arguments.command == "storage-limit": print(storage_limit(arguments.application, arguments.field)) - elif arguments.command == "storage-limits": - print("\n".join(storage_limits())) + elif arguments.command == "storage-enforcement": + check_storage_enforcement(arguments) elif arguments.command == "launch-command": print(launch_command(arguments.application)) except (KeyError, OSError, RuntimeError, ValueError, subprocess.SubprocessError) as error: From b6a892fd9f04960dd6c97ab39b17450b5809304c Mon Sep 17 00:00:00 2001 From: Nghia Dam Date: Wed, 14 Oct 2026 17:31:46 -0400 Subject: [PATCH 5/5] GEISA-LEE-tests: exercise the file layout size limits with geisa-test The file layout size tests only compared the mount size with the manifest limit. When geisa-test is the reference application, they now also have it write past the limit inside the container and check that the write fails: - FS-MUST-40: /tmp MUST be limited in size. - FS-MUST-41: /home/geisa MUST be limited in size. Change-Id: I2c8a0b1f492a52bd873d2c7ee1b420fb53a58885 Signed-off-by: Nghia Dam --- src/GEISA-LEE-tests/tests.d/check-container.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/GEISA-LEE-tests/tests.d/check-container.sh b/src/GEISA-LEE-tests/tests.d/check-container.sh index 89ba5e24..8ea13d7b 100644 --- a/src/GEISA-LEE-tests/tests.d/check-container.sh +++ b/src/GEISA-LEE-tests/tests.d/check-container.sh @@ -561,7 +561,8 @@ exercise_container_storage_limit() { } ####################################### -# Check a container-visible mount size against its manifest limit. +# Check a container-visible mount size against its manifest limit, then check +# with geisa-test that the reference application cannot write past it. ####################################### check_container_mount_limit() { local path="$1" field="$2" app limit size @@ -570,6 +571,7 @@ check_container_mount_limit() { limit="$(container_backend storage-limit "${app}" "${field}")" || return 1 size="$(get_container_mount_size "${app}" "${path}")" || return 1 test "${size}" -le "${limit}" || return 1 + exercise_container_storage_limit "${app}" "${field}" || return 1 done return 0 }