feat: ship open runtime with durable recovery, contextual tools and SDK

.github/workflows/ci.yml

@@ -2,7 +2,7 @@ name: CI
 
 on:
   push:
-    branches: [main, develop, 'feat/**', 'fix/**']
+    branches: [main, develop, 'feat/**', 'fix/**', 'codex/**']
   pull_request:
     branches: [main, develop]
 
@@ -31,7 +31,7 @@ jobs:
         if: runner.os == 'Linux'
         run: |
           sudo apt-get update
-          sudo apt-get install -y cmake ninja-build libcurl4-openssl-dev
+          sudo apt-get install -y cmake ninja-build libcurl4-openssl-dev libsqlite3-dev
 
       - name: Configure CMake
         run: |
@@ -48,38 +48,66 @@ jobs:
         working-directory: build
         run: ctest --output-on-failure --timeout 120
 
+      - name: Run all synthetic evaluations
+        run: bash eval/run_all.sh
+
+      - name: Verify SDK and CLI archive
+        run: bash scripts/package.sh build dist/sparx-sdk.tar.gz
+
       - name: Upload build artifacts
-        if: matrix.os == 'ubuntu-latest'
         uses: actions/upload-artifact@v4
         with:
-          name: sparx-linux-x64
-          path: build/cli/sparx
+          name: sparx-sdk-${{ runner.os }}-${{ runner.arch }}
+          path: |
+            dist/*.tar.gz
+            dist/*.sha256
           retention-days: 7
-          if-no-files-found: ignore
+          if-no-files-found: error
 
   lint:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-
-      - name: Install clang-tidy
+      - name: Install analysis dependencies
+        run: sudo apt-get update && sudo apt-get install -y clang-tidy libcurl4-openssl-dev libsqlite3-dev
+      - name: Configure compilation database
+        run: cmake -S . -B build-lint -DBUILD_EVAL=OFF -DCMAKE_BUILD_TYPE=Debug
+      - name: Analyze public runtime (blocking)
         run: |
-          sudo apt-get update
-          sudo apt-get install -y clang-tidy
-
-      - name: Run clang-tidy on Agent OS modules
+          clang-tidy -p build-lint \
+            --checks='-*,clang-analyzer-*,bugprone-use-after-move,bugprone-sizeof-expression' \
+            --warnings-as-errors='*' \
+            --header-filter='/(core|backends|include/master_agent/runtime|examples/inventory_agent)/' \
+            core/reference_runtime.cpp core/durable_store.cpp backends/http_model.cpp \
+            cli/src/reference_main.cpp examples/inventory_agent/main.cpp
+
+  sanitizer:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install dependencies
+        run: sudo apt-get update && sudo apt-get install -y clang libcurl4-openssl-dev libsqlite3-dev
+      - name: Build with ASan and UBSan
+        env:
+          CC: clang
+          CXX: clang++
+        run: |
+          cmake -S . -B build-sanitize -DBUILD_EVAL=OFF -DCMAKE_BUILD_TYPE=Debug \
+            -DCMAKE_CXX_FLAGS='-fsanitize=address,undefined -fno-omit-frame-pointer' \
+            -DCMAKE_EXE_LINKER_FLAGS='-fsanitize=address,undefined'
+          cmake --build build-sanitize --parallel 2 --target test_reference_runtime test_execution_context \
+            test_durable_runtime durable_probe sparx http_model_probe inventory_agent test_harness
+      - name: Runtime, network and recovery sanitizer contracts
+        env:
+          ASAN_OPTIONS: detect_leaks=1:halt_on_error=1
+          UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1
+        run: ctest --test-dir build-sanitize --output-on-failure --timeout 90 -R 'test_reference_runtime|test_execution_context|test_durable_runtime|crash_recovery_contract|http_model_contract|inventory_service_contract|test_harness'
+      - name: Build and test without optional dependencies
         run: |
-          clang-tidy \
-            --checks='-*,bugprone-*,performance-*,modernize-*,-modernize-use-trailing-return-type' \
-            cli/src/sparx_agent_scheduler.cpp \
-            cli/src/sparx_context_manager.cpp \
-            cli/src/sparx_memory_manager.cpp \
-            cli/src/sparx_access_control.cpp \
-            cli/src/sparx_tool_registry.cpp \
-            -- -std=c++17 -Icli/include -Iinclude \
-               -Ithird_party/memory_short_term/include \
-               -Ithird_party \
-            || true  # Non-blocking for now
+          cmake -S . -B build-minimal -DCMAKE_BUILD_TYPE=Release -DBUILD_EVAL=OFF \
+            -DMASTER_AGENT_ENABLE_HTTP=OFF -DMASTER_AGENT_ENABLE_STORAGE=OFF
+          cmake --build build-minimal --parallel 2
+          ctest --test-dir build-minimal --output-on-failure --timeout 90
 
   security-check:
     runs-on: ubuntu-latest

.github/workflows/release.yml

@@ -40,33 +40,28 @@ jobs:
         if: runner.os == 'Linux'
         run: |
           sudo apt-get update
-          sudo apt-get install -y cmake ninja-build libcurl4-openssl-dev
+          sudo apt-get install -y cmake ninja-build libcurl4-openssl-dev libsqlite3-dev
 
       - name: Configure
         run: |
           cmake -B build \
             -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \
             -DSPARX_VERSION=${{ steps.version.outputs.version }} \
-            -DMASTER_AGENT_BUILD_TESTS=OFF \
+            -DMASTER_AGENT_BUILD_TESTS=ON \
             -DMASTER_AGENT_BUILD_CLI=ON \
             ${{ matrix.cmake_flags || '' }}
 
       - name: Build
         run: cmake --build build --config ${{ env.BUILD_TYPE }} -j$(nproc 2>/dev/null || sysctl -n hw.logicalcpu)
 
-      - name: Package
-        run: |
-          mkdir -p dist
-          # The sparx CLI requires the proprietary kernel; in OSS builds it is
-          # absent. Package whatever binaries the build produced.
-          if [ -f build/cli/sparx ]; then
-            cp build/cli/sparx dist/
-          fi
-          for bin in build/bench_strategic build/eval_*; do
-            [ -f "$bin" ] && cp "$bin" dist/ || true
-          done
-          cp README.md LICENSE dist/
-          cd dist && tar czf ../sparx-${{ matrix.target }}.tar.gz .
+      - name: Test
+        run: ctest --test-dir build --output-on-failure --timeout 120
+
+      - name: Evaluate
+        run: bash eval/run_all.sh
+
+      - name: Package and verify installed SDK
+        run: bash scripts/package.sh build sparx-${{ matrix.target }}.tar.gz
 
       - name: Checksum
         run: shasum -a 256 sparx-${{ matrix.target }}.tar.gz > sparx-${{ matrix.target }}.tar.gz.sha256
@@ -98,4 +93,4 @@ jobs:
             artifacts/**/*.sha256
           generate_release_notes: true
           draft: false
-          prerelease: ${{ contains(github.ref, '-rc') || contains(github.ref, '-beta') }}
+          prerelease: ${{ contains(github.ref, '-') }}

.gitignore

@@ -56,3 +56,6 @@ dist/
 
 # Claude
 .claude/
+
+/build-*/
+/install-*/

ARCHITECTURE.md

@@ -1,3 +1,9 @@
+> Public 0.4.0-alpha.3: use the open reference runtime in `core/`,
+> `include/master_agent/runtime/`, `backends/`, and `cli/src/reference_main.cpp`.
+> The current runnable contract and commands are in README.md and
+> docs/reference_runtime.md. Legacy kernel descriptions below require private
+> source; experimental modules are not automatically integrated into this CLI.
+
 # Architecture & Code Map
 
 ## Directory Structure

CHANGELOG.md

@@ -1,3 +1,34 @@
+# 0.4.0-alpha.3 — execution contracts and HTTP business integration
+
+- Add contextual tools with session/request identity, stable scoped idempotency
+  keys, monotonic deadlines and shared cooperative cancellation signals.
+- Introduce typed committed/failed/unknown outcomes; preserve the legacy callback API.
+- Check cancellation before dispatch; preserve authoritative results returned after cancellation.
+- Add an independent HTTP inventory service with atomic stock/idempotency receipts,
+  lost-response injection, external reconciliation and service/runtime restart tests.
+- Make public-runtime static analysis blocking; run ASan/UBSan and dependency-free
+  builds in CI. Verify contextual consumers and the HTTP example from SDK archives.
+
+# 0.4.0-alpha.2 — durable receipts and recovery
+
+- Add an opt-in SQLite store with WAL/FULL synchronization, exclusive ownership,
+  persistent request outcomes/history, conservative UNKNOWN recovery and audited reconciliation.
+- Fail closed on storage errors; write validated dispatch information before tool callbacks.
+- Add CLI state, JSONL, recovery, reconciliation, HTTP timeout and credential-env options.
+- Test real process termination after an external fsync, storage failure injection,
+  restart replay/history, locking, corrupted stores and schema versions.
+- Package provenance metadata/checksums and validate persistent installed consumers.
+
+# 0.4.0-alpha — open reference runtime
+
+- Add an independently runnable reference CLI and installable C++ Core/Http SDK.
+- Validate tool arguments; scope in-memory request replay and history by session.
+- Keep test expectations active in Release and verify installed/relocated consumers.
+- Fix eval executable paths, failure exit status, and Bash counter portability.
+- Bound edge/cloud inference waits and preserve backend lifetime after timeouts.
+- Initialize intent metadata and isolate speculation test persistence.
+- Require verified CLI + SDK archives in CI/release; document experimental limits.
+
 # Changelog
 
 All notable changes to OAK (Open Agent Kernel) will be documented in this file.

CLAUDE.md

@@ -1,3 +1,9 @@
+> Public 0.4.0-alpha.3: use the open reference runtime in `core/`,
+> `include/master_agent/runtime/`, `backends/`, and `cli/src/reference_main.cpp`.
+> The current runnable contract and commands are in README.md and
+> docs/reference_runtime.md. Legacy kernel descriptions below require private
+> source; experimental modules are not automatically integrated into this CLI.
+
 # OAK (Open Agent Kernel) — CLAUDE.md
 
 ## Project Overview
@@ -13,8 +19,8 @@ cmake --build build -j$(nproc)
 ctest --test-dir build --output-on-failure
 ```
 
-OSS build produces: test binaries + eval harnesses + bench_strategic.
-Full CLI (`sparx`) requires proprietary kernel source in `src/`.
+OSS build produces the reference sparx CLI, installable Core/Http SDK, tests and optional eval binaries.
+The legacy full CLI still requires private kernel source. See docs/reference_runtime.md.
 
 ## Code Conventions
 
@@ -54,4 +60,4 @@ Full CLI (`sparx`) requires proprietary kernel source in `src/`.
 
 ## Version
 
-0.3.0-alpha. Versions follow semver. Don't inflate beyond actual stability.
+0.4.0-alpha.3. Versions follow semver. Don't inflate beyond actual stability.

CMakeLists.txt

@@ -1,9 +1,13 @@
 cmake_minimum_required(VERSION 3.18)
 
-project(MasterAgent VERSION 0.3.0 LANGUAGES CXX)
+project(MasterAgent VERSION 0.4.0 LANGUAGES CXX)
 
 include(GNUInstallDirs)
 include(CMakePackageConfigHelpers)
+find_package(Threads REQUIRED)
+option(MASTER_AGENT_ENABLE_STORAGE "Enable durable SQLite request storage (Linux/macOS)" ON)
+option(MASTER_AGENT_ENABLE_HTTP "Build the HTTP model adapter (libcurl)" ON)
+option(BUILD_EVAL "Build synthetic evaluation programs" ON)
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -45,10 +49,17 @@ if(MINGW)
     endfunction()
 endif()
 
+# Legacy kernel files are optional; the public build uses core/reference_runtime.cpp.
+set(_core_source_present OFF)
+if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/agent_dispatch/atomic_lineage.cpp")
+    set(_core_source_present ON)
+endif()
+
 set(MEMORY_SHORT_TERM_SOURCE_ROOT
     "${CMAKE_SOURCE_DIR}/third_party/memory_short_term"
     CACHE PATH "Short-term memory source root")
 
+if(_core_source_present)
 if(NOT EXISTS
    "${MEMORY_SHORT_TERM_SOURCE_ROOT}/include/vehicle_memory/conversation_journal.h")
     message(FATAL_ERROR
@@ -72,14 +83,7 @@ if(WIN32)
     target_link_libraries(memory_short_term_core PRIVATE bcrypt)
 endif()
 
-# Check if the core source tree is present.
-# The full kernel source (src/) is proprietary and not included in the OSS
-# distribution. When absent, we build a stub library so that downstream
-# targets (cli, tests, eval) can still link.
-set(_core_source_present OFF)
-if(EXISTS "${CMAKE_SOURCE_DIR}/src/agent_dispatch/atomic_lineage.cpp")
-    set(_core_source_present ON)
-endif()
+endif() # legacy short-term memory library
 
 if(_core_source_present)
 
@@ -214,21 +218,37 @@ set_target_properties(master_agent_core PROPERTIES EXPORT_NAME Core)
 add_library(MasterAgent::Core ALIAS master_agent_core)
 
 else()
-# ─── OSS stub: core source not present ───────────────────────────────────
-# Build an INTERFACE library so cli/ and tests/ can still compile and link.
-# The cli implements all Agent OS functionality directly; it only needs
-# headers from include/ for type definitions.
-message(STATUS "Core source not present (OSS build) — using stub library")
-add_library(master_agent_core INTERFACE)
-target_compile_features(master_agent_core INTERFACE cxx_std_17)
-target_include_directories(master_agent_core INTERFACE
-    "$<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/include>"
-    "$<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/third_party>"
+# The public reference runtime is a real, independently usable library.
+message(STATUS "Building the open reference runtime")
+add_library(master_agent_core STATIC core/reference_runtime.cpp core/durable_store.cpp)
+target_compile_features(master_agent_core PUBLIC cxx_std_17)
+target_include_directories(master_agent_core PUBLIC
+    "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>"
+    "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third_party>"
     "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>")
-target_link_libraries(master_agent_core INTERFACE memory_short_term_core)
+target_link_libraries(master_agent_core PUBLIC Threads::Threads)
+if(MASTER_AGENT_ENABLE_STORAGE)
+    if(NOT UNIX)
+        message(FATAL_ERROR "Durable store currently supports Linux/macOS; set MASTER_AGENT_ENABLE_STORAGE=OFF")
+    endif()
+    find_package(SQLite3 REQUIRED)
+    target_link_libraries(master_agent_core PRIVATE SQLite::SQLite3)
+    target_compile_definitions(master_agent_core PRIVATE MASTER_AGENT_HAS_SQLITE=1)
+endif()
+set_target_properties(master_agent_core PROPERTIES EXPORT_NAME Core POSITION_INDEPENDENT_CODE ON)
 add_library(MasterAgent::Core ALIAS master_agent_core)
 endif()
 
+if(MASTER_AGENT_ENABLE_HTTP)
+    find_package(CURL REQUIRED)
+    add_library(master_agent_http STATIC backends/http_model.cpp)
+    target_link_libraries(master_agent_http PUBLIC MasterAgent::Core PRIVATE CURL::libcurl)
+    set_target_properties(master_agent_http PROPERTIES EXPORT_NAME Http)
+    add_library(MasterAgent::Http ALIAS master_agent_http)
+    install(TARGETS master_agent_http EXPORT MasterAgentTargets
+        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
+endif()
+
 # Prompt and Skill use runtime-managed JSON/text configuration. Mirror the
 # source configuration into the build tree so local executables and CTest share
 # the same relative paths as a source-tree launch.
@@ -372,15 +392,21 @@ elseif(MASTER_AGENT_BUILD_TESTS)
     endif()
 endif()
 
-add_subdirectory(eval)
+if(BUILD_EVAL)
+    add_subdirectory(eval)
+endif()
+
+if(MASTER_AGENT_BUILD_TESTS AND NOT _core_source_present)
+    include(cmake/ReferenceTests.cmake)
+endif()
 
 if(_core_source_present)
 install(TARGETS master_agent_core memory_short_term_core
     EXPORT MasterAgentTargets
     ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
     LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
 else()
-install(TARGETS memory_short_term_core
+install(TARGETS master_agent_core
     EXPORT MasterAgentTargets
     ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
     LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})

CONTRIBUTING.md

@@ -1,3 +1,9 @@
+> Public 0.4.0-alpha.3: use the open reference runtime in `core/`,
+> `include/master_agent/runtime/`, `backends/`, and `cli/src/reference_main.cpp`.
+> The current runnable contract and commands are in README.md and
+> docs/reference_runtime.md. Legacy kernel descriptions below require private
+> source; experimental modules are not automatically integrated into this CLI.
+
 # Contributing to OAK (Open Agent Kernel)
 
 Thank you for your interest in contributing! OAK is an open-source project