From 7975bf3141a03b41e1211abc4a53eb2c1570fb90 Mon Sep 17 00:00:00 2001 From: gabewillen Date: Thu, 6 Aug 2026 18:50:17 -0500 Subject: [PATCH 1/3] feat: support caller media in text processing --- README.md | 4 ++ include/cortext/capi.h | 34 +++++++++-- include/cortext/cortext.hpp | 20 ++++-- src/capi.cpp | 59 ++++++++++++++++++ src/cortext.cpp | 63 ++++++++++++++++++- tests/cortext.test.cpp | 119 ++++++++++++++++++++++++++++++++++++ 6 files changed, 285 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index c3aa8ed1..f9b1d372 100644 --- a/README.md +++ b/README.md @@ -267,6 +267,10 @@ Core calls: - `ProcessText`, `ProcessAudio`, `ProcessImage`: process a signal (default retention is Natural — no forced turn edge; storage via write gate). +- Media overloads keep the canonical input for embedding and processing. + Non-empty caller media replaces the stored payload and MIME type. Empty Media + omits the stored payload for all media overloads; legacy calls without Media + retain their canonical payload behavior. - `Retention::Durable`: force turn boundary + force write (chat convenience). - `Retention::Boundary`: force turn boundary only; write gate still applies. - `Retention::Ephemeral`: process and retrieve without storing the input. diff --git a/include/cortext/capi.h b/include/cortext/capi.h index 609c213b..a2f97f4c 100644 --- a/include/cortext/capi.h +++ b/include/cortext/capi.h @@ -190,13 +190,14 @@ extern "C" int signal_filter_text_enabled; } cortext_config; - /// @brief Optional source media payload persisted for audio/image signals. + /// @brief Optional source media payload persisted for a processed signal. /// - /// The canonical process input is still passed through pcm/data arguments. - /// When this struct carries bytes, those bytes are stored as the durable - /// signal/memory payload using mimetype. Passing NULL, or a struct with - /// size == 0, means process without storing an audio/image payload. Media - /// bytes are copied during the call. + /// The canonical process input is still passed through text/pcm/data + /// arguments. When this struct carries bytes, those bytes are stored as the + /// durable signal/memory payload using mimetype. NULL or size == 0 omits + /// the canonical payload for every overload that accepts media. Legacy + /// process functions retain their canonical payload behavior. Media bytes + /// are copied during the call. typedef struct cortext_media { const uint8_t *data; @@ -310,6 +311,15 @@ extern "C" CORTEXT_EXPORT int cortext_process_text (cortext_handle h, const char *text, const char *source_id); + /// @brief Processes text while storing caller-provided source media bytes. + /// + /// Text remains the canonical processing input. If media is non-NULL and + /// media->size > 0, media->data is stored using media->mimetype instead of + /// the canonical UTF-8 text bytes. An empty media omits the stored payload. + CORTEXT_EXPORT int cortext_process_text_with_media ( + cortext_handle h, const char *text, const char *source_id, + const cortext_media *media); + /// @brief Processes audio input through the memory system. /// @param h Handle to a Cortext instance. /// @param pcm PCM audio samples (float32, must be non-NULL). @@ -432,6 +442,18 @@ extern "C" cortext_handle h, const char *text, const char *source_id, const cortext_process_json_options *options); + /// @brief Processes text with source media and returns Context JSON. + CORTEXT_EXPORT char *cortext_process_text_with_media_json ( + cortext_handle h, const char *text, const char *source_id, + const cortext_media *media); + + /// @brief Processes text with source media and returns configurable Context + /// JSON. + CORTEXT_EXPORT char *cortext_process_text_with_media_json_with_options ( + cortext_handle h, const char *text, const char *source_id, + const cortext_media *media, + const cortext_process_json_options *options); + /// @brief Processes audio input and returns the resulting Context as JSON. CORTEXT_EXPORT char *cortext_process_audio_json (cortext_handle h, const float *pcm, diff --git a/include/cortext/cortext.hpp b/include/cortext/cortext.hpp index d06836f1..49037768 100644 --- a/include/cortext/cortext.hpp +++ b/include/cortext/cortext.hpp @@ -161,12 +161,12 @@ class CORTEXT_EXPORT Cortext bool signal_filter_text_enabled = false; }; - /// @brief Optional source media payload to persist with an audio/image signal. + /// @brief Optional source media payload to persist with a signal. /// - /// Processing still uses the canonical audio/image input passed to - /// ProcessAudio/ProcessImage. When Media contains data, those bytes and this - /// MIME type become the durable payload instead of the canonical processing - /// representation. Media bytes are copied during the call. + /// Processing still uses the canonical input passed to ProcessText, + /// ProcessAudio, or ProcessImage. When Media contains data, those bytes and + /// this MIME type become the durable payload instead of the canonical + /// processing representation. Media bytes are copied during the call. struct Media { const std::uint8_t *data = nullptr; @@ -237,6 +237,16 @@ class CORTEXT_EXPORT Cortext Context ProcessText (const std::string &text, const std::string &source_id, Retention retention = Retention::Natural); + /// @brief Process text while storing caller-provided source media. + /// + /// The text remains the canonical embedding input. If media contains bytes, + /// those bytes are persisted as the signal/memory payload using + /// media.mimetype instead of the UTF-8 text bytes. If media is empty, no + /// text payload is persisted, matching the audio/image Media overloads. + Context ProcessText (const std::string &text, const std::string &source_id, + const Media &media, + Retention retention = Retention::Natural); + /// @brief Process text input with an explicit timestamp (ms since epoch). /// @param text The text content to process. /// @param source_id Opaque source stream identifier used for provenance and diff --git a/src/capi.cpp b/src/capi.cpp index f3765b44..ff9cf7b3 100644 --- a/src/capi.cpp +++ b/src/capi.cpp @@ -1508,6 +1508,28 @@ extern "C" cortext::Retention::Durable); }); } + int + cortext_process_text_with_media (cortext_handle h, const char *text, + const char *source_id, + const cortext_media *media) + { + auto *p = cast_handle (h); + if (!p || !text || !source_id) + { + set_last_error ("handle, text, and source_id must all be non-NULL"); + return 1; + } + if (!valid_media_or_set_error (media)) + { + return 1; + } + + return invoke_status_only ([&] { + (void)p->ProcessText (std::string (text), std::string (source_id), + media_from_c (media), cortext::Retention::Durable); + }); + } + int cortext_process_audio (cortext_handle h, const float *pcm, size_t num_samples, const char *source_id) @@ -1721,6 +1743,43 @@ extern "C" include_embedding_from_options (options)); } + char * + cortext_process_text_with_media_json (cortext_handle h, const char *text, + const char *source_id, + const cortext_media *media) + { + const cortext_process_json_options options { + sizeof (cortext_process_json_options), 1, CORTEXT_RETENTION_DURABLE, 0 + }; + return cortext_process_text_with_media_json_with_options ( + h, text, source_id, media, &options); + } + + char * + cortext_process_text_with_media_json_with_options ( + cortext_handle h, const char *text, const char *source_id, + const cortext_media *media, + const cortext_process_json_options *options) + { + auto *p = cast_handle (h); + if (!p || !text || !source_id) + { + set_last_error ("handle, text, and source_id must all be non-NULL"); + return nullptr; + } + if (!valid_media_or_set_error (media)) + { + return nullptr; + } + + const auto retention = retention_from_options (options); + return invoke_json ([&] { + return p->ProcessText (std::string (text), std::string (source_id), + media_from_c (media), retention); + }, + include_embedding_from_options (options)); + } + char * cortext_process_audio_json (cortext_handle h, const float *pcm, size_t num_samples, const char *source_id) diff --git a/src/cortext.cpp b/src/cortext.cpp index 4263877a..ab729176 100644 --- a/src/cortext.cpp +++ b/src/cortext.cpp @@ -163,7 +163,6 @@ ValidateImageInput (const std::uint8_t *data, int width, int height, } #endif -#if !defined(CORTEXT_DISABLE_AUDIO) || !defined(CORTEXT_DISABLE_IMAGE) void ValidateMedia (const Cortext::Media &media) { @@ -196,7 +195,6 @@ ApplyStoredMedia (Signal &signal, const Cortext::Media &media) media.data + media.size); signal.mimetype = media.mimetype; } -#endif #if defined(CORTEXT_DISABLE_AUDIO) [[noreturn]] void @@ -438,7 +436,11 @@ PayloadMatchesMemorySurface (const std::string &memory_modality, { return false; } - if (memory_modality == "text" || memory_mime == "text/plain" + // Text modality can carry caller-provided binary payloads (for example a + // token KV cache). Only canonical text/plain payloads require text-content + // validation; a custom MIME type declares an opaque payload surface. + if (memory_mime == "text/plain" + || (memory_mime.empty () && memory_modality == "text") || signal_mime == "text/plain") { return IsLikelyTextPayload (payload); @@ -2045,6 +2047,61 @@ Cortext::ProcessText (const std::string &text, const std::string &source_id, return result; } +Cortext::Context +Cortext::ProcessText (const std::string &text, const std::string &source_id, + const Media &media, Retention retention) +{ + if (!impl_) + { + throw std::runtime_error ("Cortext not initialized"); + } + ValidateMedia (media); + telemetry::ScopedSpan span ("cortext.api.process_text"); + const auto total_start = std::chrono::steady_clock::now (); + const auto filter_decision = impl_->signal_filter.EvaluateText (text); + if (filter_decision.evaluated && !filter_decision.accepted) + { + span.SetStatusOk (); + return MakeFilteredContext (filter_decision, total_start); + } + std::vector v; + telemetry::ScopedSpan encode_span ("cortext.encode"); + impl_->encoder->EncodeText (text, v); + encode_span.SetStatusOk (); + const auto encode_end = std::chrono::steady_clock::now (); + + // Build signal with optional caller-provided source payload. Text remains + // the canonical embedding input. + const auto process_start = std::chrono::steady_clock::now (); + cortext::Signal s; + s.embedding = ToEigen (RetrievalEmbeddingView (v)); + s.soft_anchor_embedding = SoftAnchorEmbeddingView (v); + s.timestamp = impl_->NowMillis (); + s.source_id = source_id; + s.retention = retention; + s.modality = "text"; + ApplyStoredMedia (s, media); + + auto out = impl_->processor->Process (s); + const auto process_end = std::chrono::steady_clock::now (); + span.SetAttribute ("cortext.candidate_memory_count", + static_cast (out.candidate_memory_ids.size ())); + span.SetAttribute ("cortext.used_memory_count", + static_cast (out.used_memory_ids.size ())); + span.SetStatusOk (); + telemetry::ScopedSpan hydrate_span ("cortext.hydrate"); + const auto hydrate_start = std::chrono::steady_clock::now (); + Cortext::Context result = impl_->HydrateContext (out, true, &s.embedding); + const auto hydrate_end = std::chrono::steady_clock::now (); + hydrate_span.SetStatusOk (); + result.encode_ms = ToMillis (encode_end - total_start); + result.process_ms = ToMillis (process_end - process_start); + result.hydrate_ms = ToMillis (hydrate_end - hydrate_start); + result.total_ms = ToMillis (hydrate_end - total_start); + ApplySignalFilterDecision (result.output, filter_decision); + return result; +} + Cortext::Context Cortext::ProcessTextAt (const std::string &text, const std::string &source_id, std::uint64_t timestamp, Retention retention) diff --git a/tests/cortext.test.cpp b/tests/cortext.test.cpp index 3b281bb7..c10f80c2 100644 --- a/tests/cortext.test.cpp +++ b/tests/cortext.test.cpp @@ -678,6 +678,91 @@ TEST_CASE ("Cortext media overloads validate source media before encoding", std::invalid_argument); } +TEST_CASE ("Cortext text media payload overrides storage, not embedding", + "[cortext][media][text][aist]") +{ + ScopedTempDb temp_db; + cortext::Cortext::Config cfg; + const std::string &db_path = temp_db.path (); + + auto ctx = cortext::Cortext::Create (cfg, db_path); + REQUIRE (ctx != nullptr); + + const std::string text = "canonical text remains the embedding input"; + const std::vector supplied_bytes = { + 0x00, 0x01, 0x7f, 0xff, 0x20 + }; + const cortext::Cortext::Media media{ + supplied_bytes.data (), supplied_bytes.size (), + "application/x-token-kv-cache" + }; + + const auto default_context + = ctx->ProcessText (text, "text/default", cortext::Retention::Durable); + const auto supplied_context = ctx->ProcessText ( + text, "text/media", media, cortext::Retention::Durable); + const auto empty_media_context = ctx->ProcessText ( + text, "text/empty-media", cortext::Cortext::Media{}, + cortext::Retention::Durable); + REQUIRE (default_context.output.stored_memory_id.has_value ()); + REQUIRE (supplied_context.output.stored_memory_id.has_value ()); + REQUIRE (empty_media_context.output.stored_memory_id.has_value ()); + ctx->Flush (); + + const auto default_hydrated = ctx->DebugHydrateForTest ( + { *default_context.output.stored_memory_id }, {}); + REQUIRE (default_hydrated.retrieved_memory.size () == 1); + REQUIRE (default_hydrated.retrieved_memory[0].content.size () == 1); + const std::vector default_bytes (text.begin (), text.end ()); + REQUIRE (default_hydrated.retrieved_memory[0].content[0] == default_bytes); + REQUIRE (default_hydrated.retrieved_memory[0].mimetype == "text/plain"); + + const auto supplied_hydrated = ctx->DebugHydrateForTest ( + { *supplied_context.output.stored_memory_id }, {}); + REQUIRE (supplied_hydrated.retrieved_memory.size () == 1); + REQUIRE (supplied_hydrated.retrieved_memory[0].content.size () == 1); + REQUIRE (supplied_hydrated.retrieved_memory[0].content[0] == supplied_bytes); + REQUIRE (supplied_hydrated.retrieved_memory[0].mimetype + == "application/x-token-kv-cache"); + // Caller media changes only the stored payload; the canonical text still + // drives the same retrieval embedding as the legacy overload. + REQUIRE (supplied_context.embedding == default_context.embedding); + + // Empty Media deliberately creates a contentless signal. Hydration omits + // contentless memories, so verify the persisted signal row directly. + auto store = cortext::SQLiteStore::Create (db_path.c_str ()); + const auto empty_rows = store->Execute ( + "SELECT COALESCE(mime, '') AS mime, blob_id IS NULL AS no_blob " + "FROM signals WHERE memory_id = ? ORDER BY signal_id DESC LIMIT 1", + { *empty_media_context.output.stored_memory_id }); + REQUIRE (empty_rows.size () == 1); + REQUIRE (std::any_cast (empty_rows[0].at ("mime")).empty ()); + REQUIRE (AnyToLongLong (empty_rows[0].at ("no_blob")) == 1); +} + +TEST_CASE ("Cortext text media overload validates media before encoding", + "[cortext][media][text][safety][aist]") +{ + ScopedTempDb temp_db; + auto ctx = cortext::Cortext::Create ({}, temp_db.path ()); + REQUIRE (ctx != nullptr); + + const std::uint8_t payload[] = { 0x01, 0x02 }; + const cortext::Cortext::Media missing_data{ + nullptr, sizeof (payload), "text/plain" + }; + const cortext::Cortext::Media missing_mime{ + payload, sizeof (payload), "" + }; + + REQUIRE_THROWS_AS ( + ctx->ProcessText ("text", "text/source", missing_data), + std::invalid_argument); + REQUIRE_THROWS_AS ( + ctx->ProcessText ("text", "text/source", missing_mime), + std::invalid_argument); +} + TEST_CASE ("Cortext media ingress validates processing inputs before use", "[cortext][media][safety][aist]") { @@ -2455,6 +2540,19 @@ TEST_CASE ("C API handles NULL inputs correctly", cortext_free (h); } + SECTION ("cortext_process_text_with_media rejects invalid media") + { + ScopedTempDb temp_db; + auto h = cortext_create (0.5, 0.5, 0.5, temp_db.path ().c_str ()); + REQUIRE (h != nullptr); + const std::uint8_t payload[2] = { 0x01, 0x02 }; + cortext_media media{ nullptr, sizeof (payload), "text/plain" }; + CHECK (cortext_process_text_with_media (h, "text", "src", &media) == 1); + REQUIRE (std::string (cortext_last_error ()) + == "media data must be non-NULL when media size is non-zero"); + cortext_free (h); + } + SECTION ("cortext_process_audio returns 1 for NULL pcm") { ScopedTempDb temp_db; @@ -2545,6 +2643,27 @@ TEST_CASE ("C API handles NULL inputs correctly", cortext_free (h); } + SECTION ("text-with-media C API wrappers preserve media payload contract") + { + ScopedTempDb temp_db; + auto h = cortext_create (0.5, 0.5, 0.5, temp_db.path ().c_str ()); + REQUIRE (h != nullptr); + const std::uint8_t payload[4] = { 0x00, 0x7f, 0xff, 0x01 }; + cortext_media media{ payload, sizeof (payload), "text/custom" }; + + CHECK (cortext_process_text_with_media (h, "canonical", "capi/media", + &media) + == 0); + char *json_ptr = cortext_process_text_with_media_json ( + h, "canonical json", "capi/media-json", &media); + REQUIRE (json_ptr != nullptr); + const auto parsed = nlohmann::json::parse (json_ptr); + REQUIRE (parsed.at ("boundary_type") == "explicit_turn"); + REQUIRE (parsed.at ("output").at ("stored_memory_id").is_number_integer ()); + cortext_string_free (json_ptr); + cortext_free (h); + } + SECTION ("cortext_embed_text_json reports NULL arguments") { char *json_ptr = cortext_embed_text_json (nullptr, "hello"); From e30f7de59c6556645361667477ac53244f697a73 Mon Sep 17 00:00:00 2001 From: gabewillen Date: Thu, 6 Aug 2026 18:56:19 -0500 Subject: [PATCH 2/3] fix(wasm): export media text processing APIs --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index f6f0c636..7a087789 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -806,6 +806,8 @@ if(EMSCRIPTEN AND BUILD_WASM) "_cortext_last_error" "_cortext_string_free" "_cortext_process_text_json" + "_cortext_process_text_with_media_json" + "_cortext_process_text_with_media_json_with_options" "_cortext_wasm_process_text_json" "_cortext_process_audio_json" "_cortext_process_image_json" From 1d1e891de1acf8e0acaad8006a96be3d26468174 Mon Sep 17 00:00:00 2001 From: gabewillen Date: Thu, 6 Aug 2026 21:22:12 -0500 Subject: [PATCH 3/3] test(capi): cover NULL text media wrappers --- tests/cortext.test.cpp | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/cortext.test.cpp b/tests/cortext.test.cpp index c10f80c2..f8df2dbc 100644 --- a/tests/cortext.test.cpp +++ b/tests/cortext.test.cpp @@ -2496,6 +2496,43 @@ TEST_CASE ("C API handles NULL handles without a model", } } +TEST_CASE ("C API text media wrappers accept NULL media as empty media", + "[cortext][capi][media][text][aist]") +{ + ScopedTempDb temp_db; + auto h = cortext_create (0.5, 0.5, 0.5, temp_db.path ().c_str ()); + REQUIRE (h != nullptr); + + CHECK (cortext_process_text_with_media ( + h, "canonical", "capi/null-media", nullptr) + == 0); + REQUIRE (cortext_flush (h) == 0); + + char *json_ptr = cortext_process_text_with_media_json ( + h, "canonical json", "capi/null-media-json", nullptr); + REQUIRE (json_ptr != nullptr); + const auto parsed = nlohmann::json::parse (json_ptr); + REQUIRE (parsed.at ("boundary_type") == "explicit_turn"); + REQUIRE (parsed.at ("output").at ("stored_memory_id").is_number_integer ()); + cortext_string_free (json_ptr); + REQUIRE (cortext_flush (h) == 0); + cortext_free (h); + + auto store = cortext::SQLiteStore::Create (temp_db.path ().c_str ()); + for (const char *source_id : { "capi/null-media", "capi/null-media-json" }) + { + const auto rows = store->Execute ( + "SELECT modality, COALESCE(mime, '') AS mime, " + "blob_id IS NULL AS no_blob FROM signals " + "WHERE source_id = ? ORDER BY signal_id DESC LIMIT 1", + { source_id }); + REQUIRE (rows.size () == 1); + REQUIRE (std::any_cast (rows[0].at ("modality")) == "text"); + REQUIRE (std::any_cast (rows[0].at ("mime")).empty ()); + REQUIRE (AnyToLongLong (rows[0].at ("no_blob")) == 1); + } +} + TEST_CASE ("C API handles NULL inputs correctly", "[cortext][capi][safety][aist]") {