feat: support caller media in text processing

CMakeLists.txt

@@ -806,6 +806,8 @@ if(EMSCRIPTEN AND BUILD_WASM)
     "_cortext_last_error"
     "_cortext_string_free"
     "_cortext_process_text_json"
+    "_cortext_process_text_with_media_json"
+    "_cortext_process_text_with_media_json_with_options"
     "_cortext_wasm_process_text_json"
     "_cortext_process_audio_json"
     "_cortext_process_image_json"

README.md

@@ -267,6 +267,10 @@ Core calls:
 
 - `ProcessText`, `ProcessAudio`, `ProcessImage`: process a signal (default
   retention is Natural — no forced turn edge; storage via write gate).
+- Media overloads keep the canonical input for embedding and processing.
+  Non-empty caller media replaces the stored payload and MIME type. Empty Media
+  omits the stored payload for all media overloads; legacy calls without Media
+  retain their canonical payload behavior.
 - `Retention::Durable`: force turn boundary + force write (chat convenience).
 - `Retention::Boundary`: force turn boundary only; write gate still applies.
 - `Retention::Ephemeral`: process and retrieve without storing the input.

include/cortext/capi.h

@@ -190,13 +190,14 @@ extern "C"
     int signal_filter_text_enabled;
   } cortext_config;
 
-  /// @brief Optional source media payload persisted for audio/image signals.
+  /// @brief Optional source media payload persisted for a processed signal.
   ///
-  /// The canonical process input is still passed through pcm/data arguments.
-  /// When this struct carries bytes, those bytes are stored as the durable
-  /// signal/memory payload using mimetype. Passing NULL, or a struct with
-  /// size == 0, means process without storing an audio/image payload. Media
-  /// bytes are copied during the call.
+  /// The canonical process input is still passed through text/pcm/data
+  /// arguments. When this struct carries bytes, those bytes are stored as the
+  /// durable signal/memory payload using mimetype. NULL or size == 0 omits
+  /// the canonical payload for every overload that accepts media. Legacy
+  /// process functions retain their canonical payload behavior. Media bytes
+  /// are copied during the call.
   typedef struct cortext_media
   {
     const uint8_t *data;
@@ -310,6 +311,15 @@ extern "C"
   CORTEXT_EXPORT int cortext_process_text (cortext_handle h, const char *text,
                                            const char *source_id);
 
+  /// @brief Processes text while storing caller-provided source media bytes.
+  ///
+  /// Text remains the canonical processing input. If media is non-NULL and
+  /// media->size > 0, media->data is stored using media->mimetype instead of
+  /// the canonical UTF-8 text bytes. An empty media omits the stored payload.
+  CORTEXT_EXPORT int cortext_process_text_with_media (
+      cortext_handle h, const char *text, const char *source_id,
+      const cortext_media *media);
+
   /// @brief Processes audio input through the memory system.
   /// @param h Handle to a Cortext instance.
   /// @param pcm PCM audio samples (float32, must be non-NULL).
@@ -432,6 +442,18 @@ extern "C"
       cortext_handle h, const char *text, const char *source_id,
       const cortext_process_json_options *options);
 
+  /// @brief Processes text with source media and returns Context JSON.
+  CORTEXT_EXPORT char *cortext_process_text_with_media_json (
+      cortext_handle h, const char *text, const char *source_id,
+      const cortext_media *media);
+
+  /// @brief Processes text with source media and returns configurable Context
+  /// JSON.
+  CORTEXT_EXPORT char *cortext_process_text_with_media_json_with_options (
+      cortext_handle h, const char *text, const char *source_id,
+      const cortext_media *media,
+      const cortext_process_json_options *options);
+
   /// @brief Processes audio input and returns the resulting Context as JSON.
   CORTEXT_EXPORT char *cortext_process_audio_json (cortext_handle h,
                                                    const float *pcm,

include/cortext/cortext.hpp

@@ -161,12 +161,12 @@ class CORTEXT_EXPORT Cortext
     bool signal_filter_text_enabled = false;
   };
 
-  /// @brief Optional source media payload to persist with an audio/image signal.
+  /// @brief Optional source media payload to persist with a signal.
   ///
-  /// Processing still uses the canonical audio/image input passed to
-  /// ProcessAudio/ProcessImage. When Media contains data, those bytes and this
-  /// MIME type become the durable payload instead of the canonical processing
-  /// representation. Media bytes are copied during the call.
+  /// Processing still uses the canonical input passed to ProcessText,
+  /// ProcessAudio, or ProcessImage. When Media contains data, those bytes and
+  /// this MIME type become the durable payload instead of the canonical
+  /// processing representation. Media bytes are copied during the call.
   struct Media
   {
     const std::uint8_t *data = nullptr;
@@ -237,6 +237,16 @@ class CORTEXT_EXPORT Cortext
   Context ProcessText (const std::string &text, const std::string &source_id,
                        Retention retention = Retention::Natural);
 
+  /// @brief Process text while storing caller-provided source media.
+  ///
+  /// The text remains the canonical embedding input. If media contains bytes,
+  /// those bytes are persisted as the signal/memory payload using
+  /// media.mimetype instead of the UTF-8 text bytes. If media is empty, no
+  /// text payload is persisted, matching the audio/image Media overloads.
+  Context ProcessText (const std::string &text, const std::string &source_id,
+                       const Media &media,
+                       Retention retention = Retention::Natural);
+
   /// @brief Process text input with an explicit timestamp (ms since epoch).
   /// @param text The text content to process.
   /// @param source_id Opaque source stream identifier used for provenance and

src/capi.cpp

@@ -1508,6 +1508,28 @@ extern "C"
                                     cortext::Retention::Durable); });
   }
 
+  int
+  cortext_process_text_with_media (cortext_handle h, const char *text,
+                                   const char *source_id,
+                                   const cortext_media *media)
+  {
+    auto *p = cast_handle (h);
+    if (!p || !text || !source_id)
+      {
+        set_last_error ("handle, text, and source_id must all be non-NULL");
+        return 1;
+      }
+    if (!valid_media_or_set_error (media))
+      {
+        return 1;
+      }
+
+    return invoke_status_only ([&] {
+      (void)p->ProcessText (std::string (text), std::string (source_id),
+                            media_from_c (media), cortext::Retention::Durable);
+    });
+  }
+
   int
   cortext_process_audio (cortext_handle h, const float *pcm,
                          size_t num_samples, const char *source_id)
@@ -1721,6 +1743,43 @@ extern "C"
         include_embedding_from_options (options));
   }
 
+  char *
+  cortext_process_text_with_media_json (cortext_handle h, const char *text,
+                                        const char *source_id,
+                                        const cortext_media *media)
+  {
+    const cortext_process_json_options options {
+      sizeof (cortext_process_json_options), 1, CORTEXT_RETENTION_DURABLE, 0
+    };
+    return cortext_process_text_with_media_json_with_options (
+        h, text, source_id, media, &options);
+  }
+
+  char *
+  cortext_process_text_with_media_json_with_options (
+      cortext_handle h, const char *text, const char *source_id,
+      const cortext_media *media,
+      const cortext_process_json_options *options)
+  {
+    auto *p = cast_handle (h);
+    if (!p || !text || !source_id)
+      {
+        set_last_error ("handle, text, and source_id must all be non-NULL");
+        return nullptr;
+      }
+    if (!valid_media_or_set_error (media))
+      {
+        return nullptr;
+      }
+
+    const auto retention = retention_from_options (options);
+    return invoke_json ([&] {
+      return p->ProcessText (std::string (text), std::string (source_id),
+                              media_from_c (media), retention);
+    },
+                        include_embedding_from_options (options));
+  }
+
   char *
   cortext_process_audio_json (cortext_handle h, const float *pcm,
                               size_t num_samples, const char *source_id)

src/cortext.cpp

@@ -163,7 +163,6 @@ ValidateImageInput (const std::uint8_t *data, int width, int height,
 }
 #endif
 
-#if !defined(CORTEXT_DISABLE_AUDIO) || !defined(CORTEXT_DISABLE_IMAGE)
 void
 ValidateMedia (const Cortext::Media &media)
 {
@@ -196,7 +195,6 @@ ApplyStoredMedia (Signal &signal, const Cortext::Media &media)
                                                media.data + media.size);
   signal.mimetype = media.mimetype;
 }
-#endif
 
 #if defined(CORTEXT_DISABLE_AUDIO)
 [[noreturn]] void
@@ -438,7 +436,11 @@ PayloadMatchesMemorySurface (const std::string &memory_modality,
     {
       return false;
     }
-  if (memory_modality == "text" || memory_mime == "text/plain"
+  // Text modality can carry caller-provided binary payloads (for example a
+  // token KV cache). Only canonical text/plain payloads require text-content
+  // validation; a custom MIME type declares an opaque payload surface.
+  if (memory_mime == "text/plain"
+      || (memory_mime.empty () && memory_modality == "text")
       || signal_mime == "text/plain")
     {
       return IsLikelyTextPayload (payload);
@@ -2045,6 +2047,61 @@ Cortext::ProcessText (const std::string &text, const std::string &source_id,
   return result;
 }
 
+Cortext::Context
+Cortext::ProcessText (const std::string &text, const std::string &source_id,
+                      const Media &media, Retention retention)
+{
+  if (!impl_)
+    {
+      throw std::runtime_error ("Cortext not initialized");
+    }
+  ValidateMedia (media);
+  telemetry::ScopedSpan span ("cortext.api.process_text");
+  const auto total_start = std::chrono::steady_clock::now ();
+  const auto filter_decision = impl_->signal_filter.EvaluateText (text);
+  if (filter_decision.evaluated && !filter_decision.accepted)
+    {
+      span.SetStatusOk ();
+      return MakeFilteredContext (filter_decision, total_start);
+    }
+  std::vector<float> v;
+  telemetry::ScopedSpan encode_span ("cortext.encode");
+  impl_->encoder->EncodeText (text, v);
+  encode_span.SetStatusOk ();
+  const auto encode_end = std::chrono::steady_clock::now ();
+
+  // Build signal with optional caller-provided source payload. Text remains
+  // the canonical embedding input.
+  const auto process_start = std::chrono::steady_clock::now ();
+  cortext::Signal s;
+  s.embedding = ToEigen (RetrievalEmbeddingView (v));
+  s.soft_anchor_embedding = SoftAnchorEmbeddingView (v);
+  s.timestamp = impl_->NowMillis ();
+  s.source_id = source_id;
+  s.retention = retention;
+  s.modality = "text";
+  ApplyStoredMedia (s, media);
+
+  auto out = impl_->processor->Process (s);
+  const auto process_end = std::chrono::steady_clock::now ();
+  span.SetAttribute ("cortext.candidate_memory_count",
+                     static_cast<std::int64_t> (out.candidate_memory_ids.size ()));
+  span.SetAttribute ("cortext.used_memory_count",
+                     static_cast<std::int64_t> (out.used_memory_ids.size ()));
+  span.SetStatusOk ();
+  telemetry::ScopedSpan hydrate_span ("cortext.hydrate");
+  const auto hydrate_start = std::chrono::steady_clock::now ();
+  Cortext::Context result = impl_->HydrateContext (out, true, &s.embedding);
+  const auto hydrate_end = std::chrono::steady_clock::now ();
+  hydrate_span.SetStatusOk ();
+  result.encode_ms = ToMillis (encode_end - total_start);
+  result.process_ms = ToMillis (process_end - process_start);
+  result.hydrate_ms = ToMillis (hydrate_end - hydrate_start);
+  result.total_ms = ToMillis (hydrate_end - total_start);
+  ApplySignalFilterDecision (result.output, filter_decision);
+  return result;
+}
+
 Cortext::Context
 Cortext::ProcessTextAt (const std::string &text, const std::string &source_id,
                         std::uint64_t timestamp, Retention retention)