diff --git a/lib/bumblebee/layers.ex b/lib/bumblebee/layers.ex
index 27d81990..3bd830ae 100644
--- a/lib/bumblebee/layers.ex
+++ b/lib/bumblebee/layers.ex
@@ -1239,18 +1239,206 @@ defmodule Bumblebee.Layers do
   def rotary_embedding(query, key, position_ids, attention_mask, size, opts \\ []) do
     opts = Keyword.validate!(opts, [:name, :scaling_strategy, max_positions: 2048, base: 10_000])
 
+    scaling_strategy =
+      normalize_rotary_embedding_scaling_strategy(opts[:scaling_strategy], size, opts)
+
     output =
       Axon.layer(
         &apply_rotary_embedding/5,
         [query, key, position_ids, Axon.optional(attention_mask)],
-        [size: size] ++ opts
+        [size: size, scaling_strategy: scaling_strategy] ++
+          Keyword.drop(opts, [:scaling_strategy])
       )
 
     unwrap_tuple(output, 2)
   end
 
+  defp normalize_rotary_embedding_scaling_strategy(nil, _size, _opts), do: nil
+
+  defp normalize_rotary_embedding_scaling_strategy(
+         %{type: :yarn} = strategy,
+         size,
+         opts
+       ) do
+    required_keys = [
+      :factor,
+      :original_max_positions,
+      :beta_fast,
+      :beta_slow
+    ]
+
+    case Enum.find(required_keys, &(not Map.has_key?(strategy, &1))) do
+      nil ->
+        validate_factor!(strategy.factor, :yarn)
+
+        strategy =
+          strategy
+          |> Map.put_new_lazy(:attention_factor, fn -> 0.1 * :math.log(strategy.factor) + 1.0 end)
+          |> Map.put_new(:truncate, true)
+
+        validate_yarn_strategy!(strategy)
+
+        low =
+          yarn_correction_dim(
+            strategy.beta_fast,
+            size,
+            opts[:base],
+            strategy.original_max_positions
+          )
+
+        high =
+          yarn_correction_dim(
+            strategy.beta_slow,
+            size,
+            opts[:base],
+            strategy.original_max_positions
+          )
+
+        {low, high} =
+          if strategy.truncate do
+            {:math.floor(low), :math.ceil(high)}
+          else
+            {low, high}
+          end
+
+        low = max(low, 0)
+        high = min(high, size - 1)
+        high = if low == high, do: high + 0.001, else: high
+
+        Map.merge(strategy, %{
+          correction_low: low,
+          correction_high: high
+        })
+
+      key ->
+        raise ArgumentError, "YaRN rotary embedding scaling strategy is missing #{inspect(key)}"
+    end
+  end
+
+  defp normalize_rotary_embedding_scaling_strategy(
+         %{type: :longrope, short_factor: short_factor, long_factor: long_factor} = strategy,
+         size,
+         _opts
+       ) do
+    expected_size = div(size, 2)
+
+    if longrope_factors?(short_factor, expected_size) and
+         longrope_factors?(long_factor, expected_size) do
+      strategy
+    else
+      raise ArgumentError,
+            "LongRoPE factors must each have #{expected_size} entries for a rotary size of #{size}"
+    end
+  end
+
+  defp normalize_rotary_embedding_scaling_strategy(
+         %{type: :dynamic, factor: factor} = strategy,
+         size,
+         _opts
+       ) do
+    validate_factor!(factor, :dynamic)
+
+    if size <= 2 do
+      raise ArgumentError, "dynamic RoPE requires a rotary size greater than 2"
+    end
+
+    strategy
+  end
+
+  defp normalize_rotary_embedding_scaling_strategy(
+         %{type: :linear, factor: factor} = strategy,
+         _size,
+         _opts
+       ) do
+    validate_factor!(factor, :linear)
+    strategy
+  end
+
+  defp normalize_rotary_embedding_scaling_strategy(%{type: type}, _size, _opts)
+       when type in [:linear, :dynamic] do
+    raise ArgumentError, "#{type} RoPE requires :factor to be a number >= 1"
+  end
+
+  defp normalize_rotary_embedding_scaling_strategy(
+         %{
+           type: :llama3,
+           factor: factor,
+           low_frequency_factor: low,
+           high_frequency_factor: high,
+           original_max_positions: original_max_positions
+         } = strategy,
+         _size,
+         _opts
+       ) do
+    validate_factor!(factor, :llama3)
+    validate_positive_number!(low, :low_frequency_factor, :llama3)
+    validate_positive_number!(high, :high_frequency_factor, :llama3)
+    validate_positive_integer!(original_max_positions, :original_max_positions, :llama3)
+
+    if high <= low do
+      raise ArgumentError, "llama3 RoPE requires :high_frequency_factor > :low_frequency_factor"
+    end
+
+    strategy
+  end
+
+  defp normalize_rotary_embedding_scaling_strategy(strategy, _size, _opts) do
+    raise ArgumentError, "unsupported rotary embedding scaling strategy: #{inspect(strategy)}"
+  end
+
+  defp yarn_correction_dim(num_rotations, size, base, original_max_positions) do
+    size * :math.log(original_max_positions / (num_rotations * 2 * :math.pi())) /
+      (2 * :math.log(base))
+  end
+
+  defp validate_yarn_strategy!(strategy) do
+    validate_factor!(strategy.factor, :yarn)
+    validate_positive_integer!(strategy.original_max_positions, :original_max_positions, :yarn)
+    validate_positive_number!(strategy.beta_fast, :beta_fast, :yarn)
+    validate_positive_number!(strategy.beta_slow, :beta_slow, :yarn)
+    validate_positive_number!(strategy.attention_factor, :attention_factor, :yarn)
+
+    unless is_boolean(strategy.truncate) do
+      raise ArgumentError, "YaRN RoPE requires :truncate to be a boolean"
+    end
+
+    if strategy.beta_fast < strategy.beta_slow do
+      raise ArgumentError, "YaRN RoPE requires :beta_fast >= :beta_slow"
+    end
+  end
+
+  defp longrope_factors?(factors, expected_size) do
+    is_list(factors) and length(factors) == expected_size and
+      Enum.all?(factors, &(is_number(&1) and &1 > 0))
+  end
+
+  defp validate_factor!(factor, type) do
+    if is_number(factor) and factor >= 1.0 do
+      :ok
+    else
+      raise ArgumentError, "#{type} RoPE requires :factor to be a number >= 1"
+    end
+  end
+
+  defp validate_positive_number!(value, key, type) do
+    if is_number(value) and value > 0 do
+      :ok
+    else
+      raise ArgumentError, "#{type} RoPE requires #{inspect(key)} to be a positive number"
+    end
+  end
+
+  defp validate_positive_integer!(value, key, type) do
+    if is_integer(value) and value > 0 do
+      :ok
+    else
+      raise ArgumentError, "#{type} RoPE requires #{inspect(key)} to be a positive integer"
+    end
+  end
+
   deftransformp create_sinusoidal_positions(
                   sequence_length,
+                  context_length,
                   max_positions,
                   size,
                   base,
@@ -1266,15 +1454,48 @@ defmodule Bumblebee.Layers do
         position = Nx.divide(position, factor)
         positions_cos_sin(position, inv_frequency)
 
-      %{type: :dynamic, factor: factor} when sequence_length > max_positions ->
-        base =
-          base
-          |> Nx.multiply(factor * sequence_length / max_positions - (factor - 1))
-          |> Nx.pow(size / (size - 2))
+      %{type: :dynamic, factor: factor} ->
+        context_length = Nx.max(context_length, max_positions)
+
+        scale =
+          context_length
+          |> Nx.multiply(factor)
+          |> Nx.divide(max_positions)
+          |> Nx.subtract(factor - 1)
+
+        base = Nx.multiply(base, Nx.pow(scale, size / (size - 2)))
 
         inv_frequency = inv_frequency(base, range)
         positions_cos_sin(position, inv_frequency)
 
+      %{
+        type: :yarn,
+        factor: factor,
+        attention_factor: attention_factor,
+        correction_low: low,
+        correction_high: high
+      } ->
+        inv_frequency_extrapolation = inv_frequency(base, range)
+        inv_frequency_interpolation = Nx.divide(inv_frequency_extrapolation, factor)
+
+        ramp =
+          Nx.iota({div(size, 2)})
+          |> Nx.subtract(low)
+          |> Nx.divide(high - low)
+          |> Nx.clip(0.0, 1.0)
+
+        inv_frequency_extrapolation_factor = Nx.subtract(1.0, ramp)
+
+        inv_frequency =
+          Nx.multiply(
+            inv_frequency_interpolation,
+            Nx.subtract(1.0, inv_frequency_extrapolation_factor)
+          )
+          |> Nx.add(Nx.multiply(inv_frequency_extrapolation, inv_frequency_extrapolation_factor))
+
+        {cos, sin} = positions_cos_sin(position, inv_frequency)
+        {Nx.multiply(cos, attention_factor), Nx.multiply(sin, attention_factor)}
+
       %{
         type: :longrope,
         short_factor: short_factor,
@@ -1394,17 +1615,19 @@ defmodule Bumblebee.Layers do
         _other -> Nx.axis_size(attention_mask, 1)
       end
 
+    position_ids = Nx.as_type(position_ids, :s64)
+    context_length = Nx.add(Nx.reduce_max(position_ids), 1)
+
     {cos, sin} =
       create_sinusoidal_positions(
         sequence_length,
+        context_length,
         opts[:max_positions],
         opts[:size],
         opts[:base],
         opts[:scaling_strategy]
       )
 
-    position_ids = Nx.as_type(position_ids, :s64)
-
     cos = cos |> Nx.take(position_ids) |> Nx.new_axis(2) |> Nx.as_type(Nx.type(query))
     sin = sin |> Nx.take(position_ids) |> Nx.new_axis(2) |> Nx.as_type(Nx.type(query))
 
diff --git a/lib/bumblebee/layers/decoder.ex b/lib/bumblebee/layers/decoder.ex
index fc56b7f2..36cdc50d 100644
--- a/lib/bumblebee/layers/decoder.ex
+++ b/lib/bumblebee/layers/decoder.ex
@@ -5,6 +5,24 @@ defmodule Bumblebee.Layers.Decoder do
 
   alias Bumblebee.Layers
 
+  @doc """
+  Rejects cached inference with bidirectional self-attention, since adding
+  tokens changes the hidden states of previously processed tokens.
+  """
+  def validate_attention_cache(cache, false), do: cache
+
+  def validate_attention_cache(cache, true) do
+    Axon.layer(
+      fn cache, _opts ->
+        case cache do
+          %Axon.None{} -> cache
+          _ -> raise ArgumentError, "decoding cache is not supported with bidirectional attention"
+        end
+      end,
+      [Axon.optional(cache)]
+    )
+  end
+
   defmodule Cache do
     @moduledoc false
 
diff --git a/lib/bumblebee/shared.ex b/lib/bumblebee/shared.ex
index e4ff6c55..cc9e33c8 100644
--- a/lib/bumblebee/shared.ex
+++ b/lib/bumblebee/shared.ex
@@ -38,10 +38,15 @@ defmodule Bumblebee.Shared do
 
           * `%{type: :dynamic, factor: number()}`
 
+          * `%{type: :yarn, factor: number(), original_max_positions: pos_integer(), beta_fast: number(), beta_slow: number()}`
+
           * `%{type: :llama3, factor: number(), low_frequency_factor: number(), high_frequency_factor: number(), original_max_positions: pos_integer()}`
 
           * `%{type: :longrope, short_factor: list(number()), long_factor: list(number()), original_max_positions: pos_integer()}`
 
+        YaRN also accepts `:attention_factor` (defaults to `0.1 * log(factor) + 1.0`)
+        and `:truncate` (defaults to `true`).
+
         For more details see https://www.reddit.com/r/LocalLLaMA/comments/14mrgpr/dynamically_scaled_rope_further_increases
         """
       ]
@@ -141,6 +146,47 @@ defmodule Bumblebee.Shared do
     end
   end
 
+  @doc """
+  Converts the causal-attention setting from Hugging Face config data.
+
+  Hugging Face models use `"is_causal"`, while some embedding checkpoints
+  serialize `"use_bidirectional_attention"`. `"is_causal"` takes precedence
+  when both are present.
+  """
+  @spec bidirectional_attention_options_from_transformers(map()) :: keyword()
+  def bidirectional_attention_options_from_transformers(data) do
+    case Map.fetch(data, "is_causal") do
+      {:ok, is_causal} when is_boolean(is_causal) ->
+        [use_bidirectional_attention: not is_causal]
+
+      {:ok, _value} ->
+        raise "conversion failed, expected \"is_causal\" to be a boolean"
+
+      :error ->
+        case Map.fetch(data, "use_bidirectional_attention") do
+          {:ok, use_bidirectional_attention} when is_boolean(use_bidirectional_attention) ->
+            [use_bidirectional_attention: use_bidirectional_attention]
+
+          {:ok, _value} ->
+            raise "conversion failed, expected \"use_bidirectional_attention\" to be a boolean"
+
+          :error ->
+            []
+        end
+    end
+  end
+
+  @doc false
+  @spec validate_bidirectional_attention(Bumblebee.ModelSpec.t()) :: Bumblebee.ModelSpec.t()
+  def validate_bidirectional_attention(%{use_bidirectional_attention: value} = spec)
+      when is_boolean(value),
+      do: spec
+
+  def validate_bidirectional_attention(%{use_bidirectional_attention: value}) do
+    raise ArgumentError,
+          ":use_bidirectional_attention must be a boolean, got: #{inspect(value)}"
+  end
+
   @doc """
   Converts rotary embedding options from Hugging Face config data.
 
@@ -168,8 +214,14 @@ defmodule Bumblebee.Shared do
   end
 
   defp rotary_embedding_options(params, data) do
+    base = params["rope_theta"]
+
+    if base != nil and (not is_number(base) or base <= 0) do
+      raise "conversion failed, \"rope_theta\" must be a positive number"
+    end
+
     [
-      rotary_embedding_base: params["rope_theta"],
+      rotary_embedding_base: base,
       rotary_embedding_percentage: params["partial_rotary_factor"],
       rotary_embedding_scaling_strategy: rotary_embedding_scaling_strategy(params, data)
     ]
@@ -182,40 +234,176 @@ defmodule Bumblebee.Shared do
       data["original_max_position_embeddings"] || params["original_max_position_embeddings"] ||
         data["max_position_embeddings"]
 
-    case {params["rope_type"] || params["type"] || "default", params} do
+    type = rope_type!(params)
+
+    if type == "default" and scaling_fields?(params) do
+      if Map.has_key?(params, "type") or Map.has_key?(params, "rope_type") do
+        raise "conversion failed, default rotary embedding does not accept scaling parameters"
+      else
+        raise "conversion failed, rotary embedding scaling parameters require \"type\" or \"rope_type\""
+      end
+    end
+
+    case {type, params} do
       {"default", _params} ->
         nil
 
       {"linear", %{"factor" => factor}} ->
-        %{type: :linear, factor: factor}
+        %{type: :linear, factor: validate_factor!(factor, "linear")}
 
       {"dynamic", %{"factor" => factor}} ->
-        %{type: :dynamic, factor: factor}
+        %{type: :dynamic, factor: validate_factor!(factor, type)}
 
       {"llama3", %{"factor" => factor, "low_freq_factor" => low, "high_freq_factor" => high}} ->
-        %{
-          type: :llama3,
-          factor: factor,
-          low_frequency_factor: low,
-          high_frequency_factor: high,
-          original_max_positions: original_max_positions
-        }
+        llama3_scaling_strategy(factor, low, high, original_max_positions)
 
       # Old Phi-3 checkpoints use "su" and "yarn" for LongRoPE
-      {type, %{"short_factor" => short_factor, "long_factor" => long_factor}}
+      {type, %{"short_factor" => _short_factor, "long_factor" => _long_factor} = params}
       when type in ["longrope", "su", "yarn"] ->
-        %{
-          type: :longrope,
-          short_factor: short_factor,
-          long_factor: long_factor,
-          original_max_positions: original_max_positions
-        }
+        longrope_scaling_strategy(params, original_max_positions)
+
+      {"yarn", %{"factor" => _factor}} = config ->
+        yarn_scaling_strategy(config, original_max_positions)
 
       _other ->
         raise "conversion failed, unsupported rotary embedding parameters: #{inspect(params)}"
     end
   end
 
+  defp yarn_scaling_strategy({"yarn", params}, original_max_positions) do
+    factor = validate_factor!(params["factor"], "yarn")
+    original_max_positions = validate_original_max_positions!(original_max_positions, "yarn")
+    beta_fast = validate_positive_number!(params["beta_fast"] || 32.0, "beta_fast", "yarn")
+    beta_slow = validate_positive_number!(params["beta_slow"] || 1.0, "beta_slow", "yarn")
+
+    unless is_boolean(Map.get(params, "truncate", true)) do
+      raise "conversion failed, yarn requires \"truncate\" to be a boolean"
+    end
+
+    if beta_fast < beta_slow do
+      raise "conversion failed, YaRN requires \"beta_fast\" >= \"beta_slow\""
+    end
+
+    %{
+      type: :yarn,
+      factor: factor,
+      original_max_positions: original_max_positions,
+      beta_fast: beta_fast,
+      beta_slow: beta_slow,
+      attention_factor: yarn_attention_factor(params, factor),
+      truncate: Map.get(params, "truncate", true)
+    }
+  end
+
+  defp llama3_scaling_strategy(factor, low, high, original_max_positions) do
+    factor = validate_factor!(factor, "llama3")
+    low = validate_positive_number!(low, "low_freq_factor", "llama3")
+    high = validate_positive_number!(high, "high_freq_factor", "llama3")
+
+    if high <= low do
+      raise "conversion failed, llama3 requires \"high_freq_factor\" > \"low_freq_factor\""
+    end
+
+    %{
+      type: :llama3,
+      factor: factor,
+      low_frequency_factor: low,
+      high_frequency_factor: high,
+      original_max_positions: validate_original_max_positions!(original_max_positions, "llama3")
+    }
+  end
+
+  defp rope_type!(%{"rope_type" => rope_type, "type" => type}) when rope_type != type do
+    raise "conversion failed, conflicting \"rope_type\" and \"type\" values"
+  end
+
+  defp rope_type!(%{"rope_type" => rope_type}), do: rope_type
+  defp rope_type!(%{"type" => type}), do: type
+  defp rope_type!(_params), do: "default"
+
+  defp scaling_fields?(params) do
+    Enum.any?(
+      [
+        "factor",
+        "short_factor",
+        "long_factor",
+        "low_freq_factor",
+        "high_freq_factor",
+        "beta_fast",
+        "beta_slow",
+        "attention_factor",
+        "mscale",
+        "mscale_all_dim",
+        "truncate"
+      ],
+      &Map.has_key?(params, &1)
+    )
+  end
+
+  defp longrope_scaling_strategy(params, original_max_positions) do
+    %{
+      type: :longrope,
+      short_factor: validate_number_list!(params["short_factor"], "short_factor", "longrope"),
+      long_factor: validate_number_list!(params["long_factor"], "long_factor", "longrope"),
+      original_max_positions: validate_original_max_positions!(original_max_positions, "longrope")
+    }
+  end
+
+  defp yarn_attention_factor(params, factor) do
+    case params["attention_factor"] do
+      nil ->
+        case {params["mscale"], params["mscale_all_dim"]} do
+          {nil, nil} ->
+            yarn_mscale(factor)
+
+          {mscale, mscale_all_dim} when is_number(mscale) and is_number(mscale_all_dim) ->
+            yarn_mscale(factor, mscale) / yarn_mscale(factor, mscale_all_dim)
+
+          _other ->
+            raise "conversion failed, yarn requires numeric \"mscale\" and \"mscale_all_dim\" together"
+        end
+
+      attention_factor ->
+        validate_positive_number!(attention_factor, "attention_factor", "yarn")
+    end
+  end
+
+  defp yarn_mscale(factor, mscale \\ 1.0)
+  defp yarn_mscale(factor, _mscale) when factor <= 1.0, do: 1.0
+  defp yarn_mscale(factor, mscale), do: 0.1 * mscale * :math.log(factor) + 1.0
+
+  defp validate_factor!(factor, type) do
+    if is_number(factor) and factor >= 1.0 do
+      factor
+    else
+      raise "conversion failed, #{type} requires a numeric \"factor\" >= 1"
+    end
+  end
+
+  defp validate_original_max_positions!(value, type) do
+    if is_integer(value) and value > 0 do
+      value
+    else
+      raise "conversion failed, #{type} requires a positive integer \"original_max_position_embeddings\""
+    end
+  end
+
+  defp validate_positive_number!(value, key, type) do
+    if is_number(value) and value > 0 do
+      value
+    else
+      raise "conversion failed, #{type} requires a positive numeric #{inspect(key)}"
+    end
+  end
+
+  defp validate_number_list!(value, key, type) do
+    if is_list(value) and value != [] and Enum.all?(value, &(is_number(&1) and &1 > 0)) do
+      value
+    else
+      raise "conversion failed, #{type} requires a non-empty list of positive numbers for #{inspect(key)}"
+    end
+  end
+
   @doc """
   Merges the given list of attributes into a configuration struct.
 
diff --git a/lib/bumblebee/text/gemma3_text.ex b/lib/bumblebee/text/gemma3_text.ex
index 19301315..b379c8e2 100644
--- a/lib/bumblebee/text/gemma3_text.ex
+++ b/lib/bumblebee/text/gemma3_text.ex
@@ -80,7 +80,7 @@ defmodule Bumblebee.Text.Gemma3Text do
       attention_window_size: [
         default: 4096,
         doc:
-          "the number of tokens each token can attend to in the sliding attention window, including itself. Only used for `:sliding_attention` layers"
+          "causal window width including the token itself. Bidirectional layers use half this width as the distance on either side, matching Transformers. Only used for `:sliding_attention` layers"
       ],
       layer_types: [
         default: nil,
@@ -95,7 +95,12 @@ defmodule Bumblebee.Text.Gemma3Text do
       ],
       use_bidirectional_attention: [
         default: false,
-        doc: "whether to use bidirectional attention instead of causal attention"
+        doc: """
+        whether to use bidirectional attention. Loaded from checkpoint `is_causal`
+        (inverted), or `use_bidirectional_attention`. The default is causal, including
+        the `:base` architecture. Bidirectional inference requires the full sequence
+        on each pass and does not support decoding cache or generation
+        """
       ]
     ] ++
       Shared.common_options([:rotary_embedding_scaling_strategy, :num_labels, :id_to_label]) ++
@@ -194,6 +199,7 @@ defmodule Bumblebee.Text.Gemma3Text do
     spec
     |> Shared.put_config_attrs(opts)
     |> Shared.validate_label_options()
+    |> Shared.validate_bidirectional_attention()
   end
 
   @impl true
@@ -204,6 +210,11 @@ defmodule Bumblebee.Text.Gemma3Text do
   end
 
   @impl true
+  def init_cache(%{use_bidirectional_attention: true}, _batch_size, _max_length, _inputs) do
+    raise ArgumentError,
+          "decoding cache and generation are not supported with bidirectional attention"
+  end
+
   def init_cache(spec, batch_size, max_length, _inputs) do
     Layers.Decoder.init_cache(batch_size, max_length,
       hidden_size: spec.hidden_size,
@@ -312,7 +323,10 @@ defmodule Bumblebee.Text.Gemma3Text do
         position_ids,
         inputs["attention_mask"],
         inputs["attention_head_mask"],
-        inputs["cache"],
+        Layers.Decoder.validate_attention_cache(
+          inputs["cache"],
+          spec.use_bidirectional_attention
+        ),
         spec,
         name: "decoder"
       )
@@ -395,8 +409,12 @@ defmodule Bumblebee.Text.Gemma3Text do
               {nil, spec.rotary_embedding_base, spec.rotary_embedding_scaling_strategy}
 
             :sliding_attention ->
-              {{spec.attention_window_size - 1, spec.attention_window_size - 1},
-               spec.rotary_embedding_base_local, nil}
+              radius =
+                if spec.use_bidirectional_attention,
+                  do: div(spec.attention_window_size, 2),
+                  else: spec.attention_window_size - 1
+
+              {{radius, radius}, spec.rotary_embedding_base_local, nil}
           end
 
         shortcut = hidden_state
@@ -539,9 +557,14 @@ defmodule Bumblebee.Text.Gemma3Text do
                  "full_attention" => :full_attention
                })
              )},
-          tie_word_embeddings: {"tie_word_embeddings", boolean()},
-          use_bidirectional_attention: {"use_bidirectional_attention", boolean()}
+          tie_word_embeddings: {"tie_word_embeddings", boolean()}
         ) ++
+          Shared.bidirectional_attention_options_from_transformers(
+            if(Map.has_key?(data, "use_bidirectional_attention"),
+              do: Map.delete(data, "is_causal"),
+              else: data
+            )
+          ) ++
           Shared.rotary_embedding_options_from_transformers(data) ++
           Shared.common_options_from_transformers(data, spec)
 
diff --git a/lib/bumblebee/text/llama.ex b/lib/bumblebee/text/llama.ex
index 6a6d28f8..afa063a1 100644
--- a/lib/bumblebee/text/llama.ex
+++ b/lib/bumblebee/text/llama.ex
@@ -3,6 +3,15 @@ defmodule Bumblebee.Text.Llama do
 
   options =
     [
+      use_bidirectional_attention: [
+        default: false,
+        doc: """
+        whether to use bidirectional attention. Loaded from checkpoint `is_causal`
+        (inverted), or `use_bidirectional_attention`. The default is causal, including
+        the `:base` architecture. Bidirectional inference requires the full sequence
+        on each pass and does not support decoding cache or generation
+        """
+      ],
       vocab_size: [
         default: 32000,
         doc: """
@@ -154,6 +163,7 @@ defmodule Bumblebee.Text.Llama do
     spec
     |> Shared.put_config_attrs(opts)
     |> Shared.validate_label_options()
+    |> Shared.validate_bidirectional_attention()
   end
 
   @impl true
@@ -164,6 +174,11 @@ defmodule Bumblebee.Text.Llama do
   end
 
   @impl true
+  def init_cache(%{use_bidirectional_attention: true}, _batch_size, _max_length, _inputs) do
+    raise ArgumentError,
+          "decoding cache and generation are not supported with bidirectional attention"
+  end
+
   def init_cache(spec, batch_size, max_length, _inputs) do
     Layers.Decoder.init_cache(batch_size, max_length,
       hidden_size: spec.hidden_size,
@@ -272,7 +287,10 @@ defmodule Bumblebee.Text.Llama do
         position_ids,
         inputs["attention_mask"],
         inputs["attention_head_mask"],
-        inputs["cache"],
+        Layers.Decoder.validate_attention_cache(
+          inputs["cache"],
+          spec.use_bidirectional_attention
+        ),
         spec,
         name: "decoder"
       )
@@ -338,7 +356,7 @@ defmodule Bumblebee.Text.Llama do
             num_key_value_heads: spec.num_key_value_heads,
             hidden_size: spec.hidden_size,
             attention_head_size: spec.attention_head_size,
-            causal: true,
+            causal: not spec.use_bidirectional_attention,
             rotary_embedding: [
               position_ids: position_ids,
               max_positions: spec.max_positions,
@@ -409,6 +427,7 @@ defmodule Bumblebee.Text.Llama do
           layer_norm_epsilon: {"rms_norm_eps", number()},
           tie_word_embeddings: {"tie_word_embeddings", boolean()}
         ) ++
+          Shared.bidirectional_attention_options_from_transformers(data) ++
           Shared.rotary_embedding_options_from_transformers(data) ++
           Shared.common_options_from_transformers(data, spec)
 
diff --git a/lib/bumblebee/text/mistral.ex b/lib/bumblebee/text/mistral.ex
index 51324cf2..be08b313 100644
--- a/lib/bumblebee/text/mistral.ex
+++ b/lib/bumblebee/text/mistral.ex
@@ -3,6 +3,15 @@ defmodule Bumblebee.Text.Mistral do
 
   options =
     [
+      use_bidirectional_attention: [
+        default: false,
+        doc: """
+        whether to use bidirectional attention. Loaded from checkpoint `is_causal`
+        (inverted), or `use_bidirectional_attention`. The default is causal, including
+        the `:base` architecture. Bidirectional inference requires the full sequence
+        on each pass and does not support decoding cache or generation
+        """
+      ],
       vocab_size: [
         default: 32000,
         doc: """
@@ -46,7 +55,7 @@ defmodule Bumblebee.Text.Mistral do
       attention_window_size: [
         default: 4096,
         doc:
-          "the number of tokens each token can attend to in the sliding attention window, including itself"
+          "causal window width including the token itself; in bidirectional mode, the maximum distance on either side"
       ],
       activation: [
         default: :silu,
@@ -153,6 +162,7 @@ defmodule Bumblebee.Text.Mistral do
     spec
     |> Shared.put_config_attrs(opts)
     |> Shared.validate_label_options()
+    |> Shared.validate_bidirectional_attention()
   end
 
   @impl true
@@ -163,6 +173,11 @@ defmodule Bumblebee.Text.Mistral do
   end
 
   @impl true
+  def init_cache(%{use_bidirectional_attention: true}, _batch_size, _max_length, _inputs) do
+    raise ArgumentError,
+          "decoding cache and generation are not supported with bidirectional attention"
+  end
+
   def init_cache(spec, batch_size, max_length, _inputs) do
     Layers.Decoder.init_cache(batch_size, max_length,
       hidden_size: spec.hidden_size,
@@ -270,7 +285,10 @@ defmodule Bumblebee.Text.Mistral do
         position_ids,
         inputs["attention_mask"],
         inputs["attention_head_mask"],
-        inputs["cache"],
+        Layers.Decoder.validate_attention_cache(
+          inputs["cache"],
+          spec.use_bidirectional_attention
+        ),
         spec,
         name: "decoder"
       )
@@ -337,10 +355,11 @@ defmodule Bumblebee.Text.Mistral do
             num_heads: spec.num_attention_heads,
             num_key_value_heads: spec.num_key_value_heads,
             hidden_size: spec.hidden_size,
-            causal: true,
+            causal: not spec.use_bidirectional_attention,
             attention_window_size:
               spec.attention_window_size &&
-                {spec.attention_window_size - 1, spec.attention_window_size - 1},
+                {spec.attention_window_size - if(spec.use_bidirectional_attention, do: 0, else: 1),
+                 spec.attention_window_size},
             rotary_embedding: [
               position_ids: position_ids,
               max_positions: spec.max_positions,
@@ -409,6 +428,7 @@ defmodule Bumblebee.Text.Mistral do
           initializer_scale: {"initializer_range", number()},
           layer_norm_epsilon: {"rms_norm_eps", number()}
         ) ++
+          Shared.bidirectional_attention_options_from_transformers(data) ++
           Shared.rotary_embedding_options_from_transformers(data) ++
           Shared.common_options_from_transformers(data, spec)
 
diff --git a/lib/bumblebee/text/qwen3.ex b/lib/bumblebee/text/qwen3.ex
index a959e9d0..63d907ea 100644
--- a/lib/bumblebee/text/qwen3.ex
+++ b/lib/bumblebee/text/qwen3.ex
@@ -3,6 +3,15 @@ defmodule Bumblebee.Text.Qwen3 do
 
   options =
     [
+      use_bidirectional_attention: [
+        default: false,
+        doc: """
+        whether to use bidirectional attention. Loaded from checkpoint `is_causal`
+        (inverted), or `use_bidirectional_attention`. The default is causal, including
+        the `:base` architecture. Bidirectional inference requires the full sequence
+        on each pass and does not support decoding cache or generation
+        """
+      ],
       vocab_size: [
         default: 151_936,
         doc: """
@@ -44,6 +53,22 @@ defmodule Bumblebee.Text.Qwen3 do
         default: 8,
         doc: "the number of key value heads for each attention layer in the model"
       ],
+      attention_window_size: [
+        default: nil,
+        doc: """
+        sliding window width, or nil for full attention. Causal layers attend to
+        this many tokens including themselves; bidirectional layers attend to
+        tokens within this distance on either side
+        """
+      ],
+      max_window_layers: [
+        default: 28,
+        doc: "number of initial full-attention layers when :layer_types is not specified"
+      ],
+      layer_types: [
+        default: nil,
+        doc: "per-layer :full_attention or :sliding_attention overrides"
+      ],
       activation: [
         default: :silu,
         doc: "the activation function"
@@ -157,6 +182,33 @@ defmodule Bumblebee.Text.Qwen3 do
     spec
     |> Shared.put_config_attrs(opts)
     |> Shared.validate_label_options()
+    |> Shared.validate_bidirectional_attention()
+    |> validate_attention_options()
+  end
+
+  defp validate_attention_options(spec) do
+    unless is_nil(spec.attention_window_size) or
+             (is_integer(spec.attention_window_size) and spec.attention_window_size > 0) do
+      raise ArgumentError, ":attention_window_size must be nil or a positive integer"
+    end
+
+    unless is_integer(spec.max_window_layers) and spec.max_window_layers >= 0 do
+      raise ArgumentError, ":max_window_layers must be a non-negative integer"
+    end
+
+    if spec.layer_types do
+      unless is_list(spec.layer_types) and length(spec.layer_types) == spec.num_blocks and
+               Enum.all?(spec.layer_types, &(&1 in [:full_attention, :sliding_attention])) do
+        raise ArgumentError,
+              ":layer_types must specify :full_attention or :sliding_attention for each block"
+      end
+
+      if :sliding_attention in spec.layer_types and is_nil(spec.attention_window_size) do
+        raise ArgumentError, "sliding attention layers require :attention_window_size"
+      end
+    end
+
+    spec
   end
 
   @impl true
@@ -167,6 +219,11 @@ defmodule Bumblebee.Text.Qwen3 do
   end
 
   @impl true
+  def init_cache(%{use_bidirectional_attention: true}, _batch_size, _max_length, _inputs) do
+    raise ArgumentError,
+          "decoding cache and generation are not supported with bidirectional attention"
+  end
+
   def init_cache(spec, batch_size, max_length, _inputs) do
     Layers.Decoder.init_cache(batch_size, max_length,
       hidden_size: spec.hidden_size,
@@ -275,7 +332,10 @@ defmodule Bumblebee.Text.Qwen3 do
         position_ids,
         inputs["attention_mask"],
         inputs["attention_head_mask"],
-        inputs["cache"],
+        Layers.Decoder.validate_attention_cache(
+          inputs["cache"],
+          spec.use_bidirectional_attention
+        ),
         spec,
         name: "decoder"
       )
@@ -334,6 +394,25 @@ defmodule Bumblebee.Text.Qwen3 do
       fn hidden_state, block ->
         name = block.name
 
+        layer_type =
+          if spec.layer_types do
+            Enum.fetch!(spec.layer_types, block.index)
+          else
+            if spec.attention_window_size && block.index >= spec.max_window_layers,
+              do: :sliding_attention,
+              else: :full_attention
+          end
+
+        attention_window_size =
+          if layer_type == :sliding_attention do
+            radius =
+              if spec.use_bidirectional_attention,
+                do: spec.attention_window_size,
+                else: spec.attention_window_size - 1
+
+            {radius, radius}
+          end
+
         shortcut = hidden_state
 
         {hidden_state, attention, self_attention_cache} =
@@ -347,7 +426,8 @@ defmodule Bumblebee.Text.Qwen3 do
             num_key_value_heads: spec.num_key_value_heads,
             hidden_size: spec.hidden_size,
             attention_head_size: spec.attention_head_size,
-            causal: true,
+            causal: not spec.use_bidirectional_attention,
+            attention_window_size: attention_window_size,
             query_norm: qk_norm,
             key_norm: qk_norm,
             rotary_embedding: [
@@ -403,8 +483,33 @@ defmodule Bumblebee.Text.Qwen3 do
     def load(spec, data) do
       import Shared.Converters
 
+      data =
+        case Map.get(data, "use_sliding_window", false) do
+          false ->
+            Map.put(data, "sliding_window", nil)
+
+          true ->
+            Map.put_new(data, "sliding_window", 4096)
+
+          value ->
+            raise ArgumentError,
+                  "expected use_sliding_window to be a boolean, got: #{inspect(value)}"
+        end
+
       opts =
         convert!(data,
+          attention_window_size: {"sliding_window", optional(number())},
+          max_window_layers: {"max_window_layers", number()},
+          layer_types:
+            {"layer_types",
+             optional(
+               list(
+                 mapping(%{
+                   "full_attention" => :full_attention,
+                   "sliding_attention" => :sliding_attention
+                 })
+               )
+             )},
           vocab_size: {"vocab_size", number()},
           tie_word_embeddings: {"tie_word_embeddings", boolean()},
           max_positions: {"max_position_embeddings", number()},
@@ -418,6 +523,7 @@ defmodule Bumblebee.Text.Qwen3 do
           initializer_scale: {"initializer_range", number()},
           layer_norm_epsilon: {"rms_norm_eps", number()}
         ) ++
+          Shared.bidirectional_attention_options_from_transformers(data) ++
           Shared.rotary_embedding_options_from_transformers(data) ++
           Shared.common_options_from_transformers(data, spec)
 
diff --git a/notebooks/llms_rag.livemd b/notebooks/llms_rag.livemd
index 1d3bfc18..d69f7982 100644
--- a/notebooks/llms_rag.livemd
+++ b/notebooks/llms_rag.livemd
@@ -358,3 +358,63 @@ And here we have our answer!
 <!-- livebook:{"break_markdown":true} -->
 
 For additional context you can also visit the [Mistral docs](https://docs.mistral.ai/guides/rag/) that go through a similar example.
+
+## Embeddings from decoder models
+
+Gemma3Text, Llama, Mistral and Qwen3 support `use_bidirectional_attention: true`.
+Loading a checkpoint reads `is_causal` (inverted) for Llama, Mistral and Qwen3,
+and `use_bidirectional_attention` for Gemma3Text. The other spelling is also
+accepted as a fallback. Gemma's native field takes precedence when both are
+present; the other models prefer `is_causal`. All four default to causal
+attention, including the `:base` architecture.
+
+When a checkpoint does not declare its embedding attention mode, pass it
+explicitly when loading:
+
+```elixir
+# Replace with the repository of your embedding checkpoint.
+{:ok, model_info} =
+  Bumblebee.load_model({:hf, "organization/embedding-model"},
+    architecture: :base,
+    spec_overrides: [use_bidirectional_attention: true]
+  )
+```
+
+The same option works through `Bumblebee.configure/2` when constructing a spec.
+Consumers that delegate checkpoint loading and inference to Bumblebee, such as
+Sting, automatically use the checkpoint's declared attention mode after updating
+Bumblebee. Consumers supplying their own spec must preserve this option or pass
+the override above. Selecting `architecture: :base` alone does not enable it.
+
+Padding masks are combined with causal or bidirectional attention. Gemma's local
+bidirectional layers use half the configured sliding window as their radius;
+Mistral and Qwen3 use the full configured window as their bidirectional radius.
+Causal windows include the current token in their width. Qwen3 also loads the
+checkpoint's `use_sliding_window`, `max_window_layers` and `layer_types` settings.
+Bidirectional inference requires recomputing the whole sequence. Cache inputs,
+cache initialization and generation are rejected in this mode.
+
+RoPE configurations accept both `type` and `rope_type`, including linear,
+dynamic and YaRN scaling. Unsupported variants, conflicting type keys and
+invalid scaling parameters raise an error. YaRN uses the configured factor and
+the current Transformers frequency blending and attention scaling formula.
+Dynamic RoPE chooses its scale from the highest `position_ids` value, matching
+Transformers. For padding-independent dynamic scaling, supply positions derived
+from the attention mask, for example `Nx.max(Nx.cumulative_sum(mask, axis: 1) - 1, 0)`.
+The default positions include padding slots, just as in Transformers; padding
+that crosses the original context boundary can therefore change the scale.
+
+[EmbeddingGemma](https://huggingface.co/google/embeddinggemma-300m) uses a Gemma3Text
+backbone, but its complete embedding pipeline additionally requires prompt
+formatting, masked mean pooling, two trained Dense projections and L2
+normalization. Loading the backbone alone does not apply those projections.
+Use float32 or bfloat16 for EmbeddingGemma. Its gated checkpoint integration test
+can be run against an authorized local copy:
+
+```shell
+BUMBLEBEE_EMBEDDING_GEMMA=/path/to/checkpoint mix test \
+  test/bumblebee/text/embedding_gemma_integration_test.exs --include slow
+```
+
+That test exercises the backbone, tokenizer and pooled serving; deterministic
+small-model comparisons with Python Transformers run without Hub access.
diff --git a/test/bumblebee/layers_test.exs b/test/bumblebee/layers_test.exs
new file mode 100644
index 00000000..688ee7bd
--- /dev/null
+++ b/test/bumblebee/layers_test.exs
@@ -0,0 +1,156 @@
+defmodule Bumblebee.LayersTest do
+  use ExUnit.Case, async: true
+
+  import Bumblebee.TestHelpers
+
+  alias Bumblebee.Layers
+
+  test "documented YaRN configuration uses the default attention factor and truncation" do
+    strategy = %{
+      type: :yarn,
+      factor: 4.0,
+      original_max_positions: 2,
+      beta_fast: 32.0,
+      beta_slow: 1.0
+    }
+
+    {query, _key} = apply_rotary_embedding(strategy)
+    {init, predict} = Axon.build(query)
+
+    inputs = %{
+      "query" => Nx.broadcast(1.0, {1, 4, 1, 4}),
+      "key" => Nx.broadcast(1.0, {1, 4, 1, 4}),
+      "position_ids" => Nx.tensor([[0, 1, 2, 3]]),
+      "attention_mask" => Nx.tensor([[1, 1, 1, 1]])
+    }
+
+    params = init.(inputs, Axon.ModelState.empty())
+    output = predict.(params, inputs)
+
+    assert_all_close(
+      output[[0, 1..3, 0, 0..1]],
+      Nx.tensor([[-0.3012, 0.9975], [-1.3254, 0.9950], [-1.1311, 0.9925]])
+      |> Nx.multiply(0.1 * :math.log(4.0) + 1.0)
+    )
+  end
+
+  test "rejects an explicit unknown rotary scaling strategy" do
+    assert_raise ArgumentError, ~r/unsupported rotary embedding scaling strategy/, fn ->
+      apply_rotary_embedding(%{type: :unsupported})
+    end
+  end
+
+  test "rejects an explicit malformed linear scaling strategy" do
+    assert_raise ArgumentError, ~r/linear RoPE requires :factor to be a number >= 1/, fn ->
+      apply_rotary_embedding(%{type: :linear})
+    end
+  end
+
+  test "rejects non-positive LongRoPE factors" do
+    assert_raise ArgumentError, ~r/LongRoPE factors must each have 2 entries/, fn ->
+      apply_rotary_embedding(%{
+        type: :longrope,
+        short_factor: [1.0, 0.0],
+        long_factor: [1.0, 1.0],
+        original_max_positions: 2
+      })
+    end
+  end
+
+  test "dynamic RoPE exponent applies to the multiplier, using position IDs" do
+    query = Axon.input("query", shape: {1, 4, 1, 4})
+    key = Axon.input("key", shape: {1, 4, 1, 4})
+    position_ids = Axon.input("position_ids", shape: {1, 4})
+    attention_mask = Axon.input("attention_mask", shape: {1, 8})
+
+    {query, _key} =
+      Layers.rotary_embedding(query, key, position_ids, attention_mask, 4,
+        max_positions: 2,
+        base: 10_000,
+        scaling_strategy: %{type: :dynamic, factor: 2.0}
+      )
+
+    model = Axon.container(%{query: query})
+
+    inputs = %{
+      "query" => Nx.broadcast(1.0, {1, 4, 1, 4}),
+      "key" => Nx.broadcast(1.0, {1, 4, 1, 4}),
+      "position_ids" => Nx.tensor([[0, 1, 2, 3]]),
+      "attention_mask" => Nx.tensor([[1, 1, 1, 1, 0, 0, 0, 0]])
+    }
+
+    {init, predict} = Axon.build(model)
+    params = init.(inputs, Axon.ModelState.empty())
+    outputs = predict.(params, inputs)
+
+    scale = 2.0 * 4 / 2 - (2.0 - 1.0)
+    base = 10_000 * :math.pow(scale, 4 / (4 - 2))
+    angle = 1 / :math.sqrt(base)
+
+    assert_all_close(outputs.query[[0, 1, 0, 1]], Nx.tensor(:math.cos(angle) - :math.sin(angle)))
+  end
+
+  test "rejects dynamic RoPE with a rotary size of two" do
+    assert_raise ArgumentError, ~r/requires a rotary size greater than 2/, fn ->
+      query = Axon.input("query", shape: {1, 4, 1, 2})
+      key = Axon.input("key", shape: {1, 4, 1, 2})
+      position_ids = Axon.input("position_ids", shape: {1, 4})
+      attention_mask = Axon.input("attention_mask", shape: {1, 4})
+
+      Layers.rotary_embedding(query, key, position_ids, attention_mask, 2,
+        scaling_strategy: %{type: :dynamic, factor: 2.0}
+      )
+    end
+  end
+
+  test "YaRN scales rotary frequencies at positions beyond the original context" do
+    query = Axon.input("query", shape: {1, 4, 1, 4})
+    key = Axon.input("key", shape: {1, 4, 1, 4})
+    position_ids = Axon.input("position_ids", shape: {1, 4})
+    attention_mask = Axon.input("attention_mask", shape: {1, 4})
+
+    strategy = %{
+      type: :yarn,
+      factor: 4.0,
+      original_max_positions: 2,
+      beta_fast: 32.0,
+      beta_slow: 1.0,
+      attention_factor: 1.0,
+      truncate: true
+    }
+
+    {query, _key} =
+      Layers.rotary_embedding(query, key, position_ids, attention_mask, 4,
+        scaling_strategy: strategy
+      )
+
+    model = Axon.container(%{query: query})
+
+    inputs = %{
+      "query" => Nx.broadcast(1.0, {1, 4, 1, 4}),
+      "key" => Nx.broadcast(1.0, {1, 4, 1, 4}),
+      "position_ids" => Nx.tensor([[0, 1, 2, 3]]),
+      "attention_mask" => Nx.tensor([[1, 1, 1, 1]])
+    }
+
+    {init, predict} = Axon.build(model)
+    params = init.(inputs, Axon.ModelState.empty())
+    outputs = predict.(params, inputs)
+
+    assert_all_close(
+      outputs.query[[0, 1..3, 0, 0..1]],
+      Nx.tensor([[-0.3012, 0.9975], [-1.3254, 0.9950], [-1.1311, 0.9925]])
+    )
+  end
+
+  defp apply_rotary_embedding(strategy) do
+    query = Axon.input("query", shape: {1, 4, 1, 4})
+    key = Axon.input("key", shape: {1, 4, 1, 4})
+    position_ids = Axon.input("position_ids", shape: {1, 4})
+    attention_mask = Axon.input("attention_mask", shape: {1, 4})
+
+    Layers.rotary_embedding(query, key, position_ids, attention_mask, 4,
+      scaling_strategy: strategy
+    )
+  end
+end
diff --git a/test/bumblebee/shared_test.exs b/test/bumblebee/shared_test.exs
index 3467286c..7b93f633 100644
--- a/test/bumblebee/shared_test.exs
+++ b/test/bumblebee/shared_test.exs
@@ -3,6 +3,122 @@ defmodule Bumblebee.SharedTest do
 
   alias Bumblebee.Shared
 
+  describe "rotary_embedding_options_from_transformers/1" do
+    test "accepts the legacy type key" do
+      assert [rotary_embedding_scaling_strategy: %{type: :linear, factor: 2.0}] =
+               Shared.rotary_embedding_options_from_transformers(%{
+                 "rope_scaling" => %{"type" => "linear", "factor" => 2.0}
+               })
+    end
+
+    test "accepts the rope_type key for YaRN" do
+      assert [
+               rotary_embedding_scaling_strategy: %{
+                 type: :yarn,
+                 factor: 4.0,
+                 original_max_positions: 128,
+                 beta_fast: 32.0,
+                 beta_slow: 1.0
+               }
+             ] =
+               Shared.rotary_embedding_options_from_transformers(%{
+                 "max_position_embeddings" => 512,
+                 "rope_parameters" => %{
+                   "rope_type" => "yarn",
+                   "factor" => 4.0,
+                   "original_max_position_embeddings" => 128,
+                   "beta_fast" => 32.0,
+                   "beta_slow" => 1.0
+                 }
+               })
+    end
+
+    test "raises for an unsupported RoPE type" do
+      assert_raise RuntimeError, ~r/unsupported rotary embedding parameters/, fn ->
+        Shared.rotary_embedding_options_from_transformers(%{
+          "rope_scaling" => %{"type" => "unsupported", "factor" => 2.0}
+        })
+      end
+    end
+
+    test "raises for conflicting type aliases" do
+      assert_raise RuntimeError, ~r/conflicting "rope_type" and "type" values/, fn ->
+        Shared.rotary_embedding_options_from_transformers(%{
+          "rope_scaling" => %{"rope_type" => "linear", "type" => "dynamic", "factor" => 2.0}
+        })
+      end
+    end
+
+    test "raises for invalid YaRN parameters" do
+      assert_raise RuntimeError, ~r/requires a numeric "factor" >= 1/, fn ->
+        Shared.rotary_embedding_options_from_transformers(%{
+          "rope_scaling" => %{
+            "type" => "yarn",
+            "factor" => 0.5,
+            "original_max_position_embeddings" => 128
+          }
+        })
+      end
+    end
+
+    test "rejects scaling parameters without a type" do
+      assert_raise RuntimeError, ~r/scaling parameters require "type" or "rope_type"/, fn ->
+        Shared.rotary_embedding_options_from_transformers(%{
+          "rope_scaling" => %{"factor" => 2.0}
+        })
+      end
+    end
+
+    test "rejects a non-positive rope theta" do
+      assert_raise RuntimeError, ~r/"rope_theta" must be a positive number/, fn ->
+        Shared.rotary_embedding_options_from_transformers(%{"rope_theta" => 0})
+      end
+    end
+
+    test "preserves legacy Phi-3 YaRN LongRoPE without a factor" do
+      assert [
+               rotary_embedding_scaling_strategy: %{
+                 type: :longrope,
+                 short_factor: [1.0, 1.0],
+                 long_factor: [2.0, 2.0],
+                 original_max_positions: 128
+               }
+             ] =
+               Shared.rotary_embedding_options_from_transformers(%{
+                 "max_position_embeddings" => 512,
+                 "rope_scaling" => %{
+                   "type" => "yarn",
+                   "short_factor" => [1.0, 1.0],
+                   "long_factor" => [2.0, 2.0],
+                   "original_max_position_embeddings" => 128
+                 }
+               })
+    end
+  end
+
+  describe "bidirectional_attention_options_from_transformers/1" do
+    test "uses is_causal when present" do
+      assert [use_bidirectional_attention: true] =
+               Shared.bidirectional_attention_options_from_transformers(%{
+                 "is_causal" => false,
+                 "use_bidirectional_attention" => false
+               })
+    end
+
+    test "falls back to use_bidirectional_attention" do
+      assert [use_bidirectional_attention: true] =
+               Shared.bidirectional_attention_options_from_transformers(%{
+                 "use_bidirectional_attention" => true
+               })
+    end
+
+    test "rejects a non-boolean is_causal value" do
+      assert_raise RuntimeError, ~r/expected "is_causal" to be a boolean/, fn ->
+        Shared.bidirectional_attention_options_from_transformers(%{"is_causal" => "false"})
+      end
+    end
+  end
+
   describe "validate_label_options/1" do
     test "passes when :id_to_label is empty" do
       spec = %{__struct__: TestConfig, num_labels: 3, id_to_label: %{}}
diff --git a/test/bumblebee/text/embedding_attention_test.exs b/test/bumblebee/text/embedding_attention_test.exs
new file mode 100644
index 00000000..e80a159d
--- /dev/null
+++ b/test/bumblebee/text/embedding_attention_test.exs
@@ -0,0 +1,173 @@
+defmodule Bumblebee.Text.EmbeddingAttentionTest do
+  use ExUnit.Case, async: false
+
+  import Bumblebee.TestHelpers
+
+  alias Bumblebee.HuggingFace.Transformers.Config
+
+  for module <- [
+        Bumblebee.Text.Gemma3Text,
+        Bumblebee.Text.Llama,
+        Bumblebee.Text.Mistral,
+        Bumblebee.Text.Qwen3
+      ],
+      bidirectional <- [false, true] do
+    @module module
+    @bidirectional bidirectional
+
+    test "#{inspect(module)} attention boundaries and padding, bidirectional=#{bidirectional}" do
+      spec = spec(@module, @bidirectional)
+      inputs = %{"input_ids" => Nx.tensor([[1, 2, 3, 4, 5, 6, 7]])}
+
+      {init, predict} =
+        Axon.build(Bumblebee.build_model(spec),
+          compiler: EXLA,
+          global_layer_options: [output_attentions: true]
+        )
+
+      params = init.(inputs, Axon.ModelState.empty())
+      output = predict.(params, inputs)
+
+      radius = radius(@module, @bidirectional)
+      weights = elem(output.attentions, 0) |> Nx.to_flat_list()
+
+      expected =
+        for _head <- 1..2, query <- 0..6, key <- 0..6 do
+          allowed?(query, key, radius, @bidirectional)
+        end
+
+      assert Enum.map(weights, &(&1 > 0)) == expected
+
+      if @module in [Bumblebee.Text.Gemma3Text, Bumblebee.Text.Qwen3] do
+        global = elem(output.attentions, 1) |> Nx.to_flat_list()
+
+        expected_global =
+          for _head <- 1..2, query <- 0..6, key <- 0..6, do: @bidirectional or key <= query
+
+        assert Enum.map(global, &(&1 > 0)) == expected_global
+      end
+
+      padded = Map.put(inputs, "attention_mask", Nx.tensor([[1, 1, 1, 1, 0, 0, 0]]))
+      changed_padding = Map.put(padded, "input_ids", Nx.tensor([[1, 2, 3, 4, 10, 11, 12]]))
+
+      assert_all_close(
+        predict.(params, padded).hidden_state[[.., 0..3, ..]],
+        predict.(params, changed_padding).hidden_state[[.., 0..3, ..]]
+      )
+
+      changed_future = Map.put(inputs, "input_ids", Nx.tensor([[1, 2, 3, 4, 5, 6, 8]]))
+      before = output.hidden_state[[.., 5, ..]]
+      after_change = predict.(params, changed_future).hidden_state[[.., 5, ..]]
+
+      if @bidirectional do
+        refute Nx.to_number(Nx.all_close(before, after_change, atol: 1.0e-6, rtol: 1.0e-6)) == 1
+      else
+        assert_all_close(before, after_change, atol: 1.0e-6, rtol: 1.0e-6)
+      end
+    end
+  end
+
+  for module <- [
+        Bumblebee.Text.Gemma3Text,
+        Bumblebee.Text.Llama,
+        Bumblebee.Text.Mistral,
+        Bumblebee.Text.Qwen3
+      ] do
+    @module module
+
+    test "#{inspect(module)} causal cache matches full inference" do
+      spec = spec(@module, false)
+      model = Bumblebee.build_model(spec)
+
+      inputs = %{
+        "input_ids" => Nx.tensor([[1, 2, 3, 4, 5, 6, 7]]),
+        "attention_mask" => Nx.tensor([[1, 1, 1, 1, 1, 1, 1]])
+      }
+
+      {init, predict} = Axon.build(model, compiler: EXLA)
+      params = init.(inputs, Axon.ModelState.empty())
+      full = predict.(params, inputs)
+      cache = @module.init_cache(spec, 1, 7, %{})
+
+      prefix =
+        predict.(params, %{
+          "input_ids" => Nx.tensor([[1, 2, 3, 4, 5, 6]]),
+          "attention_mask" => Nx.tensor([[1, 1, 1, 1, 1, 1]]),
+          "cache" => cache
+        })
+
+      last =
+        predict.(params, %{
+          "input_ids" => Nx.tensor([[7]]),
+          "attention_mask" => Nx.tensor([[1]]),
+          "position_ids" => Nx.tensor([[6]]),
+          "cache" => prefix.cache
+        })
+
+      assert_all_close(last.hidden_state, full.hidden_state[[.., 6..6, ..]],
+        atol: 1.0e-5,
+        rtol: 1.0e-5
+      )
+    end
+
+    test "#{inspect(module)} cached generation matches uncached greedy decoding" do
+      spec =
+        Bumblebee.configure(spec(@module, false), architecture: :for_causal_language_modeling)
+
+      model = Bumblebee.build_model(spec)
+      inputs = %{"input_ids" => Nx.tensor([[1, 2]])}
+      {init, predict} = Axon.build(model, compiler: EXLA)
+      params = init.(inputs, Axon.ModelState.empty())
+
+      {expected, _ids} =
+        Enum.map_reduce(1..2, inputs["input_ids"], fn _, ids ->
+          logits = predict.(params, %{"input_ids" => ids}).logits
+          token = logits[[0, -1, ..]] |> Nx.argmax() |> Nx.to_number()
+          {token, Nx.concatenate([ids, Nx.tensor([[token]])], axis: 1)}
+        end)
+
+      config =
+        Bumblebee.configure(Bumblebee.Text.GenerationConfig, max_new_tokens: 2, pad_token_id: 0)
+
+      generate = Bumblebee.Text.Generation.build_generate(model, spec, config)
+      output = generate.(params, Map.put(inputs, "seed", Nx.tensor([0])))
+      assert_equal(output.token_ids, Nx.tensor([expected]))
+    end
+  end
+
+  defp allowed?(_query, _key, nil, true), do: true
+  defp allowed?(query, key, nil, false), do: key <= query
+  defp allowed?(query, key, radius, true), do: abs(query - key) <= radius
+  defp allowed?(query, key, radius, false), do: key <= query and query - key <= radius
+
+  defp radius(Bumblebee.Text.Llama, _), do: nil
+  defp radius(Bumblebee.Text.Gemma3Text, true), do: 2
+  defp radius(_, true), do: 4
+  defp radius(_, false), do: 3
+
+  defp spec(module, bidirectional) do
+    data = %{
+      "vocab_size" => 32,
+      "hidden_size" => 16,
+      "intermediate_size" => 20,
+      "num_hidden_layers" => 2,
+      "num_attention_heads" => 2,
+      "num_key_value_heads" => 1,
+      "head_dim" => 8,
+      "query_pre_attn_scalar" => 8,
+      "max_position_embeddings" => 16,
+      "sliding_window" => 4,
+      "use_sliding_window" => true,
+      "max_window_layers" => 0,
+      "is_causal" => not bidirectional,
+      "use_bidirectional_attention" => bidirectional
+    }
+
+    data =
+      if module in [Bumblebee.Text.Gemma3Text, Bumblebee.Text.Qwen3],
+        do: Map.put(data, "layer_types", ["sliding_attention", "full_attention"]),
+        else: data
+
+    Config.load(Bumblebee.configure(module), data)
+  end
+end
diff --git a/test/bumblebee/text/embedding_config_test.exs b/test/bumblebee/text/embedding_config_test.exs
new file mode 100644
index 00000000..ac24c8ed
--- /dev/null
+++ b/test/bumblebee/text/embedding_config_test.exs
@@ -0,0 +1,150 @@
+defmodule Bumblebee.Text.EmbeddingConfigTest do
+  use ExUnit.Case, async: true
+
+  alias Bumblebee.HuggingFace.Transformers.Config
+
+  test "Gemma prefers its native bidirectional field" do
+    spec = Bumblebee.configure(Bumblebee.Text.Gemma3Text)
+
+    assert Config.load(spec, %{"is_causal" => true, "use_bidirectional_attention" => true}).use_bidirectional_attention
+  end
+
+  test "Qwen3 loads sliding layers and respects the upstream enable flag" do
+    data = %{
+      "num_hidden_layers" => 3,
+      "use_sliding_window" => true,
+      "sliding_window" => 2,
+      "max_window_layers" => 1
+    }
+
+    spec = Config.load(Bumblebee.configure(Bumblebee.Text.Qwen3), data)
+    assert spec.attention_window_size == 2
+    assert spec.max_window_layers == 1
+    disabled = Config.load(spec, Map.put(data, "use_sliding_window", false))
+    assert disabled.attention_window_size == nil
+    assert Config.load(spec, Map.delete(data, "use_sliding_window")).attention_window_size == nil
+    assert Config.load(spec, Map.delete(data, "sliding_window")).attention_window_size == 4096
+
+    assert_raise ArgumentError, ~r/use_sliding_window/, fn ->
+      Config.load(spec, Map.put(data, "use_sliding_window", "false"))
+    end
+
+    explicit =
+      Config.load(
+        spec,
+        Map.put(data, "layer_types", ["full_attention", "sliding_attention", "full_attention"])
+      )
+
+    assert explicit.layer_types == [:full_attention, :sliding_attention, :full_attention]
+  end
+
+  for module <- [
+        Bumblebee.Text.Gemma3Text,
+        Bumblebee.Text.Llama,
+        Bumblebee.Text.Mistral,
+        Bumblebee.Text.Qwen3
+      ] do
+    @module module
+
+    test "#{inspect(module)} loads is_causal without changing the base default" do
+      spec = Bumblebee.configure(@module, architecture: :base)
+      refute Map.get(spec, :use_bidirectional_attention)
+      assert Config.load(spec, %{"is_causal" => false}).use_bidirectional_attention
+      refute Config.load(spec, %{"is_causal" => true}).use_bidirectional_attention
+
+      assert Bumblebee.configure(spec, use_bidirectional_attention: true).use_bidirectional_attention
+    end
+
+    test "#{inspect(module)} rejects bidirectional generation cache" do
+      spec = Config.load(Bumblebee.configure(@module), %{"is_causal" => false})
+
+      assert_raise ArgumentError, ~r/bidirectional/, fn ->
+        @module.init_cache(spec, 1, 8, %{})
+      end
+    end
+
+    test "#{inspect(module)} rejects a directly supplied bidirectional cache" do
+      spec =
+        @module
+        |> Bumblebee.configure(tiny_options(@module))
+        |> Config.load(%{"is_causal" => false})
+
+      model = Bumblebee.build_model(spec)
+
+      cache =
+        Bumblebee.Layers.Decoder.init_cache(1, 8,
+          hidden_size: spec.hidden_size,
+          attention_head_size: Map.get(spec, :attention_head_size),
+          decoder_num_attention_heads: spec.num_attention_heads,
+          decoder_num_key_value_heads: spec.num_key_value_heads,
+          decoder_num_blocks: spec.num_blocks
+        )
+
+      input_template = %{"input_ids" => Nx.template({1, 1}, :s64)}
+      {init_fun, predict_fun} = Axon.build(model, compiler: EXLA)
+      params = init_fun.(input_template, Axon.ModelState.empty())
+
+      assert_raise Axon.CompileError, ~r/bidirectional/, fn ->
+        predict_fun.(params, %{
+          "input_ids" => Nx.tensor([[1]]),
+          "cache" => cache
+        })
+      end
+    end
+  end
+
+  defp tiny_options(Bumblebee.Text.Gemma3Text) do
+    [
+      vocab_size: 32,
+      max_positions: 8,
+      hidden_size: 16,
+      intermediate_size: 20,
+      attention_head_size: 8,
+      attention_scale_base: 8,
+      num_blocks: 1,
+      num_attention_heads: 2,
+      num_key_value_heads: 1,
+      attention_window_size: 4,
+      layer_types: [:full_attention]
+    ]
+  end
+
+  defp tiny_options(Bumblebee.Text.Llama) do
+    [
+      vocab_size: 32,
+      max_positions: 8,
+      hidden_size: 16,
+      intermediate_size: 20,
+      attention_head_size: 8,
+      num_blocks: 1,
+      num_attention_heads: 2,
+      num_key_value_heads: 1
+    ]
+  end
+
+  defp tiny_options(Bumblebee.Text.Mistral) do
+    [
+      vocab_size: 32,
+      max_positions: 8,
+      hidden_size: 16,
+      intermediate_size: 20,
+      num_blocks: 1,
+      num_attention_heads: 2,
+      num_key_value_heads: 1,
+      attention_window_size: 4
+    ]
+  end
+
+  defp tiny_options(Bumblebee.Text.Qwen3) do
+    [
+      vocab_size: 32,
+      max_positions: 8,
+      hidden_size: 16,
+      intermediate_size: 20,
+      attention_head_size: 8,
+      num_blocks: 1,
+      num_attention_heads: 2,
+      num_key_value_heads: 1
+    ]
+  end
+end
diff --git a/test/bumblebee/text/embedding_gemma_integration_test.exs b/test/bumblebee/text/embedding_gemma_integration_test.exs
new file mode 100644
index 00000000..7c3a46e8
--- /dev/null
+++ b/test/bumblebee/text/embedding_gemma_integration_test.exs
@@ -0,0 +1,40 @@
+defmodule Bumblebee.Text.EmbeddingGemmaIntegrationTest do
+  use ExUnit.Case, async: false
+
+  @moduletag :slow
+  @moduletag timeout: 600_000
+
+  # Opt in with BUMBLEBEE_EMBEDDING_GEMMA=/path/to/checkpoint and --include slow.
+  # The gated Google checkpoint must already be available locally.
+  if path = System.get_env("BUMBLEBEE_EMBEDDING_GEMMA") do
+    import Bumblebee.TestHelpers
+
+    @path path
+
+    test "EmbeddingGemma transformer, tokenizer and embedding serving" do
+      repository = {:local, @path}
+      assert {:ok, model_info} = Bumblebee.load_model(repository, architecture: :base)
+      assert model_info.spec.use_bidirectional_attention
+      assert {:ok, tokenizer} = Bumblebee.load_tokenizer(repository)
+
+      serving =
+        Bumblebee.Text.text_embedding(model_info, tokenizer,
+          output_attribute: :hidden_state,
+          output_pool: :mean_pooling,
+          embedding_processor: :l2_norm
+        )
+
+      short = "task: search result | title: Example | text: A bee visits a flower."
+      long = "task: search result | title: Example | text: Bees collect pollen and make honey."
+      %{embedding: single} = Nx.Serving.run(serving, short)
+      [first, second] = Nx.Serving.run(serving, [short, long])
+
+      assert Nx.shape(single) == {model_info.spec.hidden_size}
+      assert_all_close(first.embedding, single, atol: 1.0e-4, rtol: 1.0e-4)
+
+      for embedding <- [first.embedding, second.embedding] do
+        assert_all_close(Nx.sum(Nx.pow(embedding, 2)), Nx.tensor(1.0))
+      end
+    end
+  end
+end
diff --git a/test/bumblebee/text/embedding_parity_test.exs b/test/bumblebee/text/embedding_parity_test.exs
new file mode 100644
index 00000000..61f7a052
--- /dev/null
+++ b/test/bumblebee/text/embedding_parity_test.exs
@@ -0,0 +1,94 @@
+defmodule Bumblebee.Text.EmbeddingParityTest do
+  use ExUnit.Case, async: true
+
+  import Bumblebee.TestHelpers
+
+  @fixtures Path.expand("../../fixtures/embedding_models", __DIR__)
+  # Float32 CPU reductions differ slightly between PyTorch and XLA. 1e-4
+  # leaves room for rounding while detecting mask and RoPE formula errors.
+  @models ~w(gemma3_text llama mistral qwen3 qwen3_linear qwen3_dynamic)
+
+  for name <- @models, mode <- ~w(causal bidirectional) do
+    @name name
+    @mode mode
+
+    test "#{name} #{@mode} matches Transformers hidden states" do
+      {model, params, reference} = load_fixture(@name, @mode)
+
+      outputs =
+        Axon.predict(model, params, %{
+          "input_ids" => Nx.tensor(reference["input_ids"]),
+          "attention_mask" => Nx.tensor(reference["attention_mask"])
+        })
+
+      assert_all_close(outputs.hidden_state, Nx.tensor(reference["hidden_state"]), atol: 1.0e-4)
+
+      changed_outputs =
+        Axon.predict(model, params, %{
+          "input_ids" => Nx.tensor([[1, 2, 3, 4, 5, 6, 7, 9]]),
+          "attention_mask" => Nx.tensor(reference["attention_mask"])
+        })
+
+      assert_all_close(
+        changed_outputs.hidden_state,
+        Nx.tensor(reference["changed_future_hidden_state"]),
+        atol: 1.0e-4
+      )
+
+      padded_outputs =
+        Axon.predict(model, params, %{
+          "input_ids" => Nx.tensor(reference["padded_input_ids"]),
+          "attention_mask" => Nx.tensor(reference["padded_attention_mask"]),
+          "position_ids" => Nx.tensor(reference["padded_position_ids"])
+        })
+
+      assert_all_close(
+        padded_outputs.hidden_state,
+        Nx.tensor(reference["padded_hidden_state"]),
+        atol: 1.0e-4
+      )
+
+      compact_outputs =
+        Axon.predict(model, params, %{
+          "input_ids" => Nx.tensor([[1, 2, 3, 4]]),
+          "attention_mask" => Nx.tensor([[1, 1, 1, 1]])
+        })
+
+      assert_all_close(compact_outputs.hidden_state, Nx.tensor(reference["compact_hidden_state"]),
+        atol: 1.0e-4,
+        rtol: 1.0e-4
+      )
+
+      assert_all_close(
+        padded_outputs.hidden_state[[.., 0..3, ..]],
+        compact_outputs.hidden_state,
+        atol: 1.0e-4
+      )
+
+      first_token_changed? =
+        outputs.hidden_state[[.., 0, ..]]
+        |> Nx.all_close(
+          changed_outputs.hidden_state[[.., 0, ..]],
+          atol: 1.0e-6,
+          rtol: 1.0e-6
+        )
+        |> Nx.to_number() == 0
+
+      assert first_token_changed? == (@mode == "bidirectional")
+    end
+  end
+
+  defp load_fixture(name, mode) do
+    path = Path.join(@fixtures, "tiny-random-#{name}-#{mode}")
+    reference = path |> Path.join("reference.json") |> File.read!() |> Jason.decode!()
+
+    assert {:ok, %{model: model, params: params, spec: spec}} =
+             Bumblebee.load_model({:local, path})
+
+    assert reference["transformers_version"] == "5.19.0.dev0"
+    assert reference["transformers_commit"] == "3693f8d26311305e914735a6373fb03468d6aaa0"
+    assert spec.use_bidirectional_attention == (mode == "bidirectional")
+
+    {model, params, reference}
+  end
+end
diff --git a/test/fixtures/embedding_models/generate.py b/test/fixtures/embedding_models/generate.py
new file mode 100644
index 00000000..03151063
--- /dev/null
+++ b/test/fixtures/embedding_models/generate.py
@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+"""Generate deterministic embedding-model parity fixtures.
+
+Requires Python 3.10+, torch 2.7.1 and Transformers commit
+3693f8d26311305e914735a6373fb03468d6aaa0. Install it with:
+
+    pip install torch==2.7.1 \
+      git+https://github.com/huggingface/transformers.git@3693f8d26311305e914735a6373fb03468d6aaa0
+
+The generated directories are ordinary local Hugging Face repositories, so
+they can be loaded by Bumblebee with {:local, path}. `reference.json` contains
+float32 hidden states from the same checkpoint and is intentionally kept
+separate from the checkpoint to make an update to either reviewable.
+"""
+
+import argparse
+import copy
+import json
+from pathlib import Path
+
+import torch
+import transformers
+from transformers import (
+    Gemma3TextConfig,
+    Gemma3TextModel,
+    LlamaConfig,
+    LlamaModel,
+    MistralConfig,
+    MistralModel,
+    Qwen3Config,
+    Qwen3Model,
+)
+
+
+torch.manual_seed(0)
+torch.set_printoptions(precision=9)
+
+INPUT_IDS = torch.tensor([[1, 2, 3, 4, 5, 6, 7, 8]])
+CHANGED_FUTURE_IDS = torch.tensor([[1, 2, 3, 4, 5, 6, 7, 9]])
+PADDED_IDS = torch.tensor([[1, 2, 3, 4, 0, 0, 0, 0]])
+FULL_MASK = torch.ones_like(INPUT_IDS)
+PADDED_MASK = torch.tensor([[1, 1, 1, 1, 0, 0, 0, 0]])
+
+
+def common_kwargs():
+    return {
+        "vocab_size": 32,
+        "hidden_size": 16,
+        "intermediate_size": 20,
+        "num_hidden_layers": 2,
+        "num_attention_heads": 2,
+        "num_key_value_heads": 1,
+        "head_dim": 8,
+        "max_position_embeddings": 4,
+        "rms_norm_eps": 1.0e-6,
+        "pad_token_id": 0,
+        "tie_word_embeddings": False,
+    }
+
+
+def model_definitions():
+    common = common_kwargs()
+
+    return {
+        "gemma3_text": (
+            Gemma3TextConfig,
+            Gemma3TextModel,
+            {
+                **common,
+                "hidden_activation": "gelu_pytorch_tanh",
+                "query_pre_attn_scalar": 8,
+                "sliding_window": 4,
+                "layer_types": ["sliding_attention", "full_attention"],
+                "rope_parameters": {
+                    "full_attention": {
+                        "rope_type": "yarn",
+                        "factor": 2.0,
+                        "original_max_position_embeddings": 4,
+                        "beta_fast": 32.0,
+                        "beta_slow": 1.0,
+                    },
+                    "sliding_attention": {"rope_theta": 10_000.0},
+                },
+            },
+        ),
+        "llama": (LlamaConfig, LlamaModel, {**common, "rope_theta": 10_000.0}),
+        "mistral": (
+            MistralConfig,
+            MistralModel,
+            {**common, "rope_theta": 10_000.0, "sliding_window": 4},
+        ),
+        "qwen3": (
+            Qwen3Config,
+            Qwen3Model,
+            {
+                **common,
+                "rope_theta": 10_000.0,
+                "use_sliding_window": True,
+                "sliding_window": 4,
+                "max_window_layers": 1,
+                "rope_scaling": {
+                    "rope_type": "yarn",
+                    "factor": 2.0,
+                    "original_max_position_embeddings": 4,
+                    "beta_fast": 32.0,
+                    "beta_slow": 1.0,
+                },
+            },
+        ),
+        # Exercise both accepted serialized spellings. These inputs are longer
+        # than max_position_embeddings, so changing the scaling cannot be a
+        # no-op in the generated reference.
+        "qwen3_linear": (
+            Qwen3Config,
+            Qwen3Model,
+            {
+                **common,
+                "rope_theta": 10_000.0,
+                "rope_scaling": {"type": "linear", "factor": 2.0},
+            },
+        ),
+        "qwen3_dynamic": (
+            Qwen3Config,
+            Qwen3Model,
+            {
+                **common,
+                "rope_theta": 10_000.0,
+                "rope_scaling": {"rope_type": "dynamic", "factor": 2.0},
+            },
+        ),
+    }
+
+
+def tensor(value):
+    return value.detach().to(torch.float32).cpu().tolist()
+
+
+def evaluate(model, input_ids, attention_mask, position_ids=None):
+    # Dynamic RoPE caches its frequency state. Each reference represents an
+    # independent inference so prior long inputs must not affect short ones.
+    model = copy.deepcopy(model)
+    with torch.no_grad():
+        return tensor(model(input_ids=input_ids, attention_mask=attention_mask,
+                            position_ids=position_ids).last_hidden_state)
+
+
+def write_fixture(output_dir, name, config_class, model_class, kwargs, is_causal):
+    directory = output_dir / f"tiny-random-{name}-{'causal' if is_causal else 'bidirectional'}"
+    # Gemma normalizes its local window in __post_init__, so these flags must
+    # be present during construction rather than assigned afterward.
+    attention_mode = (
+        {"use_bidirectional_attention": not is_causal}
+        if name == "gemma3_text"
+        else {"is_causal": is_causal}
+    )
+    config = config_class(**kwargs, **attention_mode, _attn_implementation="eager")
+
+    model = model_class(config).eval()
+    model.save_pretrained(directory, safe_serialization=True)
+
+    # Compare against exactly the serialized checkpoint configuration. This
+    # catches model-specific config normalization (notably Gemma's window).
+    config = config_class.from_pretrained(directory)
+    model = model_class.from_pretrained(directory, attn_implementation="eager").eval()
+
+    reference = {
+        "transformers_version": transformers.__version__,
+        "transformers_commit": "3693f8d26311305e914735a6373fb03468d6aaa0",
+        "is_causal": is_causal,
+        "input_ids": INPUT_IDS.tolist(),
+        "attention_mask": FULL_MASK.tolist(),
+        "hidden_state": evaluate(model, INPUT_IDS, FULL_MASK),
+        "changed_future_hidden_state": evaluate(model, CHANGED_FUTURE_IDS, FULL_MASK),
+        "padded_input_ids": PADDED_IDS.tolist(),
+        "padded_attention_mask": PADDED_MASK.tolist(),
+        "padded_position_ids": [[0, 1, 2, 3, 0, 0, 0, 0]],
+        "padded_hidden_state": evaluate(model, PADDED_IDS, PADDED_MASK,
+                                       torch.tensor([[0, 1, 2, 3, 0, 0, 0, 0]])),
+        "compact_hidden_state": evaluate(model, INPUT_IDS[:, :4], FULL_MASK[:, :4]),
+    }
+
+    (directory / "reference.json").write_text(json.dumps(reference, indent=2) + "\n")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", type=Path, default=Path(__file__).parent)
+    args = parser.parse_args()
+    args.output.mkdir(parents=True, exist_ok=True)
+
+    for name, (config_class, model_class, kwargs) in model_definitions().items():
+        for is_causal in (True, False):
+            write_fixture(args.output, name, config_class, model_class, kwargs, is_causal)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/test/fixtures/embedding_models/tiny-random-gemma3_text-bidirectional/config.json b/test/fixtures/embedding_models/tiny-random-gemma3_text-bidirectional/config.json
new file mode 100644
index 00000000..d55cd358
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-gemma3_text-bidirectional/config.json
@@ -0,0 +1,50 @@
+{
+  "_sliding_window_pattern": 6,
+  "architectures": [
+    "Gemma3TextModel"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": null,
+  "bos_token_id": 2,
+  "dtype": "float32",
+  "eos_token_id": 1,
+  "final_logit_softcapping": null,
+  "head_dim": 8,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 20,
+  "layer_types": [
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 4,
+  "model_type": "gemma3_text",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 8,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "full_attention": {
+      "beta_fast": 32.0,
+      "beta_slow": 1.0,
+      "factor": 2.0,
+      "original_max_position_embeddings": 4,
+      "rope_theta": 1000000.0,
+      "rope_type": "yarn"
+    },
+    "sliding_attention": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    }
+  },
+  "sliding_window": 4,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.19.0.dev0",
+  "use_bidirectional_attention": true,
+  "use_cache": true,
+  "vocab_size": 32
+}
diff --git a/test/fixtures/embedding_models/tiny-random-gemma3_text-bidirectional/model.safetensors b/test/fixtures/embedding_models/tiny-random-gemma3_text-bidirectional/model.safetensors
new file mode 100644
index 00000000..2d0eea3f
Binary files /dev/null and b/test/fixtures/embedding_models/tiny-random-gemma3_text-bidirectional/model.safetensors differ
diff --git a/test/fixtures/embedding_models/tiny-random-gemma3_text-bidirectional/reference.json b/test/fixtures/embedding_models/tiny-random-gemma3_text-bidirectional/reference.json
new file mode 100644
index 00000000..b878cf3d
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-gemma3_text-bidirectional/reference.json
@@ -0,0 +1,585 @@
+{
+  "transformers_version": "5.19.0.dev0",
+  "transformers_commit": "3693f8d26311305e914735a6373fb03468d6aaa0",
+  "is_causal": false,
+  "input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8
+    ]
+  ],
+  "attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1
+    ]
+  ],
+  "hidden_state": [
+    [
+      [
+        -0.13210436701774597,
+        0.23337604105472565,
+        -1.2281488180160522,
+        0.5725199580192566,
+        -0.2724170982837677,
+        -0.5558069348335266,
+        1.6916804313659668,
+        1.0031288862228394,
+        1.8693288564682007,
+        -1.562204122543335,
+        -0.14855922758579254,
+        -1.6027384996414185,
+        -0.7507337927818298,
+        0.24168244004249573,
+        0.6303021311759949,
+        0.5438350439071655
+      ],
+      [
+        -0.46407729387283325,
+        -1.0738422870635986,
+        -0.4108755588531494,
+        -0.16676579415798187,
+        -0.6514304876327515,
+        -1.2998942136764526,
+        1.3124279975891113,
+        0.6797420382499695,
+        1.538434386253357,
+        -0.6352390646934509,
+        -1.216679334640503,
+        -2.213629961013794,
+        -0.1813531070947647,
+        -0.4744988679885864,
+        0.2821655869483948,
+        0.8048733472824097
+      ],
+      [
+        -0.32454708218574524,
+        0.563446044921875,
+        -0.9538421630859375,
+        1.387927532196045,
+        -0.6267942190170288,
+        -0.8507993817329407,
+        1.120371699333191,
+        0.6889138221740723,
+        1.9096094369888306,
+        -1.5304726362228394,
+        0.39155346155166626,
+        -1.4544148445129395,
+        -0.5737031102180481,
+        1.0799825191497803,
+        0.3197358250617981,
+        0.197853684425354
+      ],
+      [
+        -0.6982513070106506,
+        -1.259855031967163,
+        -0.5954171419143677,
+        0.45602455735206604,
+        -0.7666040062904358,
+        -0.9540395736694336,
+        0.5957540273666382,
+        0.6909412145614624,
+        0.6792640089988708,
+        -0.7413315773010254,
+        -1.4983843564987183,
+        -2.4805829524993896,
+        0.1556367427110672,
+        -0.2697519063949585,
+        -0.3573997914791107,
+        1.1825302839279175
+      ],
+      [
+        -0.5305964946746826,
+        -1.0686851739883423,
+        -0.21012404561042786,
+        0.005586860701441765,
+        -0.7698182463645935,
+        -0.10610931366682053,
+        0.01697503589093685,
+        0.7949861884117126,
+        0.5444154739379883,
+        -0.5198249220848083,
+        -1.371039628982544,
+        -2.6099109649658203,
+        0.5943748354911804,
+        -0.21777388453483582,
+        -0.9894182085990906,
+        1.6304508447647095
+      ],
+      [
+        1.3124809265136719,
+        0.7452348470687866,
+        -0.6058535575866699,
+        0.44669443368911743,
+        -0.732239842414856,
+        0.41762393712997437,
+        0.8677704930305481,
+        1.2008593082427979,
+        1.2503892183303833,
+        -1.3793572187423706,
+        -1.3278270959854126,
+        -1.3320661783218384,
+        -1.0296897888183594,
+        1.0111056566238403,
+        1.077060341835022,
+        -0.05882374942302704
+      ],
+      [
+        0.9501075148582458,
+        0.8512008190155029,
+        -0.3093390464782715,
+        1.3917850255966187,
+        -0.8773398995399475,
+        -0.284389853477478,
+        0.3124425411224365,
+        0.6063923835754395,
+        1.2674648761749268,
+        -1.520995020866394,
+        -0.29790225625038147,
+        -1.1363801956176758,
+        -0.6247493028640747,
+        2.152383327484131,
+        0.8025873899459839,
+        -0.238418310880661
+      ],
+      [
+        0.9078593254089355,
+        0.1294756382703781,
+        -0.5611308813095093,
+        0.45771363377571106,
+        -0.9327450394630432,
+        0.6462754607200623,
+        0.3561619818210602,
+        0.5378808975219727,
+        1.0970487594604492,
+        -1.1739202737808228,
+        -2.087289333343506,
+        -1.783697247505188,
+        -0.44551295042037964,
+        1.3804746866226196,
+        0.3671271800994873,
+        0.7562164068222046
+      ]
+    ]
+  ],
+  "changed_future_hidden_state": [
+    [
+      [
+        -0.9753727316856384,
+        0.6431930661201477,
+        -0.3255528211593628,
+        0.3518012762069702,
+        -0.2527543008327484,
+        0.7128898501396179,
+        0.6717832088470459,
+        1.786870002746582,
+        1.1995052099227905,
+        -1.338497281074524,
+        -0.24924075603485107,
+        -1.7627183198928833,
+        -0.3551167845726013,
+        -0.11474043875932693,
+        -0.21994325518608093,
+        1.8977800607681274
+      ],
+      [
+        -1.1723068952560425,
+        -0.4238891303539276,
+        0.2455168515443802,
+        -0.08795787394046783,
+        -0.46628570556640625,
+        0.2637508809566498,
+        -0.11278499662876129,
+        0.8986696004867554,
+        0.7016419768333435,
+        -0.671364963054657,
+        -1.0277780294418335,
+        -2.0820438861846924,
+        0.6504681706428528,
+        -0.5555765628814697,
+        -1.1048330068588257,
+        2.23248291015625
+      ],
+      [
+        -1.152288556098938,
+        0.9222949147224426,
+        -0.14148148894309998,
+        1.2826206684112549,
+        -0.4728870391845703,
+        0.44585365056991577,
+        0.3354341387748718,
+        1.3694349527359009,
+        1.326727271080017,
+        -1.2007604837417603,
+        0.2500269114971161,
+        -1.7627290487289429,
+        -0.21177469193935394,
+        0.617283046245575,
+        -0.5384304523468018,
+        1.630531668663025
+      ],
+      [
+        -1.1433430910110474,
+        -0.6229028701782227,
+        0.08498617261648178,
+        0.2991422712802887,
+        -0.5391479730606079,
+        0.2749215364456177,
+        -0.2964751720428467,
+        0.946112871170044,
+        0.28172770142555237,
+        -0.6584761738777161,
+        -1.1335771083831787,
+        -2.216383218765259,
+        0.553760826587677,
+        -0.3507816195487976,
+        -0.9403942823410034,
+        2.1987268924713135
+      ],
+      [
+        -0.7303062081336975,
+        -0.6918209195137024,
+        0.07434859871864319,
+        -0.0338260680437088,
+        -0.693679928779602,
+        0.4977053999900818,
+        -0.36925584077835083,
+        0.9168241620063782,
+        0.4288889765739441,
+        -0.44034022092819214,
+        -1.1048322916030884,
+        -2.3282554149627686,
+        0.746461808681488,
+        -0.2518112063407898,
+        -1.076302170753479,
+        2.11594557762146
+      ],
+      [
+        -0.5948643088340759,
+        0.8973981738090515,
+        -0.9399436712265015,
+        -0.3279058635234833,
+        -0.7442126870155334,
+        0.3421555459499359,
+        0.0754547119140625,
+        3.055392026901245,
+        0.3894348740577698,
+        -1.0438923835754395,
+        0.03873901814222336,
+        -0.7767350077629089,
+        -0.588545024394989,
+        0.6499618291854858,
+        -1.0527260303497314,
+        0.3383733332157135
+      ],
+      [
+        -0.6630761623382568,
+        1.4255210161209106,
+        -0.6638501286506653,
+        0.9768269062042236,
+        -0.7307347655296326,
+        -0.004438987001776695,
+        0.7805944085121155,
+        1.3486998081207275,
+        1.23172926902771,
+        -1.2375954389572144,
+        0.9031170606613159,
+        -0.6784186959266663,
+        -0.43389004468917847,
+        2.0472681522369385,
+        -0.3186670243740082,
+        0.604339063167572
+      ],
+      [
+        -0.5133271813392639,
+        0.8213608860969543,
+        -1.656144380569458,
+        0.29642045497894287,
+        -0.812745213508606,
+        -0.260500431060791,
+        0.6946814656257629,
+        2.142383337020874,
+        0.6545501351356506,
+        -1.341208577156067,
+        0.8497481942176819,
+        -0.25999322533607483,
+        -0.7901588082313538,
+        1.5306577682495117,
+        -0.5842506885528564,
+        -0.323751300573349
+      ]
+    ]
+  ],
+  "padded_input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_position_ids": [
+    [
+      0,
+      1,
+      2,
+      3,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_hidden_state": [
+    [
+      [
+        -0.6343645453453064,
+        0.32506078481674194,
+        -1.1565942764282227,
+        0.6089161038398743,
+        0.017442181706428528,
+        0.21338129043579102,
+        1.5142375230789185,
+        1.1112605333328247,
+        1.1390902996063232,
+        -2.0550549030303955,
+        0.13616551458835602,
+        -1.6543365716934204,
+        -0.10141187161207199,
+        0.3419748842716217,
+        -0.34499630331993103,
+        1.2989859580993652
+      ],
+      [
+        -0.9563459157943726,
+        -0.9621767401695251,
+        -0.23561029136180878,
+        0.03475348278880119,
+        -0.20982760190963745,
+        -0.048457805067300797,
+        1.02187979221344,
+        0.6139166355133057,
+        0.6495475172996521,
+        -1.334670066833496,
+        -0.5648036003112793,
+        -2.147329807281494,
+        0.7038346529006958,
+        -0.19039209187030792,
+        -1.1776238679885864,
+        1.8930578231811523
+      ],
+      [
+        -0.6799596548080444,
+        0.5036802291870117,
+        -1.041325569152832,
+        0.7408269643783569,
+        -0.08192671835422516,
+        0.11259033530950546,
+        1.4107047319412231,
+        0.7080919742584229,
+        1.2730152606964111,
+        -2.1199257373809814,
+        0.5192409753799438,
+        -1.584995150566101,
+        -0.1333114355802536,
+        0.8585798144340515,
+        -0.3686715066432953,
+        1.163002371788025
+      ],
+      [
+        -1.072130799293518,
+        -1.130326271057129,
+        -0.32342904806137085,
+        0.37405577301979065,
+        -0.36024054884910583,
+        -0.09502308070659637,
+        0.5731084942817688,
+        0.5162522792816162,
+        0.03796820715069771,
+        -1.227203130722046,
+        -0.8792113065719604,
+        -2.2229766845703125,
+        0.6502164602279663,
+        -0.0829479843378067,
+        -1.0404772758483887,
+        1.9647468328475952
+      ],
+      [
+        -0.9233806133270264,
+        -1.2643541097640991,
+        -0.3004501163959503,
+        0.34551334381103516,
+        -0.3248680830001831,
+        0.656758189201355,
+        0.6524069309234619,
+        0.7162616848945618,
+        -0.011871661059558392,
+        -1.3602238893508911,
+        -0.701316773891449,
+        -2.173640251159668,
+        0.47114819288253784,
+        -0.028277907520532608,
+        -0.7253252863883972,
+        2.0119130611419678
+      ],
+      [
+        -0.6347136497497559,
+        -1.7007163763046265,
+        -0.32496461272239685,
+        -0.3386383056640625,
+        -0.19950725138187408,
+        0.21478725969791412,
+        0.5630044937133789,
+        0.17124132812023163,
+        0.5554924607276917,
+        -1.0240470170974731,
+        -0.8255114555358887,
+        -2.222484588623047,
+        0.5628509521484375,
+        0.2239212989807129,
+        -1.2834008932113647,
+        1.749275803565979
+      ],
+      [
+        -0.7188950777053833,
+        -0.49560821056365967,
+        -0.5206871628761292,
+        0.3803420662879944,
+        -0.20172876119613647,
+        0.41091227531433105,
+        1.1794553995132446,
+        0.9419419169425964,
+        0.4955349862575531,
+        -1.7526850700378418,
+        -0.44055837392807007,
+        -2.032241106033325,
+        0.309419184923172,
+        0.15277352929115295,
+        -0.7071409821510315,
+        2.018211603164673
+      ],
+      [
+        -0.7188950777053833,
+        -0.49560821056365967,
+        -0.5206871628761292,
+        0.3803420662879944,
+        -0.20172876119613647,
+        0.41091227531433105,
+        1.1794553995132446,
+        0.9419419169425964,
+        0.4955349862575531,
+        -1.7526850700378418,
+        -0.44055837392807007,
+        -2.032241106033325,
+        0.309419184923172,
+        0.15277352929115295,
+        -0.7071409821510315,
+        2.018211603164673
+      ]
+    ]
+  ],
+  "compact_hidden_state": [
+    [
+      [
+        -0.6343646049499512,
+        0.32506075501441956,
+        -1.1565942764282227,
+        0.6089160442352295,
+        0.01744205690920353,
+        0.2133813500404358,
+        1.514237403869629,
+        1.1112607717514038,
+        1.1390904188156128,
+        -2.0550549030303955,
+        0.1361653208732605,
+        -1.6543365716934204,
+        -0.10141189396381378,
+        0.3419749438762665,
+        -0.34499654173851013,
+        1.2989860773086548
+      ],
+      [
+        -0.9563457369804382,
+        -0.9621763825416565,
+        -0.23561035096645355,
+        0.034753382205963135,
+        -0.20982757210731506,
+        -0.048457883298397064,
+        1.02187979221344,
+        0.6139165163040161,
+        0.6495476365089417,
+        -1.3346699476242065,
+        -0.5648035407066345,
+        -2.147329568862915,
+        0.703834593296051,
+        -0.19039218127727509,
+        -1.1776237487792969,
+        1.8930577039718628
+      ],
+      [
+        -0.6799596548080444,
+        0.5036804676055908,
+        -1.0413256883621216,
+        0.7408270239830017,
+        -0.08192683011293411,
+        0.1125902459025383,
+        1.4107047319412231,
+        0.7080918550491333,
+        1.2730152606964111,
+        -2.1199257373809814,
+        0.5192410349845886,
+        -1.584995150566101,
+        -0.13331158459186554,
+        0.8585799932479858,
+        -0.36867156624794006,
+        1.163002371788025
+      ],
+      [
+        -1.0721309185028076,
+        -1.130326271057129,
+        -0.32342901825904846,
+        0.3740558624267578,
+        -0.36024075746536255,
+        -0.095023013651371,
+        0.573108434677124,
+        0.5162522196769714,
+        0.03796820715069771,
+        -1.227203130722046,
+        -0.87921142578125,
+        -2.2229766845703125,
+        0.650216281414032,
+        -0.08294793218374252,
+        -1.0404772758483887,
+        1.9647468328475952
+      ]
+    ]
+  ]
+}
diff --git a/test/fixtures/embedding_models/tiny-random-gemma3_text-causal/config.json b/test/fixtures/embedding_models/tiny-random-gemma3_text-causal/config.json
new file mode 100644
index 00000000..523c855e
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-gemma3_text-causal/config.json
@@ -0,0 +1,50 @@
+{
+  "_sliding_window_pattern": 6,
+  "architectures": [
+    "Gemma3TextModel"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": null,
+  "bos_token_id": 2,
+  "dtype": "float32",
+  "eos_token_id": 1,
+  "final_logit_softcapping": null,
+  "head_dim": 8,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 20,
+  "layer_types": [
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 4,
+  "model_type": "gemma3_text",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 8,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "full_attention": {
+      "beta_fast": 32.0,
+      "beta_slow": 1.0,
+      "factor": 2.0,
+      "original_max_position_embeddings": 4,
+      "rope_theta": 1000000.0,
+      "rope_type": "yarn"
+    },
+    "sliding_attention": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    }
+  },
+  "sliding_window": 4,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.19.0.dev0",
+  "use_bidirectional_attention": false,
+  "use_cache": true,
+  "vocab_size": 32
+}
diff --git a/test/fixtures/embedding_models/tiny-random-gemma3_text-causal/model.safetensors b/test/fixtures/embedding_models/tiny-random-gemma3_text-causal/model.safetensors
new file mode 100644
index 00000000..12e5fdca
Binary files /dev/null and b/test/fixtures/embedding_models/tiny-random-gemma3_text-causal/model.safetensors differ
diff --git a/test/fixtures/embedding_models/tiny-random-gemma3_text-causal/reference.json b/test/fixtures/embedding_models/tiny-random-gemma3_text-causal/reference.json
new file mode 100644
index 00000000..6deb2e57
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-gemma3_text-causal/reference.json
@@ -0,0 +1,585 @@
+{
+  "transformers_version": "5.19.0.dev0",
+  "transformers_commit": "3693f8d26311305e914735a6373fb03468d6aaa0",
+  "is_causal": true,
+  "input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8
+    ]
+  ],
+  "attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1
+    ]
+  ],
+  "hidden_state": [
+    [
+      [
+        1.0479530096054077,
+        0.777589738368988,
+        0.4487590491771698,
+        0.8083864450454712,
+        -0.46256136894226074,
+        0.9383790493011475,
+        0.05815654620528221,
+        -0.3330000638961792,
+        -0.23830658197402954,
+        -0.10689257085323334,
+        -1.7566187381744385,
+        1.4047011137008667,
+        1.5862897634506226,
+        -1.4487812519073486,
+        0.34458714723587036,
+        -1.5402408838272095
+      ],
+      [
+        1.106600046157837,
+        0.88736891746521,
+        0.23645684123039246,
+        0.6962629556655884,
+        -0.6025262475013733,
+        0.8922804594039917,
+        -0.08937154710292816,
+        -0.3328021466732025,
+        -0.30558285117149353,
+        0.0372898168861866,
+        -1.7399507761001587,
+        1.2930703163146973,
+        1.527563452720642,
+        -1.574615240097046,
+        0.49537399411201477,
+        -1.522128701210022
+      ],
+      [
+        0.7689082026481628,
+        1.2421519756317139,
+        -0.008993715979158878,
+        0.5178283452987671,
+        -1.0837748050689697,
+        0.5594359636306763,
+        -0.6521210074424744,
+        -0.2873961329460144,
+        -0.4735589623451233,
+        0.2963488698005676,
+        -1.317405343055725,
+        1.1891876459121704,
+        1.605422019958496,
+        -1.5092140436172485,
+        0.7168599367141724,
+        -1.6647543907165527
+      ],
+      [
+        0.682964563369751,
+        1.1864830255508423,
+        0.3529805839061737,
+        0.5533444881439209,
+        -1.5137571096420288,
+        0.9992331266403198,
+        0.3350704610347748,
+        -0.24685613811016083,
+        0.28254324197769165,
+        -0.5005261301994324,
+        -1.438392996788025,
+        1.6214579343795776,
+        0.7328647971153259,
+        -0.6623048782348633,
+        0.07444841414690018,
+        -2.0547866821289062
+      ],
+      [
+        0.2542734444141388,
+        1.404710054397583,
+        -0.16682158410549164,
+        0.16858866810798645,
+        -1.786678671836853,
+        1.1306204795837402,
+        0.246002197265625,
+        -0.18968123197555542,
+        1.3288171291351318,
+        -0.16666997969150543,
+        -0.24833203852176666,
+        1.486351490020752,
+        0.390087366104126,
+        0.2801700830459595,
+        -0.5608059167861938,
+        -2.174682855606079
+      ],
+      [
+        -0.399098664522171,
+        -0.3210754990577698,
+        -0.7301748394966125,
+        -0.17693063616752625,
+        -2.7776947021484375,
+        -1.339138388633728,
+        0.47883638739585876,
+        -0.14695921540260315,
+        0.11954823136329651,
+        0.5356278419494629,
+        -0.8720952868461609,
+        1.1138038635253906,
+        -0.6852717399597168,
+        -0.3124436140060425,
+        0.8931537866592407,
+        -1.3214232921600342
+      ],
+      [
+        -0.11504136025905609,
+        -0.2380775660276413,
+        -1.3998239040374756,
+        -0.0863225981593132,
+        -1.8701715469360352,
+        -1.2635517120361328,
+        0.2683332562446594,
+        0.3396027684211731,
+        -0.7382551431655884,
+        1.4094324111938477,
+        0.3622351884841919,
+        2.0395216941833496,
+        0.4131108224391937,
+        -0.08965935558080673,
+        0.6820758581161499,
+        -1.1024388074874878
+      ],
+      [
+        -0.10180805623531342,
+        -0.22903907299041748,
+        -1.2326991558074951,
+        0.2551376521587372,
+        -2.5957112312316895,
+        -0.3065643012523651,
+        0.38770490884780884,
+        0.5183053612709045,
+        -0.1485462635755539,
+        0.14053478837013245,
+        -0.36415183544158936,
+        2.0778098106384277,
+        -0.09778991341590881,
+        0.6267010569572449,
+        -0.018824545666575432,
+        -1.4857618808746338
+      ]
+    ]
+  ],
+  "changed_future_hidden_state": [
+    [
+      [
+        1.0479530096054077,
+        0.777589738368988,
+        0.4487590491771698,
+        0.8083864450454712,
+        -0.46256136894226074,
+        0.9383790493011475,
+        0.05815654620528221,
+        -0.3330000638961792,
+        -0.23830658197402954,
+        -0.10689257085323334,
+        -1.7566187381744385,
+        1.4047011137008667,
+        1.5862897634506226,
+        -1.4487812519073486,
+        0.34458714723587036,
+        -1.5402408838272095
+      ],
+      [
+        1.106600046157837,
+        0.88736891746521,
+        0.23645684123039246,
+        0.6962629556655884,
+        -0.6025262475013733,
+        0.8922804594039917,
+        -0.08937154710292816,
+        -0.3328021466732025,
+        -0.30558285117149353,
+        0.0372898168861866,
+        -1.7399507761001587,
+        1.2930703163146973,
+        1.527563452720642,
+        -1.574615240097046,
+        0.49537399411201477,
+        -1.522128701210022
+      ],
+      [
+        0.7689082026481628,
+        1.2421519756317139,
+        -0.008993715979158878,
+        0.5178283452987671,
+        -1.0837748050689697,
+        0.5594359636306763,
+        -0.6521210074424744,
+        -0.2873961329460144,
+        -0.4735589623451233,
+        0.2963488698005676,
+        -1.317405343055725,
+        1.1891876459121704,
+        1.605422019958496,
+        -1.5092140436172485,
+        0.7168599367141724,
+        -1.6647543907165527
+      ],
+      [
+        0.682964563369751,
+        1.1864830255508423,
+        0.3529805839061737,
+        0.5533444881439209,
+        -1.5137571096420288,
+        0.9992331266403198,
+        0.3350704610347748,
+        -0.24685613811016083,
+        0.28254324197769165,
+        -0.5005261301994324,
+        -1.438392996788025,
+        1.6214579343795776,
+        0.7328647971153259,
+        -0.6623048782348633,
+        0.07444841414690018,
+        -2.0547866821289062
+      ],
+      [
+        0.2542734444141388,
+        1.404710054397583,
+        -0.16682158410549164,
+        0.16858866810798645,
+        -1.786678671836853,
+        1.1306204795837402,
+        0.246002197265625,
+        -0.18968123197555542,
+        1.3288171291351318,
+        -0.16666997969150543,
+        -0.24833203852176666,
+        1.486351490020752,
+        0.390087366104126,
+        0.2801700830459595,
+        -0.5608059167861938,
+        -2.174682855606079
+      ],
+      [
+        -0.399098664522171,
+        -0.3210754990577698,
+        -0.7301748394966125,
+        -0.17693063616752625,
+        -2.7776947021484375,
+        -1.339138388633728,
+        0.47883638739585876,
+        -0.14695921540260315,
+        0.11954823136329651,
+        0.5356278419494629,
+        -0.8720952868461609,
+        1.1138038635253906,
+        -0.6852717399597168,
+        -0.3124436140060425,
+        0.8931537866592407,
+        -1.3214232921600342
+      ],
+      [
+        -0.11504136025905609,
+        -0.2380775660276413,
+        -1.3998239040374756,
+        -0.0863225981593132,
+        -1.8701715469360352,
+        -1.2635517120361328,
+        0.2683332562446594,
+        0.3396027684211731,
+        -0.7382551431655884,
+        1.4094324111938477,
+        0.3622351884841919,
+        2.0395216941833496,
+        0.4131108224391937,
+        -0.08965935558080673,
+        0.6820758581161499,
+        -1.1024388074874878
+      ],
+      [
+        -0.9755199551582336,
+        -0.4071264863014221,
+        -0.5242950916290283,
+        -0.23340484499931335,
+        -0.3149244785308838,
+        -1.892197847366333,
+        0.7467259168624878,
+        0.5549261569976807,
+        -1.4197262525558472,
+        1.3532174825668335,
+        0.009386791847646236,
+        1.8732331991195679,
+        0.7576674222946167,
+        -0.7377344369888306,
+        1.2162482738494873,
+        -0.2337128221988678
+      ]
+    ]
+  ],
+  "padded_input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_position_ids": [
+    [
+      0,
+      1,
+      2,
+      3,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_hidden_state": [
+    [
+      [
+        1.0479530096054077,
+        0.777589738368988,
+        0.4487590491771698,
+        0.8083864450454712,
+        -0.46256136894226074,
+        0.9383790493011475,
+        0.05815654620528221,
+        -0.3330000638961792,
+        -0.23830658197402954,
+        -0.10689257085323334,
+        -1.7566187381744385,
+        1.4047011137008667,
+        1.5862897634506226,
+        -1.4487812519073486,
+        0.34458714723587036,
+        -1.5402408838272095
+      ],
+      [
+        1.106600046157837,
+        0.88736891746521,
+        0.23645684123039246,
+        0.6962629556655884,
+        -0.6025262475013733,
+        0.8922804594039917,
+        -0.08937154710292816,
+        -0.3328021466732025,
+        -0.30558285117149353,
+        0.0372898168861866,
+        -1.7399507761001587,
+        1.2930703163146973,
+        1.527563452720642,
+        -1.574615240097046,
+        0.49537399411201477,
+        -1.522128701210022
+      ],
+      [
+        0.7689082026481628,
+        1.2421519756317139,
+        -0.008993715979158878,
+        0.5178283452987671,
+        -1.0837748050689697,
+        0.5594359636306763,
+        -0.6521210074424744,
+        -0.2873961329460144,
+        -0.4735589623451233,
+        0.2963488698005676,
+        -1.317405343055725,
+        1.1891876459121704,
+        1.605422019958496,
+        -1.5092140436172485,
+        0.7168599367141724,
+        -1.6647543907165527
+      ],
+      [
+        0.682964563369751,
+        1.1864830255508423,
+        0.3529805839061737,
+        0.5533444881439209,
+        -1.5137571096420288,
+        0.9992331266403198,
+        0.3350704610347748,
+        -0.24685613811016083,
+        0.28254324197769165,
+        -0.5005261301994324,
+        -1.438392996788025,
+        1.6214579343795776,
+        0.7328647971153259,
+        -0.6623048782348633,
+        0.07444841414690018,
+        -2.0547866821289062
+      ],
+      [
+        0.5801084041595459,
+        1.537717342376709,
+        0.6600556969642639,
+        0.7457431554794312,
+        -2.172827959060669,
+        0.6514791250228882,
+        -0.17609137296676636,
+        -0.4080505073070526,
+        0.16530893743038177,
+        -0.28472116589546204,
+        -1.4394128322601318,
+        0.37917590141296387,
+        -0.41805875301361084,
+        -1.1754405498504639,
+        -0.005535643547773361,
+        -1.755967378616333
+      ],
+      [
+        0.4749983549118042,
+        1.6401233673095703,
+        0.7290105223655701,
+        0.7845529317855835,
+        -2.1131889820098877,
+        0.6954901218414307,
+        0.22973264753818512,
+        -0.9258361458778381,
+        0.3253975510597229,
+        -0.5409510731697083,
+        -0.8680384755134583,
+        -0.028916820883750916,
+        -0.9761810898780823,
+        -1.1065843105316162,
+        -0.24258629977703094,
+        -1.6398214101791382
+      ],
+      [
+        0.4833013713359833,
+        0.9171518683433533,
+        0.4412821829319,
+        0.9119624495506287,
+        -2.3757028579711914,
+        0.29025182127952576,
+        0.5364564061164856,
+        -1.0730129480361938,
+        0.21991688013076782,
+        -0.2867128252983093,
+        -0.7875986695289612,
+        0.15618547797203064,
+        -1.5771867036819458,
+        -0.832351803779602,
+        0.004696149844676256,
+        -1.6660926342010498
+      ],
+      [
+        0.6805307865142822,
+        1.4486762285232544,
+        0.6507560610771179,
+        0.8518795967102051,
+        -1.6908141374588013,
+        0.6718614101409912,
+        -0.12354743480682373,
+        -0.47582513093948364,
+        -0.03842397406697273,
+        -0.43928930163383484,
+        -1.6344605684280396,
+        0.8182473182678223,
+        0.28718098998069763,
+        -1.497409462928772,
+        0.21214571595191956,
+        -1.6828385591506958
+      ]
+    ]
+  ],
+  "compact_hidden_state": [
+    [
+      [
+        1.0479530096054077,
+        0.777589738368988,
+        0.4487590491771698,
+        0.8083864450454712,
+        -0.46256136894226074,
+        0.9383790493011475,
+        0.05815654620528221,
+        -0.3330000638961792,
+        -0.23830658197402954,
+        -0.10689257085323334,
+        -1.7566187381744385,
+        1.4047011137008667,
+        1.5862897634506226,
+        -1.4487812519073486,
+        0.34458714723587036,
+        -1.5402408838272095
+      ],
+      [
+        1.106600046157837,
+        0.8873687982559204,
+        0.2364569753408432,
+        0.6962629556655884,
+        -0.602526068687439,
+        0.8922803401947021,
+        -0.08937160670757294,
+        -0.3328019678592682,
+        -0.30558279156684875,
+        0.0372898168861866,
+        -1.7399507761001587,
+        1.2930703163146973,
+        1.527563452720642,
+        -1.5746151208877563,
+        0.49537393450737,
+        -1.5221285820007324
+      ],
+      [
+        0.7689083218574524,
+        1.2421518564224243,
+        -0.008993547409772873,
+        0.5178284645080566,
+        -1.0837746858596802,
+        0.5594360828399658,
+        -0.6521210670471191,
+        -0.287396103143692,
+        -0.4735589623451233,
+        0.2963491678237915,
+        -1.3174054622650146,
+        1.1891876459121704,
+        1.6054221391677856,
+        -1.5092140436172485,
+        0.7168601155281067,
+        -1.6647543907165527
+      ],
+      [
+        0.6829645037651062,
+        1.1864830255508423,
+        0.3529805541038513,
+        0.5533444285392761,
+        -1.5137568712234497,
+        0.9992331862449646,
+        0.3350703716278076,
+        -0.2468559890985489,
+        0.2825431227684021,
+        -0.5005262494087219,
+        -1.438393235206604,
+        1.6214576959609985,
+        0.7328648567199707,
+        -0.6623048782348633,
+        0.07444839179515839,
+        -2.054786443710327
+      ]
+    ]
+  ]
+}
diff --git a/test/fixtures/embedding_models/tiny-random-llama-bidirectional/config.json b/test/fixtures/embedding_models/tiny-random-llama-bidirectional/config.json
new file mode 100644
index 00000000..2f863bf2
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-llama-bidirectional/config.json
@@ -0,0 +1,33 @@
+{
+  "architectures": [
+    "LlamaModel"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "head_dim": 8,
+  "hidden_act": "silu",
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 20,
+  "is_causal": false,
+  "max_position_embeddings": 4,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.19.0.dev0",
+  "use_cache": true,
+  "vocab_size": 32
+}
diff --git a/test/fixtures/embedding_models/tiny-random-llama-bidirectional/model.safetensors b/test/fixtures/embedding_models/tiny-random-llama-bidirectional/model.safetensors
new file mode 100644
index 00000000..81b01741
Binary files /dev/null and b/test/fixtures/embedding_models/tiny-random-llama-bidirectional/model.safetensors differ
diff --git a/test/fixtures/embedding_models/tiny-random-llama-bidirectional/reference.json b/test/fixtures/embedding_models/tiny-random-llama-bidirectional/reference.json
new file mode 100644
index 00000000..af2a9446
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-llama-bidirectional/reference.json
@@ -0,0 +1,585 @@
+{
+  "transformers_version": "5.19.0.dev0",
+  "transformers_commit": "3693f8d26311305e914735a6373fb03468d6aaa0",
+  "is_causal": false,
+  "input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8
+    ]
+  ],
+  "attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1
+    ]
+  ],
+  "hidden_state": [
+    [
+      [
+        -0.4211755096912384,
+        0.5548998713493347,
+        -0.32655400037765503,
+        0.31106868386268616,
+        1.1800999641418457,
+        2.1925554275512695,
+        -0.3116692304611206,
+        -0.8538727760314941,
+        -0.30181387066841125,
+        0.954813539981842,
+        -1.996849775314331,
+        -0.26396048069000244,
+        0.5982969403266907,
+        -1.0644657611846924,
+        1.2957713603973389,
+        -0.0910843163728714
+      ],
+      [
+        1.1222102642059326,
+        0.5952421426773071,
+        -0.9055588841438293,
+        0.004667977802455425,
+        0.45469164848327637,
+        -2.1555263996124268,
+        -0.3586161732673645,
+        -1.6454957723617554,
+        -1.3811115026474,
+        0.3626633882522583,
+        -0.803831934928894,
+        -1.3610491752624512,
+        0.025878500193357468,
+        -1.0459760427474976,
+        0.14031626284122467,
+        0.4529903829097748
+      ],
+      [
+        -0.04494249075651169,
+        -1.1856218576431274,
+        0.5115281343460083,
+        0.13188545405864716,
+        -0.16881293058395386,
+        0.8704543709754944,
+        -1.425215721130371,
+        -0.828005850315094,
+        -0.5295071601867676,
+        0.24340778589248657,
+        -0.9303196668624878,
+        -0.155778169631958,
+        2.8642890453338623,
+        -0.8677822947502136,
+        -0.5800689458847046,
+        -0.489178866147995
+      ],
+      [
+        -1.2036911249160767,
+        0.15577217936515808,
+        -0.4191820025444031,
+        -0.12617067992687225,
+        -0.4126940071582794,
+        0.31229367852211,
+        0.9861965179443359,
+        -2.824021816253662,
+        0.30962464213371277,
+        -0.8869038820266724,
+        0.9801685810089111,
+        -1.4489519596099854,
+        -0.24949024617671967,
+        -0.22238801419734955,
+        0.5297660231590271,
+        0.8690586686134338
+      ],
+      [
+        -0.9739092588424683,
+        2.3802053928375244,
+        0.5561097264289856,
+        0.5726725459098816,
+        1.798372745513916,
+        0.3151684105396271,
+        0.3118177652359009,
+        -0.8357057571411133,
+        0.0026432136073708534,
+        -0.21922567486763,
+        0.900512158870697,
+        -0.8506640791893005,
+        1.093220591545105,
+        -0.15813696384429932,
+        0.5927533507347107,
+        -1.191626787185669
+      ],
+      [
+        -0.6473158597946167,
+        1.2015552520751953,
+        1.2441365718841553,
+        -0.05702007934451103,
+        0.2124946564435959,
+        -0.8172556757926941,
+        -0.7591071724891663,
+        -1.2266066074371338,
+        0.6607446670532227,
+        -0.012020907364785671,
+        1.6767390966415405,
+        -0.7594090700149536,
+        1.7541390657424927,
+        -1.594694972038269,
+        0.3785790503025055,
+        0.3901751935482025
+      ],
+      [
+        0.22390519082546234,
+        -1.622051477432251,
+        -1.0620826482772827,
+        -1.7454997301101685,
+        0.8828161358833313,
+        0.5936636924743652,
+        -1.9258983135223389,
+        0.334833562374115,
+        -0.27360230684280396,
+        -0.7059505581855774,
+        1.5033018589019775,
+        -0.6267736554145813,
+        0.21057212352752686,
+        -0.6889486908912659,
+        -0.10545934736728668,
+        0.6424998044967651
+      ],
+      [
+        0.6035855412483215,
+        -1.411379337310791,
+        -0.9199094176292419,
+        0.8431387543678284,
+        -0.6434103846549988,
+        -0.2350521683692932,
+        0.4356596767902374,
+        0.300529420375824,
+        0.46057572960853577,
+        1.682791829109192,
+        0.9992179274559021,
+        2.4860646724700928,
+        0.556033194065094,
+        0.5983346700668335,
+        -0.23859484493732452,
+        -0.5818508267402649
+      ]
+    ]
+  ],
+  "changed_future_hidden_state": [
+    [
+      [
+        -0.42947399616241455,
+        0.43172717094421387,
+        -0.2735430598258972,
+        0.2970542013645172,
+        1.0403318405151367,
+        2.36293363571167,
+        -0.3793114125728607,
+        -0.972045361995697,
+        -0.23958073556423187,
+        0.7965668439865112,
+        -2.016650676727295,
+        -0.13867203891277313,
+        0.6906659007072449,
+        -1.009495735168457,
+        1.1781905889511108,
+        -0.07582469284534454
+      ],
+      [
+        1.132727861404419,
+        0.5378140807151794,
+        -0.8984145522117615,
+        0.0028723347932100296,
+        0.39952510595321655,
+        -2.038817882537842,
+        -0.41926148533821106,
+        -1.790873408317566,
+        -1.3816356658935547,
+        0.2872554659843445,
+        -0.8943583965301514,
+        -1.3198875188827515,
+        0.10522528737783432,
+        -1.059455394744873,
+        0.09545683860778809,
+        0.4735938310623169
+      ],
+      [
+        -0.06501603871583939,
+        -1.212607502937317,
+        0.5213246941566467,
+        0.11713896691799164,
+        -0.25105053186416626,
+        1.0614131689071655,
+        -1.4094723463058472,
+        -0.9198651313781738,
+        -0.4507027566432953,
+        0.10688306391239166,
+        -0.9639285802841187,
+        -0.034188881516456604,
+        2.795624256134033,
+        -0.7964999079704285,
+        -0.6148715019226074,
+        -0.450050950050354
+      ],
+      [
+        -1.2034492492675781,
+        0.06867562979459763,
+        -0.3760203719139099,
+        -0.12626975774765015,
+        -0.4874389171600342,
+        0.5088486075401306,
+        0.9044831395149231,
+        -2.8960206508636475,
+        0.3439863622188568,
+        -0.9713032841682434,
+        0.8910763263702393,
+        -1.333152174949646,
+        -0.15472346544265747,
+        -0.20617303252220154,
+        0.46507903933525085,
+        0.8624621629714966
+      ],
+      [
+        -1.0118635892868042,
+        2.3035895824432373,
+        0.6088472008705139,
+        0.5765013694763184,
+        1.7175211906433105,
+        0.5782266855239868,
+        0.2372286170721054,
+        -0.9999766945838928,
+        0.04944703355431557,
+        -0.3460693359375,
+        0.8179343938827515,
+        -0.7452720999717712,
+        1.2233858108520508,
+        -0.14679662883281708,
+        0.5307306051254272,
+        -1.1968902349472046
+      ],
+      [
+        -0.6734292507171631,
+        1.0857077836990356,
+        1.2901209592819214,
+        -0.06111652031540871,
+        0.1044633612036705,
+        -0.5439350008964539,
+        -0.8436628580093384,
+        -1.385372519493103,
+        0.7128816246986389,
+        -0.14404186606407166,
+        1.5762749910354614,
+        -0.6374025344848633,
+        1.8790632486343384,
+        -1.5828357934951782,
+        0.3023335337638855,
+        0.4030734598636627
+      ],
+      [
+        0.2027004212141037,
+        -1.6803561449050903,
+        -1.0181939601898193,
+        -1.7309985160827637,
+        0.805685818195343,
+        0.7622591853141785,
+        -1.9597986936569214,
+        0.2278299480676651,
+        -0.2356199324131012,
+        -0.7866905927658081,
+        1.4220541715621948,
+        -0.5391963124275208,
+        0.287156879901886,
+        -0.668106198310852,
+        -0.15564242005348206,
+        0.6459328532218933
+      ],
+      [
+        0.09859059751033783,
+        -0.09219228476285934,
+        0.27842870354652405,
+        -1.4713364839553833,
+        -0.7220364212989807,
+        0.6079506278038025,
+        1.8499979972839355,
+        0.2777754068374634,
+        -0.2147877812385559,
+        -0.11358802020549774,
+        -0.9108771085739136,
+        -0.4770866334438324,
+        0.641248881816864,
+        -2.710298776626587,
+        -0.5533497929573059,
+        -0.34503594040870667
+      ]
+    ]
+  ],
+  "padded_input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_position_ids": [
+    [
+      0,
+      1,
+      2,
+      3,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_hidden_state": [
+    [
+      [
+        -0.30116498470306396,
+        0.40945062041282654,
+        -0.4931391179561615,
+        0.5643168687820435,
+        1.0892226696014404,
+        2.2711758613586426,
+        -0.2578393220901489,
+        -0.9440498948097229,
+        -0.1021723821759224,
+        0.9637549519538879,
+        -1.9057464599609375,
+        -0.09663661569356918,
+        0.7154596447944641,
+        -1.0952165126800537,
+        1.2364166975021362,
+        -0.1232721209526062
+      ],
+      [
+        1.2027949094772339,
+        0.5185537338256836,
+        -1.0432798862457275,
+        0.1949395388364792,
+        0.44129642844200134,
+        -2.0412402153015137,
+        -0.3337728977203369,
+        -1.7583130598068237,
+        -1.2709041833877563,
+        0.4101327955722809,
+        -0.8307012319564819,
+        -1.2750619649887085,
+        0.12927193939685822,
+        -1.1164424419403076,
+        0.14794181287288666,
+        0.4348050355911255
+      ],
+      [
+        0.061388321220874786,
+        -1.2344837188720703,
+        0.3010878562927246,
+        0.39913731813430786,
+        -0.18402475118637085,
+        1.024018406867981,
+        -1.308164358139038,
+        -0.9156467318534851,
+        -0.32643741369247437,
+        0.2969891428947449,
+        -0.901678204536438,
+        -0.001372458995319903,
+        2.8590710163116455,
+        -0.9099182486534119,
+        -0.5387665629386902,
+        -0.4971800446510315
+      ],
+      [
+        -1.1006641387939453,
+        0.058856263756752014,
+        -0.5697701573371887,
+        0.10045470297336578,
+        -0.43231871724128723,
+        0.4711735248565674,
+        1.0031242370605469,
+        -2.904845952987671,
+        0.4509181082248688,
+        -0.8248450756072998,
+        0.9472010135650635,
+        -1.3119609355926514,
+        -0.1213662400841713,
+        -0.29418420791625977,
+        0.531833291053772,
+        0.8259038925170898
+      ],
+      [
+        0.6116460561752319,
+        -0.5559471845626831,
+        -1.341841697692871,
+        1.3086016178131104,
+        -0.2075679898262024,
+        1.1435858011245728,
+        -0.23461145162582397,
+        -1.8223212957382202,
+        0.9442780613899231,
+        0.9182639122009277,
+        -0.4085400104522705,
+        0.5404786467552185,
+        1.9527760744094849,
+        -0.5693192481994629,
+        0.1417427510023117,
+        -0.4519355595111847
+      ],
+      [
+        0.6116460561752319,
+        -0.5559471845626831,
+        -1.341841697692871,
+        1.3086016178131104,
+        -0.2075679898262024,
+        1.1435858011245728,
+        -0.23461145162582397,
+        -1.8223212957382202,
+        0.9442780613899231,
+        0.9182639122009277,
+        -0.4085400104522705,
+        0.5404786467552185,
+        1.9527760744094849,
+        -0.5693192481994629,
+        0.1417427510023117,
+        -0.4519355595111847
+      ],
+      [
+        0.6116460561752319,
+        -0.5559471845626831,
+        -1.341841697692871,
+        1.3086016178131104,
+        -0.2075679898262024,
+        1.1435858011245728,
+        -0.23461145162582397,
+        -1.8223212957382202,
+        0.9442780613899231,
+        0.9182639122009277,
+        -0.4085400104522705,
+        0.5404786467552185,
+        1.9527760744094849,
+        -0.5693192481994629,
+        0.1417427510023117,
+        -0.4519355595111847
+      ],
+      [
+        0.6116460561752319,
+        -0.5559471845626831,
+        -1.341841697692871,
+        1.3086016178131104,
+        -0.2075679898262024,
+        1.1435858011245728,
+        -0.23461145162582397,
+        -1.8223212957382202,
+        0.9442780613899231,
+        0.9182639122009277,
+        -0.4085400104522705,
+        0.5404786467552185,
+        1.9527760744094849,
+        -0.5693192481994629,
+        0.1417427510023117,
+        -0.4519355595111847
+      ]
+    ]
+  ],
+  "compact_hidden_state": [
+    [
+      [
+        -0.30116498470306396,
+        0.40945062041282654,
+        -0.4931391179561615,
+        0.5643168687820435,
+        1.0892226696014404,
+        2.2711758613586426,
+        -0.2578393220901489,
+        -0.9440498948097229,
+        -0.1021723821759224,
+        0.9637549519538879,
+        -1.9057464599609375,
+        -0.09663661569356918,
+        0.7154596447944641,
+        -1.0952165126800537,
+        1.2364166975021362,
+        -0.1232721209526062
+      ],
+      [
+        1.2027949094772339,
+        0.5185537338256836,
+        -1.0432798862457275,
+        0.1949395388364792,
+        0.44129642844200134,
+        -2.0412402153015137,
+        -0.3337728977203369,
+        -1.7583130598068237,
+        -1.2709041833877563,
+        0.4101327955722809,
+        -0.8307012319564819,
+        -1.2750619649887085,
+        0.12927193939685822,
+        -1.1164424419403076,
+        0.14794181287288666,
+        0.4348050355911255
+      ],
+      [
+        0.061388321220874786,
+        -1.2344837188720703,
+        0.3010878562927246,
+        0.39913731813430786,
+        -0.18402475118637085,
+        1.024018406867981,
+        -1.308164358139038,
+        -0.9156467318534851,
+        -0.32643741369247437,
+        0.2969891428947449,
+        -0.901678204536438,
+        -0.001372458995319903,
+        2.8590710163116455,
+        -0.9099182486534119,
+        -0.5387665629386902,
+        -0.4971800446510315
+      ],
+      [
+        -1.1006641387939453,
+        0.058856263756752014,
+        -0.5697701573371887,
+        0.10045470297336578,
+        -0.43231871724128723,
+        0.4711735248565674,
+        1.0031242370605469,
+        -2.904845952987671,
+        0.4509181082248688,
+        -0.8248450756072998,
+        0.9472010135650635,
+        -1.3119609355926514,
+        -0.1213662400841713,
+        -0.29418420791625977,
+        0.531833291053772,
+        0.8259038925170898
+      ]
+    ]
+  ]
+}
diff --git a/test/fixtures/embedding_models/tiny-random-llama-causal/config.json b/test/fixtures/embedding_models/tiny-random-llama-causal/config.json
new file mode 100644
index 00000000..4d10f356
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-llama-causal/config.json
@@ -0,0 +1,33 @@
+{
+  "architectures": [
+    "LlamaModel"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "head_dim": 8,
+  "hidden_act": "silu",
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 20,
+  "is_causal": true,
+  "max_position_embeddings": 4,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.19.0.dev0",
+  "use_cache": true,
+  "vocab_size": 32
+}
diff --git a/test/fixtures/embedding_models/tiny-random-llama-causal/model.safetensors b/test/fixtures/embedding_models/tiny-random-llama-causal/model.safetensors
new file mode 100644
index 00000000..14b61863
Binary files /dev/null and b/test/fixtures/embedding_models/tiny-random-llama-causal/model.safetensors differ
diff --git a/test/fixtures/embedding_models/tiny-random-llama-causal/reference.json b/test/fixtures/embedding_models/tiny-random-llama-causal/reference.json
new file mode 100644
index 00000000..0d7cb8a8
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-llama-causal/reference.json
@@ -0,0 +1,585 @@
+{
+  "transformers_version": "5.19.0.dev0",
+  "transformers_commit": "3693f8d26311305e914735a6373fb03468d6aaa0",
+  "is_causal": true,
+  "input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8
+    ]
+  ],
+  "attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1
+    ]
+  ],
+  "hidden_state": [
+    [
+      [
+        -1.5929151773452759,
+        -0.6003653407096863,
+        -0.5094078779220581,
+        -0.962164044380188,
+        -0.31567907333374023,
+        0.630582869052887,
+        -1.6334172487258911,
+        1.6027878522872925,
+        0.23477952182292938,
+        -0.8364869952201843,
+        0.15759843587875366,
+        0.7943824529647827,
+        -0.2593148350715637,
+        0.10758619755506516,
+        -0.7669921517372131,
+        2.0198476314544678
+      ],
+      [
+        0.6898882985115051,
+        -1.389062523841858,
+        -1.0428372621536255,
+        -0.7545837759971619,
+        -0.6884869933128357,
+        1.1694477796554565,
+        -0.8826923370361328,
+        -0.11817003786563873,
+        1.098335862159729,
+        0.7683865427970886,
+        -0.7247151732444763,
+        0.6575729846954346,
+        1.129040002822876,
+        1.5497493743896484,
+        0.1274891495704651,
+        1.681316614151001
+      ],
+      [
+        0.3429621756076813,
+        1.7419458627700806,
+        -0.753235936164856,
+        -0.4981689453125,
+        -0.9175847172737122,
+        0.5624348521232605,
+        -1.3638721704483032,
+        0.9257065653800964,
+        1.6528384685516357,
+        -0.3569645881652832,
+        -1.5816876888275146,
+        0.03878334164619446,
+        -0.8804912567138672,
+        1.0846880674362183,
+        -0.3427111506462097,
+        0.8187417387962341
+      ],
+      [
+        -0.44830402731895447,
+        -0.21460671722888947,
+        0.12261118739843369,
+        -0.43910181522369385,
+        -1.4307810068130493,
+        -0.08696458488702774,
+        0.27569085359573364,
+        0.3632393479347229,
+        0.33360517024993896,
+        -1.5959644317626953,
+        0.3499789834022522,
+        2.512471914291382,
+        1.274120569229126,
+        1.333441972732544,
+        0.22759853303432465,
+        0.8352688550949097
+      ],
+      [
+        -0.37356293201446533,
+        0.16278886795043945,
+        -0.9381667375564575,
+        0.32777032256126404,
+        -1.333550214767456,
+        0.5953577756881714,
+        0.29960277676582336,
+        -1.2891823053359985,
+        -0.004067094996571541,
+        1.309119701385498,
+        -0.9793269038200378,
+        0.5088264346122742,
+        -1.743228554725647,
+        0.5554646253585815,
+        -0.9512976408004761,
+        1.937324047088623
+      ],
+      [
+        0.9740347266197205,
+        -0.020724942907691002,
+        0.0052393702790141106,
+        1.4428623914718628,
+        -0.9838392734527588,
+        -1.3757606744766235,
+        -1.9937610626220703,
+        -1.9541552066802979,
+        0.011807242408394814,
+        -0.8277478218078613,
+        -0.2320166528224945,
+        0.4832310974597931,
+        -0.8893502354621887,
+        -0.1255970150232315,
+        0.5280473828315735,
+        -0.48376184701919556
+      ],
+      [
+        -0.5678181052207947,
+        2.4606494903564453,
+        -0.29888927936553955,
+        -0.05167769640684128,
+        -0.929226279258728,
+        -0.966411828994751,
+        -0.23025651276111603,
+        0.8395761847496033,
+        0.4954826533794403,
+        0.385222464799881,
+        -0.8163310885429382,
+        0.3191235363483429,
+        0.9865652918815613,
+        1.9905973672866821,
+        -0.7614292502403259,
+        0.5184648633003235
+      ],
+      [
+        -1.1133098602294922,
+        0.7935789227485657,
+        -0.24836866557598114,
+        -0.7486827373504639,
+        0.8966212272644043,
+        1.3970950841903687,
+        0.5570847988128662,
+        -1.355514407157898,
+        -0.037265222519636154,
+        1.4263502359390259,
+        -1.3563772439956665,
+        -0.13238456845283508,
+        -0.6878980398178101,
+        -0.2922036051750183,
+        -1.9965641498565674,
+        -0.36700403690338135
+      ]
+    ]
+  ],
+  "changed_future_hidden_state": [
+    [
+      [
+        -1.5929151773452759,
+        -0.6003653407096863,
+        -0.5094078779220581,
+        -0.962164044380188,
+        -0.31567907333374023,
+        0.630582869052887,
+        -1.6334172487258911,
+        1.6027878522872925,
+        0.23477952182292938,
+        -0.8364869952201843,
+        0.15759843587875366,
+        0.7943824529647827,
+        -0.2593148350715637,
+        0.10758619755506516,
+        -0.7669921517372131,
+        2.0198476314544678
+      ],
+      [
+        0.6898882985115051,
+        -1.389062523841858,
+        -1.0428372621536255,
+        -0.7545837759971619,
+        -0.6884869933128357,
+        1.1694477796554565,
+        -0.8826923370361328,
+        -0.11817003786563873,
+        1.098335862159729,
+        0.7683865427970886,
+        -0.7247151732444763,
+        0.6575729846954346,
+        1.129040002822876,
+        1.5497493743896484,
+        0.1274891495704651,
+        1.681316614151001
+      ],
+      [
+        0.3429621756076813,
+        1.7419458627700806,
+        -0.753235936164856,
+        -0.4981689453125,
+        -0.9175847172737122,
+        0.5624348521232605,
+        -1.3638721704483032,
+        0.9257065653800964,
+        1.6528384685516357,
+        -0.3569645881652832,
+        -1.5816876888275146,
+        0.03878334164619446,
+        -0.8804912567138672,
+        1.0846880674362183,
+        -0.3427111506462097,
+        0.8187417387962341
+      ],
+      [
+        -0.44830402731895447,
+        -0.21460671722888947,
+        0.12261118739843369,
+        -0.43910181522369385,
+        -1.4307810068130493,
+        -0.08696458488702774,
+        0.27569085359573364,
+        0.3632393479347229,
+        0.33360517024993896,
+        -1.5959644317626953,
+        0.3499789834022522,
+        2.512471914291382,
+        1.274120569229126,
+        1.333441972732544,
+        0.22759853303432465,
+        0.8352688550949097
+      ],
+      [
+        -0.37356293201446533,
+        0.16278886795043945,
+        -0.9381667375564575,
+        0.32777032256126404,
+        -1.333550214767456,
+        0.5953577756881714,
+        0.29960277676582336,
+        -1.2891823053359985,
+        -0.004067094996571541,
+        1.309119701385498,
+        -0.9793269038200378,
+        0.5088264346122742,
+        -1.743228554725647,
+        0.5554646253585815,
+        -0.9512976408004761,
+        1.937324047088623
+      ],
+      [
+        0.9740347266197205,
+        -0.020724942907691002,
+        0.0052393702790141106,
+        1.4428623914718628,
+        -0.9838392734527588,
+        -1.3757606744766235,
+        -1.9937610626220703,
+        -1.9541552066802979,
+        0.011807242408394814,
+        -0.8277478218078613,
+        -0.2320166528224945,
+        0.4832310974597931,
+        -0.8893502354621887,
+        -0.1255970150232315,
+        0.5280473828315735,
+        -0.48376184701919556
+      ],
+      [
+        -0.5678181052207947,
+        2.4606494903564453,
+        -0.29888927936553955,
+        -0.05167769640684128,
+        -0.929226279258728,
+        -0.966411828994751,
+        -0.23025651276111603,
+        0.8395761847496033,
+        0.4954826533794403,
+        0.385222464799881,
+        -0.8163310885429382,
+        0.3191235363483429,
+        0.9865652918815613,
+        1.9905973672866821,
+        -0.7614292502403259,
+        0.5184648633003235
+      ],
+      [
+        -0.1458314061164856,
+        2.203958034515381,
+        0.3634493947029114,
+        -0.3175545632839203,
+        0.4547146260738373,
+        -0.8177770376205444,
+        -0.8347190618515015,
+        0.25345662236213684,
+        -0.259469598531723,
+        -1.5115933418273926,
+        0.21156080067157745,
+        1.0331226587295532,
+        0.6968410611152649,
+        1.5568182468414307,
+        1.6687116622924805,
+        -0.2433205097913742
+      ]
+    ]
+  ],
+  "padded_input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_position_ids": [
+    [
+      0,
+      1,
+      2,
+      3,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_hidden_state": [
+    [
+      [
+        -1.5929151773452759,
+        -0.6003653407096863,
+        -0.5094078779220581,
+        -0.962164044380188,
+        -0.31567907333374023,
+        0.630582869052887,
+        -1.6334172487258911,
+        1.6027878522872925,
+        0.23477952182292938,
+        -0.8364869952201843,
+        0.15759843587875366,
+        0.7943824529647827,
+        -0.2593148350715637,
+        0.10758619755506516,
+        -0.7669921517372131,
+        2.0198476314544678
+      ],
+      [
+        0.6898882985115051,
+        -1.389062523841858,
+        -1.0428372621536255,
+        -0.7545837759971619,
+        -0.6884869933128357,
+        1.1694477796554565,
+        -0.8826923370361328,
+        -0.11817003786563873,
+        1.098335862159729,
+        0.7683865427970886,
+        -0.7247151732444763,
+        0.6575729846954346,
+        1.129040002822876,
+        1.5497493743896484,
+        0.1274891495704651,
+        1.681316614151001
+      ],
+      [
+        0.3429621756076813,
+        1.7419458627700806,
+        -0.753235936164856,
+        -0.4981689453125,
+        -0.9175847172737122,
+        0.5624348521232605,
+        -1.3638721704483032,
+        0.9257065653800964,
+        1.6528384685516357,
+        -0.3569645881652832,
+        -1.5816876888275146,
+        0.03878334164619446,
+        -0.8804912567138672,
+        1.0846880674362183,
+        -0.3427111506462097,
+        0.8187417387962341
+      ],
+      [
+        -0.44830402731895447,
+        -0.21460671722888947,
+        0.12261118739843369,
+        -0.43910181522369385,
+        -1.4307810068130493,
+        -0.08696458488702774,
+        0.27569085359573364,
+        0.3632393479347229,
+        0.33360517024993896,
+        -1.5959644317626953,
+        0.3499789834022522,
+        2.512471914291382,
+        1.274120569229126,
+        1.333441972732544,
+        0.22759853303432465,
+        0.8352688550949097
+      ],
+      [
+        0.44188886880874634,
+        0.5267016887664795,
+        0.7889223098754883,
+        -1.126582145690918,
+        -0.9818824529647827,
+        0.6240435838699341,
+        -1.705146074295044,
+        -0.6350654363632202,
+        2.041469097137451,
+        -0.3892385959625244,
+        -0.29268360137939453,
+        0.5754352807998657,
+        0.6632300019264221,
+        0.5014443397521973,
+        -1.6303359270095825,
+        0.7556922435760498
+      ],
+      [
+        0.44188886880874634,
+        0.5267016887664795,
+        0.7889223098754883,
+        -1.126582145690918,
+        -0.9818824529647827,
+        0.6240435838699341,
+        -1.705146074295044,
+        -0.6350654363632202,
+        2.041469097137451,
+        -0.3892385959625244,
+        -0.29268360137939453,
+        0.5754352807998657,
+        0.6632300019264221,
+        0.5014443397521973,
+        -1.6303359270095825,
+        0.7556922435760498
+      ],
+      [
+        0.44188886880874634,
+        0.5267016887664795,
+        0.7889223098754883,
+        -1.126582145690918,
+        -0.9818824529647827,
+        0.6240435838699341,
+        -1.705146074295044,
+        -0.6350654363632202,
+        2.041469097137451,
+        -0.3892385959625244,
+        -0.29268360137939453,
+        0.5754352807998657,
+        0.6632300019264221,
+        0.5014443397521973,
+        -1.6303359270095825,
+        0.7556922435760498
+      ],
+      [
+        0.44188886880874634,
+        0.5267016887664795,
+        0.7889223098754883,
+        -1.126582145690918,
+        -0.9818824529647827,
+        0.6240435838699341,
+        -1.705146074295044,
+        -0.6350654363632202,
+        2.041469097137451,
+        -0.3892385959625244,
+        -0.29268360137939453,
+        0.5754352807998657,
+        0.6632300019264221,
+        0.5014443397521973,
+        -1.6303359270095825,
+        0.7556922435760498
+      ]
+    ]
+  ],
+  "compact_hidden_state": [
+    [
+      [
+        -1.5929151773452759,
+        -0.6003653407096863,
+        -0.5094078779220581,
+        -0.962164044380188,
+        -0.31567907333374023,
+        0.630582869052887,
+        -1.6334172487258911,
+        1.6027878522872925,
+        0.23477952182292938,
+        -0.8364869952201843,
+        0.15759843587875366,
+        0.7943824529647827,
+        -0.2593148350715637,
+        0.10758619755506516,
+        -0.7669921517372131,
+        2.0198476314544678
+      ],
+      [
+        0.6898882985115051,
+        -1.389062523841858,
+        -1.0428372621536255,
+        -0.7545837759971619,
+        -0.6884869933128357,
+        1.1694477796554565,
+        -0.8826923370361328,
+        -0.11817003786563873,
+        1.098335862159729,
+        0.7683865427970886,
+        -0.7247151732444763,
+        0.6575729846954346,
+        1.129040002822876,
+        1.5497493743896484,
+        0.1274891495704651,
+        1.681316614151001
+      ],
+      [
+        0.3429621756076813,
+        1.7419458627700806,
+        -0.753235936164856,
+        -0.4981689453125,
+        -0.9175847172737122,
+        0.5624348521232605,
+        -1.3638721704483032,
+        0.9257065653800964,
+        1.6528384685516357,
+        -0.3569645881652832,
+        -1.5816876888275146,
+        0.03878334164619446,
+        -0.8804912567138672,
+        1.0846880674362183,
+        -0.3427111506462097,
+        0.8187417387962341
+      ],
+      [
+        -0.44830402731895447,
+        -0.21460671722888947,
+        0.12261118739843369,
+        -0.43910181522369385,
+        -1.4307810068130493,
+        -0.08696458488702774,
+        0.27569085359573364,
+        0.3632393479347229,
+        0.33360517024993896,
+        -1.5959644317626953,
+        0.3499789834022522,
+        2.512471914291382,
+        1.274120569229126,
+        1.333441972732544,
+        0.22759853303432465,
+        0.8352688550949097
+      ]
+    ]
+  ]
+}
diff --git a/test/fixtures/embedding_models/tiny-random-mistral-bidirectional/config.json b/test/fixtures/embedding_models/tiny-random-mistral-bidirectional/config.json
new file mode 100644
index 00000000..a5325087
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-mistral-bidirectional/config.json
@@ -0,0 +1,31 @@
+{
+  "architectures": [
+    "MistralModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "head_dim": 8,
+  "hidden_act": "silu",
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 20,
+  "is_causal": false,
+  "max_position_embeddings": 4,
+  "model_type": "mistral",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": 4,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.19.0.dev0",
+  "use_cache": true,
+  "vocab_size": 32
+}
diff --git a/test/fixtures/embedding_models/tiny-random-mistral-bidirectional/model.safetensors b/test/fixtures/embedding_models/tiny-random-mistral-bidirectional/model.safetensors
new file mode 100644
index 00000000..b503051b
Binary files /dev/null and b/test/fixtures/embedding_models/tiny-random-mistral-bidirectional/model.safetensors differ
diff --git a/test/fixtures/embedding_models/tiny-random-mistral-bidirectional/reference.json b/test/fixtures/embedding_models/tiny-random-mistral-bidirectional/reference.json
new file mode 100644
index 00000000..1b8ade08
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-mistral-bidirectional/reference.json
@@ -0,0 +1,585 @@
+{
+  "transformers_version": "5.19.0.dev0",
+  "transformers_commit": "3693f8d26311305e914735a6373fb03468d6aaa0",
+  "is_causal": false,
+  "input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8
+    ]
+  ],
+  "attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1
+    ]
+  ],
+  "hidden_state": [
+    [
+      [
+        -2.0247020721435547,
+        -0.012201372534036636,
+        1.7035696506500244,
+        0.7370375990867615,
+        -0.5783039927482605,
+        0.1447124183177948,
+        0.8927332758903503,
+        -1.554565668106079,
+        -0.6524656414985657,
+        1.125311017036438,
+        1.5565158128738403,
+        -0.07736732810735703,
+        0.17577055096626282,
+        -0.4254700839519501,
+        0.1328285038471222,
+        0.7147100567817688
+      ],
+      [
+        -1.5079776048660278,
+        -2.021573066711426,
+        0.7333519458770752,
+        -0.6778014898300171,
+        1.222817301750183,
+        -0.4644806683063507,
+        0.8939060568809509,
+        -1.1685874462127686,
+        0.6928260922431946,
+        -0.6972037553787231,
+        0.6837774515151978,
+        -0.6337920427322388,
+        1.283979058265686,
+        0.8037490248680115,
+        0.4375670850276947,
+        0.6400811672210693
+      ],
+      [
+        0.7522222995758057,
+        -2.4407713413238525,
+        -0.6910439133644104,
+        0.06596823781728745,
+        -0.4407411813735962,
+        -1.1143351793289185,
+        0.40500980615615845,
+        1.4900282621383667,
+        -1.0456336736679077,
+        0.1007714569568634,
+        -0.9451706409454346,
+        -1.172706127166748,
+        0.7552176713943481,
+        0.8097615838050842,
+        -0.22225351631641388,
+        0.6885299682617188
+      ],
+      [
+        0.6646749377250671,
+        -0.6112914085388184,
+        0.5806158185005188,
+        0.2580931782722473,
+        -1.8095028400421143,
+        -0.8893306851387024,
+        0.02545849420130253,
+        -0.02969667688012123,
+        1.3413195610046387,
+        0.2543677091598511,
+        0.19161836802959442,
+        0.28271499276161194,
+        2.297266960144043,
+        1.3095048666000366,
+        -1.2959986925125122,
+        -0.1208648830652237
+      ],
+      [
+        0.7496078014373779,
+        -2.2097177505493164,
+        -0.7815321683883667,
+        0.4869077801704407,
+        -0.10121357440948486,
+        0.2663804590702057,
+        -1.069189190864563,
+        -0.5793210864067078,
+        0.8195260167121887,
+        -0.9879010915756226,
+        1.2082843780517578,
+        -1.1771281957626343,
+        -0.6697749495506287,
+        1.5572588443756104,
+        -0.7321873903274536,
+        0.45191946625709534
+      ],
+      [
+        -0.15355224907398224,
+        -1.1929396390914917,
+        0.20983542501926422,
+        -0.8824790716171265,
+        1.554892659187317,
+        -0.728230893611908,
+        1.9105623960494995,
+        -0.5609907507896423,
+        -1.7941912412643433,
+        1.1221245527267456,
+        -0.6664549708366394,
+        -0.9410739541053772,
+        -0.6750085353851318,
+        0.22097043693065643,
+        -0.6755238175392151,
+        0.12003744393587112
+      ],
+      [
+        -0.8482521772384644,
+        -0.062093254178762436,
+        0.727107048034668,
+        2.3487868309020996,
+        -1.2736424207687378,
+        -0.32234397530555725,
+        -0.1389290988445282,
+        -0.22715575993061066,
+        0.7971732020378113,
+        -0.8311808705329895,
+        -0.6845999360084534,
+        1.0399690866470337,
+        1.7368683815002441,
+        -0.7837607860565186,
+        0.5346148610115051,
+        0.7268665432929993
+      ],
+      [
+        0.267392635345459,
+        0.0448216013610363,
+        0.07543287426233292,
+        1.5457935333251953,
+        0.4062495529651642,
+        -0.1640452891588211,
+        -0.8455737829208374,
+        -1.2664391994476318,
+        0.9405146241188049,
+        -0.524522066116333,
+        -0.1804894506931305,
+        -2.6236255168914795,
+        -0.05992063879966736,
+        -1.1540672779083252,
+        1.1948506832122803,
+        0.36509034037590027
+      ]
+    ]
+  ],
+  "changed_future_hidden_state": [
+    [
+      [
+        -2.0353455543518066,
+        -0.009777919389307499,
+        1.7037909030914307,
+        0.7331627011299133,
+        -0.5816856026649475,
+        0.1446327120065689,
+        0.8884427547454834,
+        -1.5551533699035645,
+        -0.6485012769699097,
+        1.1244816780090332,
+        1.5519068241119385,
+        -0.07203176617622375,
+        0.17506134510040283,
+        -0.4227631986141205,
+        0.1347687989473343,
+        0.706200897693634
+      ],
+      [
+        -1.5273430347442627,
+        -2.0181384086608887,
+        0.7364054322242737,
+        -0.6806715130805969,
+        1.216414213180542,
+        -0.4635556936264038,
+        0.886526346206665,
+        -1.170629620552063,
+        0.6995099186897278,
+        -0.6974316835403442,
+        0.6749190092086792,
+        -0.6280335187911987,
+        1.2805250883102417,
+        0.8086708784103394,
+        0.4403485357761383,
+        0.6240567564964294
+      ],
+      [
+        0.7175267934799194,
+        -2.4465043544769287,
+        -0.6843821406364441,
+        0.06272272020578384,
+        -0.45572224259376526,
+        -1.1230920553207397,
+        0.3993740975856781,
+        1.4922150373458862,
+        -1.0365406274795532,
+        0.10133982449769974,
+        -0.968696653842926,
+        -1.1672077178955078,
+        0.7489297389984131,
+        0.825299084186554,
+        -0.21780666708946228,
+        0.6662425398826599
+      ],
+      [
+        0.628162682056427,
+        -0.6684461236000061,
+        0.5838244557380676,
+        0.21614059805870056,
+        -1.7899967432022095,
+        -0.8198503851890564,
+        -0.21753622591495514,
+        -0.021028436720371246,
+        1.5345687866210938,
+        0.1504795253276825,
+        0.20272524654865265,
+        0.3116346597671509,
+        2.219223737716675,
+        1.364264726638794,
+        -1.2181761264801025,
+        -0.1361822932958603
+      ],
+      [
+        0.7046382427215576,
+        -2.1784708499908447,
+        -0.7131818532943726,
+        0.4324449896812439,
+        -0.16237497329711914,
+        0.27515873312950134,
+        -1.2362092733383179,
+        -0.5422430038452148,
+        1.0155054330825806,
+        -1.0187900066375732,
+        1.1623311042785645,
+        -1.0775691270828247,
+        -0.6002969741821289,
+        1.590712308883667,
+        -0.6781668066978455,
+        0.4117414951324463
+      ],
+      [
+        -0.17539867758750916,
+        -1.370245337486267,
+        0.2588801085948944,
+        -0.9761856198310852,
+        1.5842220783233643,
+        -0.753889262676239,
+        1.7800787687301636,
+        -0.5911240577697754,
+        -1.6404273509979248,
+        1.1025878190994263,
+        -0.6935837864875793,
+        -0.9677101373672485,
+        -0.6832735538482666,
+        0.37043076753616333,
+        -0.7044968605041504,
+        0.10762422531843185
+      ],
+      [
+        -0.7835953831672668,
+        -0.2401110827922821,
+        0.7416219115257263,
+        2.136293888092041,
+        -1.3266798257827759,
+        -0.23845501244068146,
+        -0.6135538816452026,
+        -0.199204221367836,
+        1.2656277418136597,
+        -0.9655046463012695,
+        -0.6081408262252808,
+        1.0705150365829468,
+        1.7072529792785645,
+        -0.48098939657211304,
+        0.5165702700614929,
+        0.6487212181091309
+      ],
+      [
+        -1.8421461582183838,
+        1.15996515750885,
+        0.6344616413116455,
+        -0.8227241635322571,
+        0.9418267607688904,
+        -1.3762634992599487,
+        0.13901816308498383,
+        -0.8082731366157532,
+        0.34216225147247314,
+        -0.1821904480457306,
+        -0.3073679208755493,
+        -0.027936596423387527,
+        1.023556113243103,
+        2.2654120922088623,
+        0.4819644093513489,
+        0.1678839474916458
+      ]
+    ]
+  ],
+  "padded_input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_position_ids": [
+    [
+      0,
+      1,
+      2,
+      3,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_hidden_state": [
+    [
+      [
+        -2.024979591369629,
+        0.00422435300424695,
+        1.6957898139953613,
+        0.7697137594223022,
+        -0.6883586645126343,
+        0.11442556977272034,
+        0.7328180074691772,
+        -1.5748454332351685,
+        -0.5792887806892395,
+        1.0610051155090332,
+        1.6441532373428345,
+        -0.06880106776952744,
+        0.17206580936908722,
+        -0.3806779682636261,
+        0.17469821870326996,
+        0.7253968119621277
+      ],
+      [
+        -1.574650526046753,
+        -1.9975262880325317,
+        0.6534299254417419,
+        -0.642878532409668,
+        1.156783938407898,
+        -0.6467214822769165,
+        0.7542971968650818,
+        -1.1479636430740356,
+        0.8172124028205872,
+        -0.7986409068107605,
+        0.671444296836853,
+        -0.5100952982902527,
+        1.3076112270355225,
+        0.8520255088806152,
+        0.43006426095962524,
+        0.6130269765853882
+      ],
+      [
+        0.5974754691123962,
+        -2.4806430339813232,
+        -0.6777498722076416,
+        0.07211479544639587,
+        -0.5010150074958801,
+        -1.263437032699585,
+        0.21863402426242828,
+        1.4749866724014282,
+        -0.8187174797058105,
+        -0.030426712706685066,
+        -0.9761536121368408,
+        -1.1424365043640137,
+        0.7796595096588135,
+        0.9137669801712036,
+        -0.23002482950687408,
+        0.6912825703620911
+      ],
+      [
+        0.5695361495018005,
+        -0.7074604630470276,
+        0.5785335302352905,
+        0.26793089509010315,
+        -1.741084098815918,
+        -0.9064237475395203,
+        -0.27818211913108826,
+        -0.06530818343162537,
+        1.5608278512954712,
+        0.10034053027629852,
+        0.18459738790988922,
+        0.2896520495414734,
+        2.228212833404541,
+        1.3797621726989746,
+        -1.1619044542312622,
+        -0.10607702285051346
+      ],
+      [
+        -1.473647117614746,
+        -1.2038339376449585,
+        1.2881907224655151,
+        0.3939710557460785,
+        -0.05986154079437256,
+        -0.6614582538604736,
+        -0.9326079487800598,
+        -0.9152061343193054,
+        2.1059892177581787,
+        -0.34020259976387024,
+        -0.6582565307617188,
+        0.17788299918174744,
+        1.24713134765625,
+        1.0128247737884521,
+        -0.3762119710445404,
+        -0.35254669189453125
+      ],
+      [
+        -1.9578449726104736,
+        -0.8402428030967712,
+        1.8692209720611572,
+        0.32894837856292725,
+        -0.3004555404186249,
+        -0.5150565505027771,
+        0.7866722345352173,
+        -0.4721951484680176,
+        1.8520021438598633,
+        0.07333793491125107,
+        -1.147714376449585,
+        0.06321533769369125,
+        0.5042028427124023,
+        0.9146731495857239,
+        -0.2515338659286499,
+        -0.6100011467933655
+      ],
+      [
+        -1.3096731901168823,
+        -0.5322571396827698,
+        2.1369197368621826,
+        0.6385367512702942,
+        0.29661819338798523,
+        -0.2601790130138397,
+        1.7370480298995972,
+        -0.9429073333740234,
+        1.0160404443740845,
+        0.765198826789856,
+        -1.10575532913208,
+        -0.28133368492126465,
+        0.6474294066429138,
+        0.47843703627586365,
+        0.4757455587387085,
+        -0.9193580150604248
+      ],
+      [
+        -2.9883406162261963,
+        -1.278907299041748,
+        1.2906922101974487,
+        0.4815269410610199,
+        0.27497509121894836,
+        -0.8659355044364929,
+        0.6521288156509399,
+        -0.3194761872291565,
+        0.40679335594177246,
+        0.20416444540023804,
+        -1.134086012840271,
+        -0.2472257912158966,
+        0.004673673305660486,
+        0.3670925199985504,
+        0.03405305743217468,
+        -0.5286491513252258
+      ]
+    ]
+  ],
+  "compact_hidden_state": [
+    [
+      [
+        -2.024979591369629,
+        0.00422435300424695,
+        1.6957898139953613,
+        0.7697137594223022,
+        -0.6883586645126343,
+        0.11442556977272034,
+        0.7328180074691772,
+        -1.5748454332351685,
+        -0.5792887806892395,
+        1.0610051155090332,
+        1.6441532373428345,
+        -0.06880106776952744,
+        0.17206580936908722,
+        -0.3806779682636261,
+        0.17469821870326996,
+        0.7253968119621277
+      ],
+      [
+        -1.574650526046753,
+        -1.9975262880325317,
+        0.6534299254417419,
+        -0.642878532409668,
+        1.156783938407898,
+        -0.6467214822769165,
+        0.7542971968650818,
+        -1.1479636430740356,
+        0.8172124028205872,
+        -0.7986409068107605,
+        0.671444296836853,
+        -0.5100952982902527,
+        1.3076112270355225,
+        0.8520255088806152,
+        0.43006426095962524,
+        0.6130269765853882
+      ],
+      [
+        0.5974754691123962,
+        -2.4806430339813232,
+        -0.6777498722076416,
+        0.07211479544639587,
+        -0.5010150074958801,
+        -1.263437032699585,
+        0.21863402426242828,
+        1.4749866724014282,
+        -0.8187174797058105,
+        -0.030426712706685066,
+        -0.9761536121368408,
+        -1.1424365043640137,
+        0.7796595096588135,
+        0.9137669801712036,
+        -0.23002482950687408,
+        0.6912825703620911
+      ],
+      [
+        0.5695361495018005,
+        -0.7074604630470276,
+        0.5785335302352905,
+        0.26793089509010315,
+        -1.741084098815918,
+        -0.9064237475395203,
+        -0.27818211913108826,
+        -0.06530818343162537,
+        1.5608278512954712,
+        0.10034053027629852,
+        0.18459738790988922,
+        0.2896520495414734,
+        2.228212833404541,
+        1.3797621726989746,
+        -1.1619044542312622,
+        -0.10607702285051346
+      ]
+    ]
+  ]
+}
diff --git a/test/fixtures/embedding_models/tiny-random-mistral-causal/config.json b/test/fixtures/embedding_models/tiny-random-mistral-causal/config.json
new file mode 100644
index 00000000..c8a6d7a0
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-mistral-causal/config.json
@@ -0,0 +1,31 @@
+{
+  "architectures": [
+    "MistralModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "head_dim": 8,
+  "hidden_act": "silu",
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 20,
+  "is_causal": true,
+  "max_position_embeddings": 4,
+  "model_type": "mistral",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": 4,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.19.0.dev0",
+  "use_cache": true,
+  "vocab_size": 32
+}
diff --git a/test/fixtures/embedding_models/tiny-random-mistral-causal/model.safetensors b/test/fixtures/embedding_models/tiny-random-mistral-causal/model.safetensors
new file mode 100644
index 00000000..ba8fccc1
Binary files /dev/null and b/test/fixtures/embedding_models/tiny-random-mistral-causal/model.safetensors differ
diff --git a/test/fixtures/embedding_models/tiny-random-mistral-causal/reference.json b/test/fixtures/embedding_models/tiny-random-mistral-causal/reference.json
new file mode 100644
index 00000000..67d60a4c
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-mistral-causal/reference.json
@@ -0,0 +1,585 @@
+{
+  "transformers_version": "5.19.0.dev0",
+  "transformers_commit": "3693f8d26311305e914735a6373fb03468d6aaa0",
+  "is_causal": true,
+  "input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8
+    ]
+  ],
+  "attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1
+    ]
+  ],
+  "hidden_state": [
+    [
+      [
+        1.0457245111465454,
+        0.39682313799858093,
+        -1.3060978651046753,
+        0.2762241065502167,
+        -0.0689995065331459,
+        -1.3047702312469482,
+        0.19061890244483948,
+        0.3507913649082184,
+        1.3623313903808594,
+        -0.6598694920539856,
+        -1.6109312772750854,
+        1.263261318206787,
+        -0.8873047232627869,
+        -0.5571349859237671,
+        -1.6569924354553223,
+        -0.8687465786933899
+      ],
+      [
+        0.4716459810733795,
+        1.4045262336730957,
+        -0.2396407127380371,
+        -0.5017956495285034,
+        0.3653152585029602,
+        -0.9995490312576294,
+        -0.9438543915748596,
+        0.8067591786384583,
+        1.3381080627441406,
+        -0.6531286239624023,
+        0.17440663278102875,
+        1.004299283027649,
+        -1.9331988096237183,
+        1.610564112663269,
+        -1.0952517986297607,
+        0.09830035269260406
+      ],
+      [
+        -0.19974857568740845,
+        0.9967615008354187,
+        -0.26934054493904114,
+        0.8806093335151672,
+        -1.5961196422576904,
+        -0.3943643867969513,
+        -0.29033923149108887,
+        -0.5705398917198181,
+        0.5701212286949158,
+        -0.26539695262908936,
+        -1.1706457138061523,
+        -2.2563321590423584,
+        0.6113836765289307,
+        0.7895926237106323,
+        -1.716211199760437,
+        -0.3684362471103668
+      ],
+      [
+        -0.7963177561759949,
+        0.6328848600387573,
+        -1.6562623977661133,
+        -0.16359733045101166,
+        -0.44419386982917786,
+        0.2800651490688324,
+        0.14490863680839539,
+        2.2762582302093506,
+        0.9736656546592712,
+        0.03109324909746647,
+        -1.387130856513977,
+        -0.06400174647569656,
+        1.8648878335952759,
+        0.3368796706199646,
+        -0.39344727993011475,
+        -0.23220060765743256
+      ],
+      [
+        1.2459036111831665,
+        1.5128790140151978,
+        0.07607493549585342,
+        -0.8895657658576965,
+        -1.148682713508606,
+        0.9390343427658081,
+        -0.7024195194244385,
+        0.540144145488739,
+        0.15769976377487183,
+        0.3839298486709595,
+        -0.06393706053495407,
+        -0.43047428131103516,
+        1.2859246730804443,
+        -0.15021176636219025,
+        -2.170762777328491,
+        -1.2617021799087524
+      ],
+      [
+        -0.6428190469741821,
+        0.18813402950763702,
+        -2.841829299926758,
+        -0.7375192046165466,
+        1.0558021068572998,
+        0.31101593375205994,
+        1.6937586069107056,
+        0.9238065481185913,
+        0.08136066049337387,
+        -0.477201372385025,
+        -0.6063969135284424,
+        0.38450005650520325,
+        0.5868251323699951,
+        -0.19347746670246124,
+        -0.3715249001979828,
+        0.8177655339241028
+      ],
+      [
+        1.5168864727020264,
+        -0.43909311294555664,
+        -0.19021211564540863,
+        -0.02470860630273819,
+        -0.7569276094436646,
+        -1.2698664665222168,
+        1.3209482431411743,
+        1.0716817378997803,
+        -0.30340901017189026,
+        2.240630865097046,
+        -0.3053584396839142,
+        1.0077475309371948,
+        0.8149222731590271,
+        0.5360587239265442,
+        -1.0651097297668457,
+        0.18320679664611816
+      ],
+      [
+        0.9690777659416199,
+        0.9503204822540283,
+        -1.5491340160369873,
+        -0.17327119410037994,
+        -1.672007441520691,
+        0.5232784748077393,
+        1.715999960899353,
+        0.08890306204557419,
+        -0.1796051263809204,
+        -1.420764684677124,
+        -1.0932753086090088,
+        0.6969444751739502,
+        0.7366764545440674,
+        -0.16549089550971985,
+        0.7841010689735413,
+        0.823165774345398
+      ]
+    ]
+  ],
+  "changed_future_hidden_state": [
+    [
+      [
+        1.0457245111465454,
+        0.39682313799858093,
+        -1.3060978651046753,
+        0.2762241065502167,
+        -0.0689995065331459,
+        -1.3047702312469482,
+        0.19061890244483948,
+        0.3507913649082184,
+        1.3623313903808594,
+        -0.6598694920539856,
+        -1.6109312772750854,
+        1.263261318206787,
+        -0.8873047232627869,
+        -0.5571349859237671,
+        -1.6569924354553223,
+        -0.8687465786933899
+      ],
+      [
+        0.4716459810733795,
+        1.4045262336730957,
+        -0.2396407127380371,
+        -0.5017956495285034,
+        0.3653152585029602,
+        -0.9995490312576294,
+        -0.9438543915748596,
+        0.8067591786384583,
+        1.3381080627441406,
+        -0.6531286239624023,
+        0.17440663278102875,
+        1.004299283027649,
+        -1.9331988096237183,
+        1.610564112663269,
+        -1.0952517986297607,
+        0.09830035269260406
+      ],
+      [
+        -0.19974857568740845,
+        0.9967615008354187,
+        -0.26934054493904114,
+        0.8806093335151672,
+        -1.5961196422576904,
+        -0.3943643867969513,
+        -0.29033923149108887,
+        -0.5705398917198181,
+        0.5701212286949158,
+        -0.26539695262908936,
+        -1.1706457138061523,
+        -2.2563321590423584,
+        0.6113836765289307,
+        0.7895926237106323,
+        -1.716211199760437,
+        -0.3684362471103668
+      ],
+      [
+        -0.7963177561759949,
+        0.6328848600387573,
+        -1.6562623977661133,
+        -0.16359733045101166,
+        -0.44419386982917786,
+        0.2800651490688324,
+        0.14490863680839539,
+        2.2762582302093506,
+        0.9736656546592712,
+        0.03109324909746647,
+        -1.387130856513977,
+        -0.06400174647569656,
+        1.8648878335952759,
+        0.3368796706199646,
+        -0.39344727993011475,
+        -0.23220060765743256
+      ],
+      [
+        1.2459036111831665,
+        1.5128790140151978,
+        0.07607493549585342,
+        -0.8895657658576965,
+        -1.148682713508606,
+        0.9390343427658081,
+        -0.7024195194244385,
+        0.540144145488739,
+        0.15769976377487183,
+        0.3839298486709595,
+        -0.06393706053495407,
+        -0.43047428131103516,
+        1.2859246730804443,
+        -0.15021176636219025,
+        -2.170762777328491,
+        -1.2617021799087524
+      ],
+      [
+        -0.6428190469741821,
+        0.18813402950763702,
+        -2.841829299926758,
+        -0.7375192046165466,
+        1.0558021068572998,
+        0.31101593375205994,
+        1.6937586069107056,
+        0.9238065481185913,
+        0.08136066049337387,
+        -0.477201372385025,
+        -0.6063969135284424,
+        0.38450005650520325,
+        0.5868251323699951,
+        -0.19347746670246124,
+        -0.3715249001979828,
+        0.8177655339241028
+      ],
+      [
+        1.5168864727020264,
+        -0.43909311294555664,
+        -0.19021211564540863,
+        -0.02470860630273819,
+        -0.7569276094436646,
+        -1.2698664665222168,
+        1.3209482431411743,
+        1.0716817378997803,
+        -0.30340901017189026,
+        2.240630865097046,
+        -0.3053584396839142,
+        1.0077475309371948,
+        0.8149222731590271,
+        0.5360587239265442,
+        -1.0651097297668457,
+        0.18320679664611816
+      ],
+      [
+        0.9496896862983704,
+        -0.47796133160591125,
+        -0.004653741139918566,
+        -0.7508406639099121,
+        -0.25656858086586,
+        -0.3712241053581238,
+        0.4718755781650543,
+        1.8938292264938354,
+        -0.0568883903324604,
+        -0.8258190155029297,
+        -0.24048510193824768,
+        0.03883671388030052,
+        1.9553927183151245,
+        1.0270168781280518,
+        -1.4293417930603027,
+        -1.6071233749389648
+      ]
+    ]
+  ],
+  "padded_input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_position_ids": [
+    [
+      0,
+      1,
+      2,
+      3,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_hidden_state": [
+    [
+      [
+        1.0457245111465454,
+        0.39682313799858093,
+        -1.3060978651046753,
+        0.2762241065502167,
+        -0.0689995065331459,
+        -1.3047702312469482,
+        0.19061890244483948,
+        0.3507913649082184,
+        1.3623313903808594,
+        -0.6598694920539856,
+        -1.6109312772750854,
+        1.263261318206787,
+        -0.8873047232627869,
+        -0.5571349859237671,
+        -1.6569924354553223,
+        -0.8687465786933899
+      ],
+      [
+        0.4716459810733795,
+        1.4045262336730957,
+        -0.2396407127380371,
+        -0.5017956495285034,
+        0.3653152585029602,
+        -0.9995490312576294,
+        -0.9438543915748596,
+        0.8067591786384583,
+        1.3381080627441406,
+        -0.6531286239624023,
+        0.17440663278102875,
+        1.004299283027649,
+        -1.9331988096237183,
+        1.610564112663269,
+        -1.0952517986297607,
+        0.09830035269260406
+      ],
+      [
+        -0.19974857568740845,
+        0.9967615008354187,
+        -0.26934054493904114,
+        0.8806093335151672,
+        -1.5961196422576904,
+        -0.3943643867969513,
+        -0.29033923149108887,
+        -0.5705398917198181,
+        0.5701212286949158,
+        -0.26539695262908936,
+        -1.1706457138061523,
+        -2.2563321590423584,
+        0.6113836765289307,
+        0.7895926237106323,
+        -1.716211199760437,
+        -0.3684362471103668
+      ],
+      [
+        -0.7963177561759949,
+        0.6328848600387573,
+        -1.6562623977661133,
+        -0.16359733045101166,
+        -0.44419386982917786,
+        0.2800651490688324,
+        0.14490863680839539,
+        2.2762582302093506,
+        0.9736656546592712,
+        0.03109324909746647,
+        -1.387130856513977,
+        -0.06400174647569656,
+        1.8648878335952759,
+        0.3368796706199646,
+        -0.39344727993011475,
+        -0.23220060765743256
+      ],
+      [
+        1.4604097604751587,
+        1.417560338973999,
+        -1.239171028137207,
+        -0.4389970004558563,
+        -0.6755046248435974,
+        1.6083509922027588,
+        0.818422257900238,
+        0.43579769134521484,
+        -0.07843512296676636,
+        -0.26081621646881104,
+        -0.08191128820180893,
+        -0.8643302917480469,
+        -0.8045240640640259,
+        0.7373144626617432,
+        -1.9217536449432373,
+        -0.20476743578910828
+      ],
+      [
+        1.5395121574401855,
+        1.7385066747665405,
+        -1.2784159183502197,
+        -0.25923117995262146,
+        -0.31848573684692383,
+        1.7036038637161255,
+        0.6433312892913818,
+        0.19692188501358032,
+        0.7023715972900391,
+        -0.9438970685005188,
+        -0.7388767600059509,
+        -0.44891852140426636,
+        -0.1452265977859497,
+        0.20128528773784637,
+        -1.7371362447738647,
+        0.15751835703849792
+      ],
+      [
+        1.0805162191390991,
+        2.525688409805298,
+        -1.3378835916519165,
+        0.15099506080150604,
+        -0.7410507202148438,
+        1.2186410427093506,
+        0.20132021605968475,
+        0.6892490983009338,
+        -0.10679129511117935,
+        -0.5358866453170776,
+        -0.8643946647644043,
+        -0.4570711553096771,
+        -0.6711523532867432,
+        0.06001877039670944,
+        -1.492336630821228,
+        -0.08808988332748413
+      ],
+      [
+        1.116092324256897,
+        1.5184133052825928,
+        -1.3790533542633057,
+        0.870972752571106,
+        0.31387078762054443,
+        1.187323808670044,
+        0.9923067688941956,
+        0.1025753766298294,
+        0.23870109021663666,
+        -0.9075555801391602,
+        -0.4964388310909271,
+        0.45422685146331787,
+        -0.6726132035255432,
+        0.39164018630981445,
+        -2.038893461227417,
+        0.23064832389354706
+      ]
+    ]
+  ],
+  "compact_hidden_state": [
+    [
+      [
+        1.0457245111465454,
+        0.39682313799858093,
+        -1.3060978651046753,
+        0.2762241065502167,
+        -0.0689995065331459,
+        -1.3047702312469482,
+        0.19061890244483948,
+        0.3507913649082184,
+        1.3623313903808594,
+        -0.6598694920539856,
+        -1.6109312772750854,
+        1.263261318206787,
+        -0.8873047232627869,
+        -0.5571349859237671,
+        -1.6569924354553223,
+        -0.8687465786933899
+      ],
+      [
+        0.4716459810733795,
+        1.4045262336730957,
+        -0.2396407127380371,
+        -0.5017956495285034,
+        0.3653152585029602,
+        -0.9995490312576294,
+        -0.9438543915748596,
+        0.8067591786384583,
+        1.3381080627441406,
+        -0.6531286239624023,
+        0.17440663278102875,
+        1.004299283027649,
+        -1.9331988096237183,
+        1.610564112663269,
+        -1.0952517986297607,
+        0.09830035269260406
+      ],
+      [
+        -0.19974857568740845,
+        0.9967615008354187,
+        -0.26934054493904114,
+        0.8806093335151672,
+        -1.5961196422576904,
+        -0.3943643867969513,
+        -0.29033923149108887,
+        -0.5705398917198181,
+        0.5701212286949158,
+        -0.26539695262908936,
+        -1.1706457138061523,
+        -2.2563321590423584,
+        0.6113836765289307,
+        0.7895926237106323,
+        -1.716211199760437,
+        -0.3684362471103668
+      ],
+      [
+        -0.7963177561759949,
+        0.6328848600387573,
+        -1.6562623977661133,
+        -0.16359733045101166,
+        -0.44419386982917786,
+        0.2800651490688324,
+        0.14490863680839539,
+        2.2762582302093506,
+        0.9736656546592712,
+        0.03109324909746647,
+        -1.387130856513977,
+        -0.06400174647569656,
+        1.8648878335952759,
+        0.3368796706199646,
+        -0.39344727993011475,
+        -0.23220060765743256
+      ]
+    ]
+  ]
+}
diff --git a/test/fixtures/embedding_models/tiny-random-qwen3-bidirectional/config.json b/test/fixtures/embedding_models/tiny-random-qwen3-bidirectional/config.json
new file mode 100644
index 00000000..19c6850a
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-qwen3-bidirectional/config.json
@@ -0,0 +1,42 @@
+{
+  "architectures": [
+    "Qwen3Model"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 8,
+  "hidden_act": "silu",
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 20,
+  "is_causal": false,
+  "layer_types": [
+    "full_attention",
+    "sliding_attention"
+  ],
+  "max_position_embeddings": 4,
+  "max_window_layers": 1,
+  "model_type": "qwen3",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "beta_fast": 32.0,
+    "beta_slow": 1.0,
+    "factor": 2.0,
+    "original_max_position_embeddings": 4,
+    "rope_theta": 10000.0,
+    "rope_type": "yarn"
+  },
+  "sliding_window": 4,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.19.0.dev0",
+  "use_cache": true,
+  "use_sliding_window": true,
+  "vocab_size": 32
+}
diff --git a/test/fixtures/embedding_models/tiny-random-qwen3-bidirectional/model.safetensors b/test/fixtures/embedding_models/tiny-random-qwen3-bidirectional/model.safetensors
new file mode 100644
index 00000000..760c739b
Binary files /dev/null and b/test/fixtures/embedding_models/tiny-random-qwen3-bidirectional/model.safetensors differ
diff --git a/test/fixtures/embedding_models/tiny-random-qwen3-bidirectional/reference.json b/test/fixtures/embedding_models/tiny-random-qwen3-bidirectional/reference.json
new file mode 100644
index 00000000..e7a446d7
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-qwen3-bidirectional/reference.json
@@ -0,0 +1,585 @@
+{
+  "transformers_version": "5.19.0.dev0",
+  "transformers_commit": "3693f8d26311305e914735a6373fb03468d6aaa0",
+  "is_causal": false,
+  "input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8
+    ]
+  ],
+  "attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1
+    ]
+  ],
+  "hidden_state": [
+    [
+      [
+        1.350559949874878,
+        0.3178101181983948,
+        0.7481591701507568,
+        0.10766077786684036,
+        -1.185314416885376,
+        -1.0836466550827026,
+        1.447921872138977,
+        0.21750709414482117,
+        -0.4462021589279175,
+        1.689261794090271,
+        1.2657628059387207,
+        -0.15816742181777954,
+        -0.36326873302459717,
+        -0.15119625627994537,
+        1.8396574258804321,
+        0.7196674346923828
+      ],
+      [
+        -2.5102901458740234,
+        -0.30366250872612,
+        -0.4036041796207428,
+        0.4208015203475952,
+        2.1718509197235107,
+        1.1856632232666016,
+        0.1593940407037735,
+        0.523223340511322,
+        -0.5114821791648865,
+        -0.0543123222887516,
+        -1.3282155990600586,
+        -0.3653411567211151,
+        0.43281289935112,
+        -0.541247546672821,
+        -0.39816170930862427,
+        0.08309747278690338
+      ],
+      [
+        0.7306246757507324,
+        0.12441401928663254,
+        0.6518114805221558,
+        -0.3301393389701843,
+        -1.4737542867660522,
+        -0.1687028855085373,
+        -1.7721201181411743,
+        1.2681777477264404,
+        0.5531604886054993,
+        -1.115384578704834,
+        -1.2415684461593628,
+        -0.1820027083158493,
+        -0.002038809238001704,
+        -0.7249341607093811,
+        -1.8173599243164062,
+        0.9943010210990906
+      ],
+      [
+        -0.8314095735549927,
+        1.3219454288482666,
+        0.1978011429309845,
+        -0.42745015025138855,
+        1.625260353088379,
+        0.8051959276199341,
+        -1.0418092012405396,
+        0.8208600878715515,
+        0.3777369558811188,
+        0.3412087559700012,
+        2.1885459423065186,
+        -0.5775662660598755,
+        -0.3247236907482147,
+        -1.1269084215164185,
+        0.9108465909957886,
+        0.761423647403717
+      ],
+      [
+        -0.693884015083313,
+        -0.419795960187912,
+        2.1682629585266113,
+        -0.27729102969169617,
+        -1.0372825860977173,
+        0.07565928250551224,
+        -0.9706767201423645,
+        -0.17928685247898102,
+        -0.38533830642700195,
+        -1.4658259153366089,
+        -1.526716947555542,
+        -0.008815225213766098,
+        0.04878310486674309,
+        -1.5404285192489624,
+        1.1013493537902832,
+        0.49744367599487305
+      ],
+      [
+        -0.9191871285438538,
+        0.17874133586883545,
+        -0.33160439133644104,
+        0.5771011114120483,
+        -0.4949254095554352,
+        0.9238427877426147,
+        0.16718795895576477,
+        -1.6037296056747437,
+        0.2548547387123108,
+        1.253710389137268,
+        -1.0416696071624756,
+        -0.42111527919769287,
+        2.6438965797424316,
+        -0.1659298837184906,
+        -0.4348093271255493,
+        -0.9151893258094788
+      ],
+      [
+        -1.1895089149475098,
+        -0.47694122791290283,
+        -0.1395302563905716,
+        -0.3498130440711975,
+        0.02275407873094082,
+        -2.089066505432129,
+        -0.7014834880828857,
+        0.6677780747413635,
+        -0.6957371830940247,
+        -1.1855604648590088,
+        1.3445868492126465,
+        0.9922950863838196,
+        -1.6594120264053345,
+        -0.7500167489051819,
+        0.9355742931365967,
+        0.07462596148252487
+      ],
+      [
+        0.7948005199432373,
+        -0.18521204590797424,
+        -0.2928406000137329,
+        0.7305781245231628,
+        -0.862913191318512,
+        -2.6246449947357178,
+        0.12370120733976364,
+        -0.8796436786651611,
+        0.33533576130867004,
+        -0.5405341982841492,
+        2.0834686756134033,
+        0.39228567481040955,
+        -0.5441868305206299,
+        -0.6471394896507263,
+        -0.014103304594755173,
+        0.8107578754425049
+      ]
+    ]
+  ],
+  "changed_future_hidden_state": [
+    [
+      [
+        1.4414962530136108,
+        0.09841890633106232,
+        0.7620136737823486,
+        0.1345861256122589,
+        -1.1889086961746216,
+        -1.0942615270614624,
+        1.4072258472442627,
+        0.4142148196697235,
+        -0.4905776083469391,
+        1.733771800994873,
+        1.1686686277389526,
+        -0.18469959497451782,
+        -0.3427717387676239,
+        -0.11726730316877365,
+        1.7936660051345825,
+        0.7185011506080627
+      ],
+      [
+        -2.4099230766296387,
+        -0.2945424020290375,
+        -0.467713326215744,
+        0.46636512875556946,
+        2.242757558822632,
+        1.2059688568115234,
+        0.1350792795419693,
+        0.5382947325706482,
+        -0.5196685194969177,
+        0.07179037481546402,
+        -1.2868114709854126,
+        -0.2803109884262085,
+        0.529522716999054,
+        -0.5841517448425293,
+        -0.4425698220729828,
+        0.11863147467374802
+      ],
+      [
+        0.7297210693359375,
+        0.13074249029159546,
+        0.637159526348114,
+        -0.3227149248123169,
+        -1.439537763595581,
+        -0.17192091047763824,
+        -1.773032546043396,
+        1.2696226835250854,
+        0.5859816074371338,
+        -1.0509506464004517,
+        -1.232175588607788,
+        -0.15594977140426636,
+        0.00725382287055254,
+        -0.7201060652732849,
+        -1.8666884899139404,
+        1.0310509204864502
+      ],
+      [
+        -0.6917160749435425,
+        1.266114354133606,
+        0.23325835168361664,
+        -0.36127549409866333,
+        1.562750220298767,
+        0.817488968372345,
+        -1.0286016464233398,
+        0.9030247926712036,
+        0.36570534110069275,
+        0.4619066119194031,
+        2.2426395416259766,
+        -0.4287980794906616,
+        -0.20749486982822418,
+        -1.2254705429077148,
+        0.9226468801498413,
+        0.8229857683181763
+      ],
+      [
+        -0.5881412625312805,
+        -0.4333111047744751,
+        2.246696710586548,
+        -0.22508276998996735,
+        -1.085333228111267,
+        0.11135563254356384,
+        -0.9380852580070496,
+        -0.16662044823169708,
+        -0.3979543447494507,
+        -1.4052706956863403,
+        -1.4411982297897339,
+        0.05163854360580444,
+        0.08101881295442581,
+        -1.5833849906921387,
+        1.1093673706054688,
+        0.5187309980392456
+      ],
+      [
+        -0.914634644985199,
+        0.14577659964561462,
+        -0.329280823469162,
+        0.5972010493278503,
+        -0.5309459567070007,
+        0.9620389938354492,
+        0.1622711569070816,
+        -1.6158705949783325,
+        0.24506776034832,
+        1.246222972869873,
+        -1.0167968273162842,
+        -0.3781801760196686,
+        2.643556594848633,
+        -0.1671152114868164,
+        -0.44846856594085693,
+        -0.887309193611145
+      ],
+      [
+        -1.193820595741272,
+        -0.4629804790019989,
+        -0.04415268450975418,
+        -0.2173684686422348,
+        -0.10564259439706802,
+        -2.002591609954834,
+        -0.6138955950737,
+        0.6902562975883484,
+        -0.7546002268791199,
+        -1.06553316116333,
+        1.3638418912887573,
+        1.1256906986236572,
+        -1.6926573514938354,
+        -0.7759847044944763,
+        1.0480232238769531,
+        0.07866396009922028
+      ],
+      [
+        0.20235583186149597,
+        -0.6441594362258911,
+        -0.010680070146918297,
+        1.195014476776123,
+        0.7041913866996765,
+        -0.6370631456375122,
+        1.0560811758041382,
+        -1.9728020429611206,
+        -1.238986849784851,
+        1.5660455226898193,
+        -1.0383538007736206,
+        -0.411517471075058,
+        0.09145431220531464,
+        0.17536330223083496,
+        0.3741169571876526,
+        1.6555171012878418
+      ]
+    ]
+  ],
+  "padded_input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_position_ids": [
+    [
+      0,
+      1,
+      2,
+      3,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_hidden_state": [
+    [
+      [
+        1.3575406074523926,
+        -0.03442586585879326,
+        0.7975572943687439,
+        0.043932538479566574,
+        -1.268054485321045,
+        -1.2300266027450562,
+        1.2785450220108032,
+        0.6153780221939087,
+        -0.517633318901062,
+        1.5294262170791626,
+        0.9677543044090271,
+        -0.05646393820643425,
+        -0.5775828957557678,
+        -0.06296776235103607,
+        1.997388482093811,
+        0.6863288879394531
+      ],
+      [
+        -2.4353442192077637,
+        -0.40793001651763916,
+        -0.3860514163970947,
+        0.3504772186279297,
+        2.314417839050293,
+        1.0902633666992188,
+        0.14918528497219086,
+        0.6800119876861572,
+        -0.6139646768569946,
+        0.11720925569534302,
+        -1.1450222730636597,
+        -0.10086202621459961,
+        0.4207999110221863,
+        -0.8064197301864624,
+        -0.1503995656967163,
+        0.04525348171591759
+      ],
+      [
+        0.927562415599823,
+        -0.05159541592001915,
+        0.7610282897949219,
+        -0.2615455985069275,
+        -1.6773104667663574,
+        -0.19513805210590363,
+        -1.6960629224777222,
+        1.4776091575622559,
+        0.4502253532409668,
+        -1.1510564088821411,
+        -1.0565249919891357,
+        -0.06514640152454376,
+        -0.04341081157326698,
+        -0.8075702786445618,
+        -1.5556124448776245,
+        0.9114256501197815
+      ],
+      [
+        -0.749329149723053,
+        1.0385620594024658,
+        -0.08191359043121338,
+        0.0050321160815656185,
+        1.7405470609664917,
+        0.6614091992378235,
+        -1.3665612936019897,
+        1.4097896814346313,
+        0.06597322970628738,
+        0.3986719250679016,
+        1.8160037994384766,
+        -0.26435616612434387,
+        -0.565814733505249,
+        -0.872330904006958,
+        1.3404967784881592,
+        0.7134331464767456
+      ],
+      [
+        1.0785601139068604,
+        -1.4294650554656982,
+        1.6954798698425293,
+        -0.48260682821273804,
+        -1.1471539735794067,
+        -0.4656665325164795,
+        0.05205651745200157,
+        0.9297119379043579,
+        -0.5659959316253662,
+        -0.30224794149398804,
+        0.8040741086006165,
+        1.23289954662323,
+        -0.28226423263549805,
+        -0.5336360335350037,
+        1.9778586626052856,
+        0.22060096263885498
+      ],
+      [
+        0.010865113697946072,
+        -1.6466420888900757,
+        0.9573414325714111,
+        0.8008619546890259,
+        -0.8302528262138367,
+        -0.8151628971099854,
+        0.8893851637840271,
+        -0.15942765772342682,
+        -0.9731195569038391,
+        0.8599136471748352,
+        0.24824008345603943,
+        1.4142239093780518,
+        0.9409717321395874,
+        -1.029392957687378,
+        1.6234378814697266,
+        -0.3334019184112549
+      ],
+      [
+        -0.7485479712486267,
+        -1.3795349597930908,
+        0.9399937987327576,
+        0.009572915732860565,
+        -0.2578974664211273,
+        -1.2208712100982666,
+        0.7124056816101074,
+        0.37055331468582153,
+        1.0618648529052734,
+        2.2095797061920166,
+        0.039419129490852356,
+        1.3752845525741577,
+        0.47434699535369873,
+        -0.9282047748565674,
+        -0.5037508606910706,
+        0.8588907718658447
+      ],
+      [
+        -0.07892175763845444,
+        -1.5499411821365356,
+        2.063124895095825,
+        -0.5545370578765869,
+        -1.3263733386993408,
+        -1.0968677997589111,
+        -0.05683986842632294,
+        0.729322612285614,
+        0.449887216091156,
+        0.8971385955810547,
+        0.23625093698501587,
+        1.3742499351501465,
+        0.1003219410777092,
+        -1.1050864458084106,
+        0.5938446521759033,
+        0.9019147157669067
+      ]
+    ]
+  ],
+  "compact_hidden_state": [
+    [
+      [
+        1.3575406074523926,
+        -0.034425847232341766,
+        0.7975572943687439,
+        0.04393252730369568,
+        -1.268054485321045,
+        -1.2300264835357666,
+        1.2785450220108032,
+        0.6153780817985535,
+        -0.517633318901062,
+        1.5294259786605835,
+        0.9677543044090271,
+        -0.05646399036049843,
+        -0.5775829553604126,
+        -0.06296775490045547,
+        1.997388482093811,
+        0.6863288879394531
+      ],
+      [
+        -2.4353442192077637,
+        -0.40793001651763916,
+        -0.3860514461994171,
+        0.3504771888256073,
+        2.314417839050293,
+        1.0902633666992188,
+        0.14918525516986847,
+        0.6800119876861572,
+        -0.6139647364616394,
+        0.11720927059650421,
+        -1.1450222730636597,
+        -0.10086201876401901,
+        0.4207999110221863,
+        -0.8064196109771729,
+        -0.15039949119091034,
+        0.04525347054004669
+      ],
+      [
+        0.927562415599823,
+        -0.051595427095890045,
+        0.7610281705856323,
+        -0.2615455389022827,
+        -1.6773104667663574,
+        -0.195138081908226,
+        -1.6960629224777222,
+        1.4776091575622559,
+        0.4502253234386444,
+        -1.1510564088821411,
+        -1.0565249919891357,
+        -0.06514637917280197,
+        -0.04341082647442818,
+        -0.807570219039917,
+        -1.555612325668335,
+        0.9114256501197815
+      ],
+      [
+        -0.7493290901184082,
+        1.0385620594024658,
+        -0.08191357553005219,
+        0.005032079294323921,
+        1.7405468225479126,
+        0.6614091992378235,
+        -1.3665612936019897,
+        1.4097896814346313,
+        0.06597322970628738,
+        0.3986719250679016,
+        1.8160039186477661,
+        -0.26435616612434387,
+        -0.5658146739006042,
+        -0.872330904006958,
+        1.3404966592788696,
+        0.7134332656860352
+      ]
+    ]
+  ]
+}
diff --git a/test/fixtures/embedding_models/tiny-random-qwen3-causal/config.json b/test/fixtures/embedding_models/tiny-random-qwen3-causal/config.json
new file mode 100644
index 00000000..abfb1977
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-qwen3-causal/config.json
@@ -0,0 +1,42 @@
+{
+  "architectures": [
+    "Qwen3Model"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 8,
+  "hidden_act": "silu",
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 20,
+  "is_causal": true,
+  "layer_types": [
+    "full_attention",
+    "sliding_attention"
+  ],
+  "max_position_embeddings": 4,
+  "max_window_layers": 1,
+  "model_type": "qwen3",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "beta_fast": 32.0,
+    "beta_slow": 1.0,
+    "factor": 2.0,
+    "original_max_position_embeddings": 4,
+    "rope_theta": 10000.0,
+    "rope_type": "yarn"
+  },
+  "sliding_window": 4,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.19.0.dev0",
+  "use_cache": true,
+  "use_sliding_window": true,
+  "vocab_size": 32
+}
diff --git a/test/fixtures/embedding_models/tiny-random-qwen3-causal/model.safetensors b/test/fixtures/embedding_models/tiny-random-qwen3-causal/model.safetensors
new file mode 100644
index 00000000..1dc33909
Binary files /dev/null and b/test/fixtures/embedding_models/tiny-random-qwen3-causal/model.safetensors differ
diff --git a/test/fixtures/embedding_models/tiny-random-qwen3-causal/reference.json b/test/fixtures/embedding_models/tiny-random-qwen3-causal/reference.json
new file mode 100644
index 00000000..819b7d4c
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-qwen3-causal/reference.json
@@ -0,0 +1,585 @@
+{
+  "transformers_version": "5.19.0.dev0",
+  "transformers_commit": "3693f8d26311305e914735a6373fb03468d6aaa0",
+  "is_causal": true,
+  "input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8
+    ]
+  ],
+  "attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1
+    ]
+  ],
+  "hidden_state": [
+    [
+      [
+        0.2310911864042282,
+        -0.7421520352363586,
+        1.6158628463745117,
+        1.041075587272644,
+        -0.9046542048454285,
+        -0.3888615071773529,
+        -1.164892315864563,
+        -0.2920728325843811,
+        0.3573792278766632,
+        1.2735074758529663,
+        0.21380965411663055,
+        -0.9841861128807068,
+        1.8372576236724854,
+        -1.0434606075286865,
+        -1.1192810535430908,
+        -0.8689664006233215
+      ],
+      [
+        0.4481666386127472,
+        -0.8397944569587708,
+        0.8949003219604492,
+        -0.13214075565338135,
+        0.9811152219772339,
+        -1.9255461692810059,
+        -0.12754251062870026,
+        0.6743159294128418,
+        -0.8005005717277527,
+        -0.3586670756340027,
+        1.056210994720459,
+        2.3233449459075928,
+        0.6125563383102417,
+        0.020705752074718475,
+        -1.101619005203247,
+        0.4829452633857727
+      ],
+      [
+        -0.014537354931235313,
+        -0.10732890665531158,
+        -0.6133922934532166,
+        -0.7499865889549255,
+        1.6835945844650269,
+        0.9814766049385071,
+        -2.036954402923584,
+        0.7924901247024536,
+        0.1670203059911728,
+        0.13999894261360168,
+        -0.042475804686546326,
+        -1.0875784158706665,
+        -0.8745710253715515,
+        1.6335707902908325,
+        -1.2464081048965454,
+        -0.4732646346092224
+      ],
+      [
+        -1.2924543619155884,
+        0.2852725684642792,
+        0.6633800864219666,
+        0.20630985498428345,
+        1.4976096153259277,
+        0.7437268495559692,
+        0.9700919389724731,
+        0.31334388256073,
+        -0.36697787046432495,
+        0.8076547980308533,
+        -2.5204460620880127,
+        -0.8195761442184448,
+        -0.785367488861084,
+        0.39937910437583923,
+        1.0936617851257324,
+        -0.32385197281837463
+      ],
+      [
+        -0.8187649846076965,
+        -0.031066808849573135,
+        -1.520588994026184,
+        -0.8547930717468262,
+        -0.7586885690689087,
+        -0.2189294695854187,
+        -0.1354806274175644,
+        -0.7698339819908142,
+        1.0613222122192383,
+        -0.679656445980072,
+        1.2781957387924194,
+        1.6260697841644287,
+        0.22157464921474457,
+        -1.4033583402633667,
+        -1.6005682945251465,
+        0.7598890662193298
+      ],
+      [
+        0.21045728027820587,
+        1.1181769371032715,
+        -0.9845547080039978,
+        0.32182592153549194,
+        -1.2873862981796265,
+        0.3569393754005432,
+        -1.728271722793579,
+        0.28536954522132874,
+        -0.4134802520275116,
+        0.08405973762273788,
+        -0.2025788128376007,
+        1.7028783559799194,
+        1.1107999086380005,
+        1.8868170976638794,
+        0.45848122239112854,
+        0.7590603232383728
+      ],
+      [
+        -0.5329175591468811,
+        -0.8314939737319946,
+        -1.0893213748931885,
+        -1.1047228574752808,
+        -1.8496001958847046,
+        -0.7118915915489197,
+        1.143576741218567,
+        0.7482010722160339,
+        0.33854928612709045,
+        0.7331032752990723,
+        -0.015596220269799232,
+        0.34838658571243286,
+        -0.30203190445899963,
+        0.8370251059532166,
+        -2.2639589309692383,
+        -0.27264583110809326
+      ],
+      [
+        1.3531838655471802,
+        0.8530457615852356,
+        1.0800665616989136,
+        0.4601617455482483,
+        0.06451592594385147,
+        -0.931238055229187,
+        0.42339977622032166,
+        -2.249089479446411,
+        -1.042013168334961,
+        -0.5827427506446838,
+        -1.4722959995269775,
+        0.6953997611999512,
+        0.5702813863754272,
+        -1.0545657873153687,
+        0.2684260308742523,
+        0.5552740693092346
+      ]
+    ]
+  ],
+  "changed_future_hidden_state": [
+    [
+      [
+        0.2310911864042282,
+        -0.7421520352363586,
+        1.6158628463745117,
+        1.041075587272644,
+        -0.9046542048454285,
+        -0.3888615071773529,
+        -1.164892315864563,
+        -0.2920728325843811,
+        0.3573792278766632,
+        1.2735074758529663,
+        0.21380965411663055,
+        -0.9841861128807068,
+        1.8372576236724854,
+        -1.0434606075286865,
+        -1.1192810535430908,
+        -0.8689664006233215
+      ],
+      [
+        0.4481666386127472,
+        -0.8397944569587708,
+        0.8949003219604492,
+        -0.13214075565338135,
+        0.9811152219772339,
+        -1.9255461692810059,
+        -0.12754251062870026,
+        0.6743159294128418,
+        -0.8005005717277527,
+        -0.3586670756340027,
+        1.056210994720459,
+        2.3233449459075928,
+        0.6125563383102417,
+        0.020705752074718475,
+        -1.101619005203247,
+        0.4829452633857727
+      ],
+      [
+        -0.014537354931235313,
+        -0.10732890665531158,
+        -0.6133922934532166,
+        -0.7499865889549255,
+        1.6835945844650269,
+        0.9814766049385071,
+        -2.036954402923584,
+        0.7924901247024536,
+        0.1670203059911728,
+        0.13999894261360168,
+        -0.042475804686546326,
+        -1.0875784158706665,
+        -0.8745710253715515,
+        1.6335707902908325,
+        -1.2464081048965454,
+        -0.4732646346092224
+      ],
+      [
+        -1.2924543619155884,
+        0.2852725684642792,
+        0.6633800864219666,
+        0.20630985498428345,
+        1.4976096153259277,
+        0.7437268495559692,
+        0.9700919389724731,
+        0.31334388256073,
+        -0.36697787046432495,
+        0.8076547980308533,
+        -2.5204460620880127,
+        -0.8195761442184448,
+        -0.785367488861084,
+        0.39937910437583923,
+        1.0936617851257324,
+        -0.32385197281837463
+      ],
+      [
+        -0.8187649846076965,
+        -0.031066808849573135,
+        -1.520588994026184,
+        -0.8547930717468262,
+        -0.7586885690689087,
+        -0.2189294695854187,
+        -0.1354806274175644,
+        -0.7698339819908142,
+        1.0613222122192383,
+        -0.679656445980072,
+        1.2781957387924194,
+        1.6260697841644287,
+        0.22157464921474457,
+        -1.4033583402633667,
+        -1.6005682945251465,
+        0.7598890662193298
+      ],
+      [
+        0.21045728027820587,
+        1.1181769371032715,
+        -0.9845547080039978,
+        0.32182592153549194,
+        -1.2873862981796265,
+        0.3569393754005432,
+        -1.728271722793579,
+        0.28536954522132874,
+        -0.4134802520275116,
+        0.08405973762273788,
+        -0.2025788128376007,
+        1.7028783559799194,
+        1.1107999086380005,
+        1.8868170976638794,
+        0.45848122239112854,
+        0.7590603232383728
+      ],
+      [
+        -0.5329175591468811,
+        -0.8314939737319946,
+        -1.0893213748931885,
+        -1.1047228574752808,
+        -1.8496001958847046,
+        -0.7118915915489197,
+        1.143576741218567,
+        0.7482010722160339,
+        0.33854928612709045,
+        0.7331032752990723,
+        -0.015596220269799232,
+        0.34838658571243286,
+        -0.30203190445899963,
+        0.8370251059532166,
+        -2.2639589309692383,
+        -0.27264583110809326
+      ],
+      [
+        0.853391706943512,
+        -1.2616376876831055,
+        0.6194024085998535,
+        -1.207357406616211,
+        1.384700894355774,
+        -0.9697015881538391,
+        -1.0826444625854492,
+        -0.517204999923706,
+        -1.7552224397659302,
+        0.12296659499406815,
+        -0.5402941703796387,
+        -1.3667089939117432,
+        -0.007396686356514692,
+        -0.8923590779304504,
+        0.7019681930541992,
+        0.9878334999084473
+      ]
+    ]
+  ],
+  "padded_input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_position_ids": [
+    [
+      0,
+      1,
+      2,
+      3,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_hidden_state": [
+    [
+      [
+        0.2310911864042282,
+        -0.7421520352363586,
+        1.6158628463745117,
+        1.041075587272644,
+        -0.9046542048454285,
+        -0.3888615071773529,
+        -1.164892315864563,
+        -0.2920728325843811,
+        0.3573792278766632,
+        1.2735074758529663,
+        0.21380965411663055,
+        -0.9841861128807068,
+        1.8372576236724854,
+        -1.0434606075286865,
+        -1.1192810535430908,
+        -0.8689664006233215
+      ],
+      [
+        0.4481666386127472,
+        -0.8397944569587708,
+        0.8949003219604492,
+        -0.13214075565338135,
+        0.9811152219772339,
+        -1.9255461692810059,
+        -0.12754251062870026,
+        0.6743159294128418,
+        -0.8005005717277527,
+        -0.3586670756340027,
+        1.056210994720459,
+        2.3233449459075928,
+        0.6125563383102417,
+        0.020705752074718475,
+        -1.101619005203247,
+        0.4829452633857727
+      ],
+      [
+        -0.014537354931235313,
+        -0.10732890665531158,
+        -0.6133922934532166,
+        -0.7499865889549255,
+        1.6835945844650269,
+        0.9814766049385071,
+        -2.036954402923584,
+        0.7924901247024536,
+        0.1670203059911728,
+        0.13999894261360168,
+        -0.042475804686546326,
+        -1.0875784158706665,
+        -0.8745710253715515,
+        1.6335707902908325,
+        -1.2464081048965454,
+        -0.4732646346092224
+      ],
+      [
+        -1.2924543619155884,
+        0.2852725684642792,
+        0.6633800864219666,
+        0.20630985498428345,
+        1.4976096153259277,
+        0.7437268495559692,
+        0.9700919389724731,
+        0.31334388256073,
+        -0.36697787046432495,
+        0.8076547980308533,
+        -2.5204460620880127,
+        -0.8195761442184448,
+        -0.785367488861084,
+        0.39937910437583923,
+        1.0936617851257324,
+        -0.32385197281837463
+      ],
+      [
+        -0.4585987329483032,
+        1.269760251045227,
+        -0.1518075168132782,
+        -2.4535744190216064,
+        1.158497929573059,
+        0.5909342765808105,
+        0.902363121509552,
+        1.000800371170044,
+        0.31494027376174927,
+        1.299875259399414,
+        -1.0597964525222778,
+        -0.17137113213539124,
+        0.345499724149704,
+        -0.3656998872756958,
+        -0.002300509950146079,
+        0.45605215430259705
+      ],
+      [
+        -1.0602632761001587,
+        1.03390371799469,
+        -0.24304801225662231,
+        -2.613170623779297,
+        1.2658607959747314,
+        0.512658417224884,
+        0.9798953533172607,
+        0.20381927490234375,
+        -0.055538710206747055,
+        0.3681679666042328,
+        -0.27888986468315125,
+        -0.7863332033157349,
+        0.7969784736633301,
+        -0.9557739496231079,
+        -0.6515271663665771,
+        0.5398895144462585
+      ],
+      [
+        -1.3975939750671387,
+        0.37306395173072815,
+        0.32698971033096313,
+        -2.746215581893921,
+        1.8203470706939697,
+        0.08790260553359985,
+        0.5294983983039856,
+        -0.294023334980011,
+        -0.8523790240287781,
+        0.11878150701522827,
+        0.5885274410247803,
+        -0.09451255947351456,
+        0.23042303323745728,
+        0.008393523283302784,
+        -0.04092834144830704,
+        0.8441029787063599
+      ],
+      [
+        -0.08862346410751343,
+        -0.8135499358177185,
+        -0.3278692364692688,
+        -0.22375516593456268,
+        -0.5012823343276978,
+        -1.357567310333252,
+        0.2663370668888092,
+        1.3048303127288818,
+        1.0638865232467651,
+        1.1903012990951538,
+        -1.9993650913238525,
+        -0.06445223838090897,
+        0.8879125714302063,
+        0.38748928904533386,
+        -1.287636160850525,
+        -0.7257612943649292
+      ]
+    ]
+  ],
+  "compact_hidden_state": [
+    [
+      [
+        0.2310911864042282,
+        -0.7421520352363586,
+        1.6158628463745117,
+        1.041075587272644,
+        -0.9046542048454285,
+        -0.3888615071773529,
+        -1.164892315864563,
+        -0.2920728325843811,
+        0.3573792278766632,
+        1.2735074758529663,
+        0.21380965411663055,
+        -0.9841861128807068,
+        1.8372576236724854,
+        -1.0434606075286865,
+        -1.1192810535430908,
+        -0.8689664006233215
+      ],
+      [
+        0.4481666386127472,
+        -0.8397944569587708,
+        0.8949003219604492,
+        -0.13214075565338135,
+        0.9811152219772339,
+        -1.9255461692810059,
+        -0.12754251062870026,
+        0.6743159294128418,
+        -0.8005005717277527,
+        -0.3586670756340027,
+        1.056210994720459,
+        2.3233449459075928,
+        0.6125563383102417,
+        0.020705752074718475,
+        -1.101619005203247,
+        0.4829452633857727
+      ],
+      [
+        -0.014537394978106022,
+        -0.10732890665531158,
+        -0.6133922934532166,
+        -0.7499865293502808,
+        1.6835945844650269,
+        0.9814766049385071,
+        -2.036954164505005,
+        0.7924900650978088,
+        0.16702033579349518,
+        0.13999886810779572,
+        -0.04247580096125603,
+        -1.087578296661377,
+        -0.8745710253715515,
+        1.6335707902908325,
+        -1.2464081048965454,
+        -0.4732646048069
+      ],
+      [
+        -1.2924543619155884,
+        0.2852725684642792,
+        0.6633801460266113,
+        0.20630985498428345,
+        1.4976096153259277,
+        0.7437268495559692,
+        0.9700919389724731,
+        0.3133438229560852,
+        -0.3669778108596802,
+        0.8076547980308533,
+        -2.5204460620880127,
+        -0.8195762038230896,
+        -0.785367488861084,
+        0.39937910437583923,
+        1.0936617851257324,
+        -0.32385197281837463
+      ]
+    ]
+  ]
+}
diff --git a/test/fixtures/embedding_models/tiny-random-qwen3_dynamic-bidirectional/config.json b/test/fixtures/embedding_models/tiny-random-qwen3_dynamic-bidirectional/config.json
new file mode 100644
index 00000000..ac76a8eb
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-qwen3_dynamic-bidirectional/config.json
@@ -0,0 +1,39 @@
+{
+  "architectures": [
+    "Qwen3Model"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 8,
+  "hidden_act": "silu",
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 20,
+  "is_causal": false,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 4,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "factor": 2.0,
+    "rope_theta": 10000.0,
+    "rope_type": "dynamic"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.19.0.dev0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 32
+}
diff --git a/test/fixtures/embedding_models/tiny-random-qwen3_dynamic-bidirectional/model.safetensors b/test/fixtures/embedding_models/tiny-random-qwen3_dynamic-bidirectional/model.safetensors
new file mode 100644
index 00000000..7bb78761
Binary files /dev/null and b/test/fixtures/embedding_models/tiny-random-qwen3_dynamic-bidirectional/model.safetensors differ
diff --git a/test/fixtures/embedding_models/tiny-random-qwen3_dynamic-bidirectional/reference.json b/test/fixtures/embedding_models/tiny-random-qwen3_dynamic-bidirectional/reference.json
new file mode 100644
index 00000000..59aafaf8
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-qwen3_dynamic-bidirectional/reference.json
@@ -0,0 +1,585 @@
+{
+  "transformers_version": "5.19.0.dev0",
+  "transformers_commit": "3693f8d26311305e914735a6373fb03468d6aaa0",
+  "is_causal": false,
+  "input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8
+    ]
+  ],
+  "attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1
+    ]
+  ],
+  "hidden_state": [
+    [
+      [
+        1.1271569728851318,
+        -0.7582653164863586,
+        0.8165315389633179,
+        -1.3774099349975586,
+        -0.7914233803749084,
+        -0.9857174158096313,
+        0.9311455488204956,
+        -1.4444749355316162,
+        -0.14191028475761414,
+        -0.5160574913024902,
+        0.176486536860466,
+        -0.8215921521186829,
+        -1.7398711442947388,
+        -1.0090317726135254,
+        -0.1253630518913269,
+        -1.395457148551941
+      ],
+      [
+        1.725915789604187,
+        1.9359350204467773,
+        -1.9838529825210571,
+        -0.6760861873626709,
+        0.24782492220401764,
+        0.7966299653053284,
+        -0.7023797631263733,
+        -0.6819608807563782,
+        -0.5663787722587585,
+        0.13066789507865906,
+        0.5673474073410034,
+        1.3372117280960083,
+        -0.11706450581550598,
+        0.661521315574646,
+        0.5005055665969849,
+        0.19719849526882172
+      ],
+      [
+        1.1402204036712646,
+        1.4049279689788818,
+        -0.3498368263244629,
+        -0.16071827709674835,
+        0.8978187441825867,
+        -1.08040452003479,
+        0.8626967072486877,
+        -0.6211226582527161,
+        -0.8658261895179749,
+        0.12395675480365753,
+        -0.35193467140197754,
+        -1.7533090114593506,
+        1.235106348991394,
+        1.0021789073944092,
+        -1.1957495212554932,
+        1.2359678745269775
+      ],
+      [
+        0.9064061641693115,
+        -1.4237779378890991,
+        0.5965151190757751,
+        0.17993266880512238,
+        0.7842872142791748,
+        0.5714245438575745,
+        0.7923699617385864,
+        0.44263264536857605,
+        0.05639713630080223,
+        0.4921652674674988,
+        1.6990852355957031,
+        0.45630499720573425,
+        -2.7084317207336426,
+        -0.5364041328430176,
+        -0.054048266261816025,
+        -0.005280205048620701
+      ],
+      [
+        0.8060490489006042,
+        1.341726303100586,
+        0.27916547656059265,
+        1.009084939956665,
+        -0.6927196383476257,
+        1.3357529640197754,
+        -1.5284298658370972,
+        0.4113866090774536,
+        -0.16098205745220184,
+        0.6253282427787781,
+        -0.06754513084888458,
+        0.772090494632721,
+        -1.5532135963439941,
+        -1.7967253923416138,
+        -0.9762897491455078,
+        -0.10893910378217697
+      ],
+      [
+        -0.36883270740509033,
+        -1.217573642730713,
+        1.1168620586395264,
+        0.3989535868167877,
+        -0.10991460084915161,
+        -0.4918246269226074,
+        -2.292295217514038,
+        -1.309532880783081,
+        0.2906244695186615,
+        1.0043928623199463,
+        -1.227964162826538,
+        0.1774507462978363,
+        -1.6185284852981567,
+        0.49963581562042236,
+        0.44591888785362244,
+        -0.108094222843647
+      ],
+      [
+        -1.5884953737258911,
+        0.120477095246315,
+        0.7643815875053406,
+        1.0780409574508667,
+        1.5580939054489136,
+        1.3419119119644165,
+        1.8668664693832397,
+        0.05969002842903137,
+        -0.39662712812423706,
+        -0.158731609582901,
+        -0.3439045548439026,
+        -0.6511425971984863,
+        0.12451092153787613,
+        -0.12835204601287842,
+        -1.6682751178741455,
+        -0.6567022204399109
+      ],
+      [
+        0.29405996203422546,
+        1.5244441032409668,
+        1.570932149887085,
+        -0.15298259258270264,
+        1.273855447769165,
+        1.454079031944275,
+        0.5239161252975464,
+        -1.186195731163025,
+        0.4173121154308319,
+        0.3396446704864502,
+        -0.8117257356643677,
+        -0.32810312509536743,
+        0.38716232776641846,
+        0.06600216776132584,
+        -1.2210077047348022,
+        -1.7188111543655396
+      ]
+    ]
+  ],
+  "changed_future_hidden_state": [
+    [
+      [
+        1.0640114545822144,
+        -0.7521106004714966,
+        0.8113104104995728,
+        -1.359920859336853,
+        -0.7734811902046204,
+        -1.0184860229492188,
+        0.8627182841300964,
+        -1.4570690393447876,
+        -0.11519775539636612,
+        -0.47596296668052673,
+        0.25830602645874023,
+        -0.8864796757698059,
+        -1.740402102470398,
+        -1.025914192199707,
+        -0.27029234170913696,
+        -1.4165949821472168
+      ],
+      [
+        1.695641040802002,
+        1.932089924812317,
+        -2.013043165206909,
+        -0.7331581115722656,
+        0.27800995111465454,
+        0.8325716257095337,
+        -0.6975470781326294,
+        -0.6860038042068481,
+        -0.5559238791465759,
+        0.1868758648633957,
+        0.6751341223716736,
+        1.2357227802276611,
+        -0.05516676604747772,
+        0.6842001080513,
+        0.448165625333786,
+        0.1935003250837326
+      ],
+      [
+        1.0917906761169434,
+        1.390878438949585,
+        -0.3191741406917572,
+        -0.14776921272277832,
+        0.9162519574165344,
+        -1.1432591676712036,
+        0.8497404456138611,
+        -0.6195980906486511,
+        -0.8784134984016418,
+        0.14461283385753632,
+        -0.3301628530025482,
+        -1.7719957828521729,
+        1.2280431985855103,
+        0.9631430506706238,
+        -1.213257908821106,
+        1.2346714735031128
+      ],
+      [
+        0.7918721437454224,
+        -1.4587533473968506,
+        0.6120910048484802,
+        0.11607936769723892,
+        0.9100660085678101,
+        0.4520818591117859,
+        0.8686382174491882,
+        0.36101752519607544,
+        -0.023491481319069862,
+        0.5214682221412659,
+        1.572606086730957,
+        0.48730799555778503,
+        -2.7417361736297607,
+        -0.6149212718009949,
+        -0.13356411457061768,
+        0.11706696450710297
+      ],
+      [
+        0.7281784415245056,
+        1.3577688932418823,
+        0.2812657356262207,
+        0.9618713855743408,
+        -0.7007272243499756,
+        1.380702257156372,
+        -1.5643706321716309,
+        0.36610862612724304,
+        -0.13721197843551636,
+        0.7244709134101868,
+        0.07688789069652557,
+        0.6453123688697815,
+        -1.508171796798706,
+        -1.8014029264450073,
+        -1.038694977760315,
+        -0.13182327151298523
+      ],
+      [
+        -0.39883387088775635,
+        -1.186034917831421,
+        1.1224665641784668,
+        0.3767022490501404,
+        -0.10678105801343918,
+        -0.5460745692253113,
+        -2.3369178771972656,
+        -1.3210846185684204,
+        0.3135673999786377,
+        1.0191569328308105,
+        -1.2291591167449951,
+        0.12858717143535614,
+        -1.5558300018310547,
+        0.4824967086315155,
+        0.3933362364768982,
+        -0.07146643102169037
+      ],
+      [
+        -1.6205922365188599,
+        0.07588270306587219,
+        0.7522038817405701,
+        1.0805602073669434,
+        1.6044853925704956,
+        1.217560887336731,
+        1.8729029893875122,
+        -0.03431827202439308,
+        -0.47557809948921204,
+        -0.10070766508579254,
+        -0.3100138306617737,
+        -0.5617149472236633,
+        0.06524547934532166,
+        -0.19499433040618896,
+        -1.7096116542816162,
+        -0.655633270740509
+      ],
+      [
+        0.28617945313453674,
+        1.30093252658844,
+        -0.5860463380813599,
+        1.2087839841842651,
+        0.15136563777923584,
+        0.3867217004299164,
+        -0.3065159022808075,
+        0.2986409068107605,
+        -1.8185943365097046,
+        -1.7794506549835205,
+        -0.4264552593231201,
+        -0.9863840341567993,
+        1.0143297910690308,
+        1.1607985496520996,
+        1.2821377515792847,
+        -0.608453631401062
+      ]
+    ]
+  ],
+  "padded_input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_position_ids": [
+    [
+      0,
+      1,
+      2,
+      3,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_hidden_state": [
+    [
+      [
+        1.2589613199234009,
+        -0.9332112073898315,
+        0.8169341683387756,
+        -1.2974823713302612,
+        -0.821860671043396,
+        -0.7151171565055847,
+        0.9545788168907166,
+        -1.456567406654358,
+        -0.11707993596792221,
+        -0.5965086221694946,
+        0.12152868509292603,
+        -0.6738951802253723,
+        -1.7720755338668823,
+        -1.006718397140503,
+        -0.19282230734825134,
+        -1.3807276487350464
+      ],
+      [
+        1.8205491304397583,
+        1.6359100341796875,
+        -1.9377731084823608,
+        -0.678888738155365,
+        -0.01859140768647194,
+        1.3638032674789429,
+        -0.29819542169570923,
+        -0.5468883514404297,
+        -0.3616829216480255,
+        0.14609098434448242,
+        0.7182790040969849,
+        1.3606576919555664,
+        -0.18572111427783966,
+        0.7793359756469727,
+        0.5557401776313782,
+        0.19852028787136078
+      ],
+      [
+        1.0317168235778809,
+        1.2923389673233032,
+        -0.47558173537254333,
+        -0.1925874948501587,
+        0.8752397894859314,
+        -0.9514361023902893,
+        1.2922052145004272,
+        -0.5741048455238342,
+        -0.9378992915153503,
+        0.05776228383183479,
+        -0.2987219989299774,
+        -1.7695854902267456,
+        1.2143572568893433,
+        0.9532588720321655,
+        -1.072176456451416,
+        1.2929821014404297
+      ],
+      [
+        0.8377544283866882,
+        -1.4684553146362305,
+        0.49971887469291687,
+        0.21701914072036743,
+        0.7516922354698181,
+        0.7905600666999817,
+        1.1274656057357788,
+        0.40176650881767273,
+        -0.014716638252139091,
+        0.3117108941078186,
+        1.6634773015975952,
+        0.472106009721756,
+        -2.621835470199585,
+        -0.47042346000671387,
+        0.12005565315485,
+        -0.017355579882860184
+      ],
+      [
+        0.2177358716726303,
+        -0.7000093460083008,
+        0.5814934372901917,
+        0.5447192788124084,
+        -0.018158674240112305,
+        1.904797911643982,
+        2.527592420578003,
+        0.14341603219509125,
+        -0.30553480982780457,
+        -0.9878469109535217,
+        0.6047174334526062,
+        0.10696088522672653,
+        -1.3823214769363403,
+        -0.2662399709224701,
+        0.5746049284934998,
+        0.23749154806137085
+      ],
+      [
+        0.2177358716726303,
+        -0.7000093460083008,
+        0.5814934372901917,
+        0.5447192788124084,
+        -0.018158674240112305,
+        1.904797911643982,
+        2.527592420578003,
+        0.14341603219509125,
+        -0.30553480982780457,
+        -0.9878469109535217,
+        0.6047174334526062,
+        0.10696088522672653,
+        -1.3823214769363403,
+        -0.2662399709224701,
+        0.5746049284934998,
+        0.23749154806137085
+      ],
+      [
+        0.2177358716726303,
+        -0.7000093460083008,
+        0.5814934372901917,
+        0.5447192788124084,
+        -0.018158674240112305,
+        1.904797911643982,
+        2.527592420578003,
+        0.14341603219509125,
+        -0.30553480982780457,
+        -0.9878469109535217,
+        0.6047174334526062,
+        0.10696088522672653,
+        -1.3823214769363403,
+        -0.2662399709224701,
+        0.5746049284934998,
+        0.23749154806137085
+      ],
+      [
+        0.2177358716726303,
+        -0.7000093460083008,
+        0.5814934372901917,
+        0.5447192788124084,
+        -0.018158674240112305,
+        1.904797911643982,
+        2.527592420578003,
+        0.14341603219509125,
+        -0.30553480982780457,
+        -0.9878469109535217,
+        0.6047174334526062,
+        0.10696088522672653,
+        -1.3823214769363403,
+        -0.2662399709224701,
+        0.5746049284934998,
+        0.23749154806137085
+      ]
+    ]
+  ],
+  "compact_hidden_state": [
+    [
+      [
+        1.2589613199234009,
+        -0.9332111477851868,
+        0.8169341683387756,
+        -1.2974823713302612,
+        -0.821860671043396,
+        -0.7151171565055847,
+        0.9545789361000061,
+        -1.456567406654358,
+        -0.11707989871501923,
+        -0.5965086221694946,
+        0.12152866274118423,
+        -0.6738951206207275,
+        -1.7720751762390137,
+        -1.006718397140503,
+        -0.19282230734825134,
+        -1.3807276487350464
+      ],
+      [
+        1.8205490112304688,
+        1.6359100341796875,
+        -1.9377731084823608,
+        -0.678888738155365,
+        -0.018591420724987984,
+        1.3638031482696533,
+        -0.29819536209106445,
+        -0.5468883514404297,
+        -0.3616829216480255,
+        0.14609098434448242,
+        0.7182789444923401,
+        1.3606576919555664,
+        -0.18572111427783966,
+        0.7793359756469727,
+        0.5557401776313782,
+        0.19852031767368317
+      ],
+      [
+        1.0317168235778809,
+        1.2923389673233032,
+        -0.47558173537254333,
+        -0.1925874948501587,
+        0.8752397894859314,
+        -0.9514361023902893,
+        1.2922052145004272,
+        -0.5741048455238342,
+        -0.9378993511199951,
+        0.0577622689306736,
+        -0.2987220287322998,
+        -1.7695854902267456,
+        1.2143572568893433,
+        0.9532589316368103,
+        -1.072176456451416,
+        1.2929821014404297
+      ],
+      [
+        0.8377544283866882,
+        -1.4684553146362305,
+        0.49971887469291687,
+        0.21701914072036743,
+        0.7516921758651733,
+        0.7905600666999817,
+        1.1274657249450684,
+        0.4017665386199951,
+        -0.014716658741235733,
+        0.3117108643054962,
+        1.6634774208068848,
+        0.472106009721756,
+        -2.621835470199585,
+        -0.47042346000671387,
+        0.1200556606054306,
+        -0.017355570569634438
+      ]
+    ]
+  ]
+}
diff --git a/test/fixtures/embedding_models/tiny-random-qwen3_dynamic-causal/config.json b/test/fixtures/embedding_models/tiny-random-qwen3_dynamic-causal/config.json
new file mode 100644
index 00000000..d06227af
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-qwen3_dynamic-causal/config.json
@@ -0,0 +1,39 @@
+{
+  "architectures": [
+    "Qwen3Model"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 8,
+  "hidden_act": "silu",
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 20,
+  "is_causal": true,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 4,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "factor": 2.0,
+    "rope_theta": 10000.0,
+    "rope_type": "dynamic"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.19.0.dev0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 32
+}
diff --git a/test/fixtures/embedding_models/tiny-random-qwen3_dynamic-causal/model.safetensors b/test/fixtures/embedding_models/tiny-random-qwen3_dynamic-causal/model.safetensors
new file mode 100644
index 00000000..5ed1fc92
Binary files /dev/null and b/test/fixtures/embedding_models/tiny-random-qwen3_dynamic-causal/model.safetensors differ
diff --git a/test/fixtures/embedding_models/tiny-random-qwen3_dynamic-causal/reference.json b/test/fixtures/embedding_models/tiny-random-qwen3_dynamic-causal/reference.json
new file mode 100644
index 00000000..aadf713c
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-qwen3_dynamic-causal/reference.json
@@ -0,0 +1,585 @@
+{
+  "transformers_version": "5.19.0.dev0",
+  "transformers_commit": "3693f8d26311305e914735a6373fb03468d6aaa0",
+  "is_causal": true,
+  "input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8
+    ]
+  ],
+  "attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1
+    ]
+  ],
+  "hidden_state": [
+    [
+      [
+        -0.4182742238044739,
+        1.5112836360931396,
+        -0.025699011981487274,
+        -0.7797371745109558,
+        -0.13699187338352203,
+        0.2942725121974945,
+        -1.487064003944397,
+        0.6858462691307068,
+        0.8765056729316711,
+        -1.8516476154327393,
+        -1.5692821741104126,
+        -1.0727769136428833,
+        -1.050380825996399,
+        -0.8316739201545715,
+        0.3493972420692444,
+        0.6111904978752136
+      ],
+      [
+        1.4220627546310425,
+        0.27736592292785645,
+        0.7255986928939819,
+        0.04643205553293228,
+        -0.33639198541641235,
+        1.1324095726013184,
+        0.501779317855835,
+        1.8326257467269897,
+        0.09026020020246506,
+        0.8267349004745483,
+        0.9334233999252319,
+        -1.8538038730621338,
+        -0.015311451628804207,
+        0.5632038712501526,
+        1.3654264211654663,
+        -1.071913480758667
+      ],
+      [
+        0.12224620580673218,
+        -1.7805639505386353,
+        0.5182938575744629,
+        0.3151543438434601,
+        -1.02891206741333,
+        1.7755043506622314,
+        -1.3350296020507812,
+        -0.394748330116272,
+        -1.928560495376587,
+        -0.802724301815033,
+        0.030400831252336502,
+        -0.25020918250083923,
+        -0.12602803111076355,
+        -0.11729856580495834,
+        1.304673671722412,
+        -0.32380563020706177
+      ],
+      [
+        -1.5125590562820435,
+        0.3046088218688965,
+        -0.6275227069854736,
+        2.155548334121704,
+        1.1568942070007324,
+        -0.306565523147583,
+        -0.3077033460140228,
+        -0.7024491429328918,
+        0.09575822949409485,
+        0.11629553139209747,
+        1.132570743560791,
+        0.9242783188819885,
+        -0.11231645941734314,
+        1.3881763219833374,
+        1.1013861894607544,
+        1.0966095924377441
+      ],
+      [
+        1.9884411096572876,
+        0.361234575510025,
+        0.23765923082828522,
+        0.2582189440727234,
+        0.8649953603744507,
+        -0.49694254994392395,
+        -0.5676278471946716,
+        0.02620985172688961,
+        0.08442974835634232,
+        -1.1107805967330933,
+        1.8562204837799072,
+        -0.5003494620323181,
+        0.6689697504043579,
+        0.08006023615598679,
+        -2.2170612812042236,
+        -0.35246214270591736
+      ],
+      [
+        -1.1601098775863647,
+        -0.5565108060836792,
+        1.7388681173324585,
+        -0.8363213539123535,
+        0.9898486733436584,
+        2.444310188293457,
+        -0.5241672396659851,
+        0.30723533034324646,
+        -0.45568767189979553,
+        -0.4653460383415222,
+        0.4462859630584717,
+        0.6351938843727112,
+        0.48426809906959534,
+        -0.04627907648682594,
+        1.2444204092025757,
+        -0.6541460752487183
+      ],
+      [
+        -0.7450334429740906,
+        -0.36560744047164917,
+        -1.348331332206726,
+        0.03933992609381676,
+        0.599586546421051,
+        1.3102186918258667,
+        -0.6898004412651062,
+        -2.680666208267212,
+        -0.3923966586589813,
+        1.0113565921783447,
+        0.3248187005519867,
+        -1.0695972442626953,
+        0.788663387298584,
+        -0.2565804719924927,
+        0.1526925414800644,
+        0.7670185565948486
+      ],
+      [
+        0.7545332312583923,
+        -1.5008366107940674,
+        0.6859128475189209,
+        -1.7492886781692505,
+        -0.8584056496620178,
+        0.11235889792442322,
+        0.8676731586456299,
+        0.21482735872268677,
+        0.15913145244121552,
+        -1.1214385032653809,
+        1.2091416120529175,
+        0.8795013427734375,
+        0.012556985020637512,
+        0.8023542761802673,
+        0.1268131285905838,
+        -1.967818260192871
+      ]
+    ]
+  ],
+  "changed_future_hidden_state": [
+    [
+      [
+        -0.4182742238044739,
+        1.5112836360931396,
+        -0.025699011981487274,
+        -0.7797371745109558,
+        -0.13699187338352203,
+        0.2942725121974945,
+        -1.487064003944397,
+        0.6858462691307068,
+        0.8765056729316711,
+        -1.8516476154327393,
+        -1.5692821741104126,
+        -1.0727769136428833,
+        -1.050380825996399,
+        -0.8316739201545715,
+        0.3493972420692444,
+        0.6111904978752136
+      ],
+      [
+        1.4220627546310425,
+        0.27736592292785645,
+        0.7255986928939819,
+        0.04643205553293228,
+        -0.33639198541641235,
+        1.1324095726013184,
+        0.501779317855835,
+        1.8326257467269897,
+        0.09026020020246506,
+        0.8267349004745483,
+        0.9334233999252319,
+        -1.8538038730621338,
+        -0.015311451628804207,
+        0.5632038712501526,
+        1.3654264211654663,
+        -1.071913480758667
+      ],
+      [
+        0.12224620580673218,
+        -1.7805639505386353,
+        0.5182938575744629,
+        0.3151543438434601,
+        -1.02891206741333,
+        1.7755043506622314,
+        -1.3350296020507812,
+        -0.394748330116272,
+        -1.928560495376587,
+        -0.802724301815033,
+        0.030400831252336502,
+        -0.25020918250083923,
+        -0.12602803111076355,
+        -0.11729856580495834,
+        1.304673671722412,
+        -0.32380563020706177
+      ],
+      [
+        -1.5125590562820435,
+        0.3046088218688965,
+        -0.6275227069854736,
+        2.155548334121704,
+        1.1568942070007324,
+        -0.306565523147583,
+        -0.3077033460140228,
+        -0.7024491429328918,
+        0.09575822949409485,
+        0.11629553139209747,
+        1.132570743560791,
+        0.9242783188819885,
+        -0.11231645941734314,
+        1.3881763219833374,
+        1.1013861894607544,
+        1.0966095924377441
+      ],
+      [
+        1.9884411096572876,
+        0.361234575510025,
+        0.23765923082828522,
+        0.2582189440727234,
+        0.8649953603744507,
+        -0.49694254994392395,
+        -0.5676278471946716,
+        0.02620985172688961,
+        0.08442974835634232,
+        -1.1107805967330933,
+        1.8562204837799072,
+        -0.5003494620323181,
+        0.6689697504043579,
+        0.08006023615598679,
+        -2.2170612812042236,
+        -0.35246214270591736
+      ],
+      [
+        -1.1601098775863647,
+        -0.5565108060836792,
+        1.7388681173324585,
+        -0.8363213539123535,
+        0.9898486733436584,
+        2.444310188293457,
+        -0.5241672396659851,
+        0.30723533034324646,
+        -0.45568767189979553,
+        -0.4653460383415222,
+        0.4462859630584717,
+        0.6351938843727112,
+        0.48426809906959534,
+        -0.04627907648682594,
+        1.2444204092025757,
+        -0.6541460752487183
+      ],
+      [
+        -0.7450334429740906,
+        -0.36560744047164917,
+        -1.348331332206726,
+        0.03933992609381676,
+        0.599586546421051,
+        1.3102186918258667,
+        -0.6898004412651062,
+        -2.680666208267212,
+        -0.3923966586589813,
+        1.0113565921783447,
+        0.3248187005519867,
+        -1.0695972442626953,
+        0.788663387298584,
+        -0.2565804719924927,
+        0.1526925414800644,
+        0.7670185565948486
+      ],
+      [
+        0.8485293388366699,
+        0.21174556016921997,
+        -0.7083292603492737,
+        -0.3837893009185791,
+        0.4921610653400421,
+        0.17720545828342438,
+        -1.3712587356567383,
+        -1.4570472240447998,
+        0.27693814039230347,
+        1.2866679430007935,
+        2.038839101791382,
+        -1.0481922626495361,
+        -0.33188512921333313,
+        -0.6351959705352783,
+        -1.55901300907135,
+        -0.5915477275848389
+      ]
+    ]
+  ],
+  "padded_input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_position_ids": [
+    [
+      0,
+      1,
+      2,
+      3,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_hidden_state": [
+    [
+      [
+        -0.4182742238044739,
+        1.5112836360931396,
+        -0.025699011981487274,
+        -0.7797371745109558,
+        -0.13699187338352203,
+        0.2942725121974945,
+        -1.487064003944397,
+        0.6858462691307068,
+        0.8765056729316711,
+        -1.8516476154327393,
+        -1.5692821741104126,
+        -1.0727769136428833,
+        -1.050380825996399,
+        -0.8316739201545715,
+        0.3493972420692444,
+        0.6111904978752136
+      ],
+      [
+        1.4226020574569702,
+        0.2768370509147644,
+        0.7244383692741394,
+        0.044443804770708084,
+        -0.3361719846725464,
+        1.1311790943145752,
+        0.5026962757110596,
+        1.8327158689498901,
+        0.09035629034042358,
+        0.8271207213401794,
+        0.9325364828109741,
+        -1.854024052619934,
+        -0.015438954345881939,
+        0.5642936825752258,
+        1.3660928010940552,
+        -1.0716432332992554
+      ],
+      [
+        0.12117406725883484,
+        -1.7803711891174316,
+        0.5216084718704224,
+        0.3140164315700531,
+        -1.029453158378601,
+        1.776227593421936,
+        -1.3324857950210571,
+        -0.3984460234642029,
+        -1.9280109405517578,
+        -0.806088387966156,
+        0.02698143944144249,
+        -0.24862496554851532,
+        -0.12284760177135468,
+        -0.11554059386253357,
+        1.303626537322998,
+        -0.3237568140029907
+      ],
+      [
+        -1.5048562288284302,
+        0.30200761556625366,
+        -0.6270103454589844,
+        2.1612040996551514,
+        1.1521599292755127,
+        -0.30691203474998474,
+        -0.30504342913627625,
+        -0.7017474174499512,
+        0.0974915623664856,
+        0.11040645837783813,
+        1.1320863962173462,
+        0.9267258644104004,
+        -0.115012526512146,
+        1.3851051330566406,
+        1.1032369136810303,
+        1.1038659811019897
+      ],
+      [
+        1.0181776285171509,
+        0.7616355419158936,
+        -0.6478985548019409,
+        -1.41938316822052,
+        1.4242634773254395,
+        0.4871251583099365,
+        -0.8162285685539246,
+        -1.328191876411438,
+        -0.28537213802337646,
+        1.353971242904663,
+        1.0180273056030273,
+        -1.5634214878082275,
+        -0.03228190541267395,
+        -0.08149232715368271,
+        -1.0069823265075684,
+        0.16483289003372192
+      ],
+      [
+        1.0181776285171509,
+        0.7616355419158936,
+        -0.6478985548019409,
+        -1.41938316822052,
+        1.4242634773254395,
+        0.4871251583099365,
+        -0.8162285685539246,
+        -1.328191876411438,
+        -0.28537213802337646,
+        1.353971242904663,
+        1.0180273056030273,
+        -1.5634214878082275,
+        -0.03228190541267395,
+        -0.08149232715368271,
+        -1.0069823265075684,
+        0.16483289003372192
+      ],
+      [
+        1.0181776285171509,
+        0.7616355419158936,
+        -0.6478985548019409,
+        -1.41938316822052,
+        1.4242634773254395,
+        0.4871251583099365,
+        -0.8162285685539246,
+        -1.328191876411438,
+        -0.28537213802337646,
+        1.353971242904663,
+        1.0180273056030273,
+        -1.5634214878082275,
+        -0.03228190541267395,
+        -0.08149232715368271,
+        -1.0069823265075684,
+        0.16483289003372192
+      ],
+      [
+        1.0181776285171509,
+        0.7616355419158936,
+        -0.6478985548019409,
+        -1.41938316822052,
+        1.4242634773254395,
+        0.4871251583099365,
+        -0.8162285685539246,
+        -1.328191876411438,
+        -0.28537213802337646,
+        1.353971242904663,
+        1.0180273056030273,
+        -1.5634214878082275,
+        -0.03228190541267395,
+        -0.08149232715368271,
+        -1.0069823265075684,
+        0.16483289003372192
+      ]
+    ]
+  ],
+  "compact_hidden_state": [
+    [
+      [
+        -0.4182742238044739,
+        1.5112836360931396,
+        -0.025699011981487274,
+        -0.7797371745109558,
+        -0.13699187338352203,
+        0.2942725121974945,
+        -1.487064003944397,
+        0.6858462691307068,
+        0.8765056729316711,
+        -1.8516476154327393,
+        -1.5692821741104126,
+        -1.0727769136428833,
+        -1.050380825996399,
+        -0.8316739201545715,
+        0.3493972420692444,
+        0.6111904978752136
+      ],
+      [
+        1.4226020574569702,
+        0.276837021112442,
+        0.7244383692741394,
+        0.044443804770708084,
+        -0.336171954870224,
+        1.1311790943145752,
+        0.5026962757110596,
+        1.8327161073684692,
+        0.09035629034042358,
+        0.8271207213401794,
+        0.9325364828109741,
+        -1.8540241718292236,
+        -0.015438982285559177,
+        0.5642938017845154,
+        1.3660928010940552,
+        -1.071643352508545
+      ],
+      [
+        0.12117404490709305,
+        -1.7803711891174316,
+        0.5216084718704224,
+        0.3140164315700531,
+        -1.029453158378601,
+        1.776227593421936,
+        -1.3324857950210571,
+        -0.39844605326652527,
+        -1.9280109405517578,
+        -0.806088387966156,
+        0.026981450617313385,
+        -0.24862496554851532,
+        -0.12284760922193527,
+        -0.11554061621427536,
+        1.303626537322998,
+        -0.3237568140029907
+      ],
+      [
+        -1.5048561096191406,
+        0.3020075559616089,
+        -0.6270102858543396,
+        2.1612040996551514,
+        1.1521598100662231,
+        -0.30691200494766235,
+        -0.30504336953163147,
+        -0.7017473578453064,
+        0.0974915474653244,
+        0.11040650308132172,
+        1.1320862770080566,
+        0.9267258048057556,
+        -0.1150125041604042,
+        1.385105013847351,
+        1.1032369136810303,
+        1.1038659811019897
+      ]
+    ]
+  ]
+}
diff --git a/test/fixtures/embedding_models/tiny-random-qwen3_linear-bidirectional/config.json b/test/fixtures/embedding_models/tiny-random-qwen3_linear-bidirectional/config.json
new file mode 100644
index 00000000..af73b29a
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-qwen3_linear-bidirectional/config.json
@@ -0,0 +1,40 @@
+{
+  "architectures": [
+    "Qwen3Model"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 8,
+  "hidden_act": "silu",
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 20,
+  "is_causal": false,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 4,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "factor": 2.0,
+    "rope_theta": 10000.0,
+    "rope_type": "linear",
+    "type": "linear"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.19.0.dev0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 32
+}
diff --git a/test/fixtures/embedding_models/tiny-random-qwen3_linear-bidirectional/model.safetensors b/test/fixtures/embedding_models/tiny-random-qwen3_linear-bidirectional/model.safetensors
new file mode 100644
index 00000000..3d773f48
Binary files /dev/null and b/test/fixtures/embedding_models/tiny-random-qwen3_linear-bidirectional/model.safetensors differ
diff --git a/test/fixtures/embedding_models/tiny-random-qwen3_linear-bidirectional/reference.json b/test/fixtures/embedding_models/tiny-random-qwen3_linear-bidirectional/reference.json
new file mode 100644
index 00000000..5b78907f
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-qwen3_linear-bidirectional/reference.json
@@ -0,0 +1,585 @@
+{
+  "transformers_version": "5.19.0.dev0",
+  "transformers_commit": "3693f8d26311305e914735a6373fb03468d6aaa0",
+  "is_causal": false,
+  "input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8
+    ]
+  ],
+  "attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1
+    ]
+  ],
+  "hidden_state": [
+    [
+      [
+        2.3804707527160645,
+        -0.47000035643577576,
+        0.4663759768009186,
+        0.21189892292022705,
+        0.9254993796348572,
+        0.13283444941043854,
+        -0.5416399836540222,
+        -0.5064319372177124,
+        0.402098685503006,
+        -0.627611517906189,
+        -0.021320605650544167,
+        -0.8468173146247864,
+        -1.7060489654541016,
+        0.8879084587097168,
+        0.21694938838481903,
+        -1.829768419265747
+      ],
+      [
+        0.17248137295246124,
+        -1.4456835985183716,
+        1.3732117414474487,
+        -0.639765202999115,
+        0.5857303142547607,
+        1.8158798217773438,
+        -0.13544635474681854,
+        0.9852367639541626,
+        1.3312965631484985,
+        -1.7727092504501343,
+        -0.3874310851097107,
+        0.878923237323761,
+        -0.16760191321372986,
+        0.8765888810157776,
+        -0.290028840303421,
+        -0.4286009669303894
+      ],
+      [
+        0.5260079503059387,
+        1.255531668663025,
+        0.1318393349647522,
+        0.4174288511276245,
+        -1.7375743389129639,
+        0.09830798953771591,
+        1.8797768354415894,
+        -1.2352778911590576,
+        0.9927306175231934,
+        1.2362178564071655,
+        1.3285902738571167,
+        -0.07969822734594345,
+        0.015381804667413235,
+        0.6812264919281006,
+        -1.0378978252410889,
+        0.04090458154678345
+      ],
+      [
+        0.5007434487342834,
+        2.0936684608459473,
+        -0.30570748448371887,
+        0.2589314579963684,
+        1.0930927991867065,
+        0.7757241725921631,
+        -0.14021842181682587,
+        -1.6860812902450562,
+        -0.957875669002533,
+        0.1710398644208908,
+        1.7559993267059326,
+        1.056137204170227,
+        0.34663906693458557,
+        0.1105915978550911,
+        0.3989846110343933,
+        1.0401036739349365
+      ],
+      [
+        -1.3355661630630493,
+        -0.872868537902832,
+        -0.8113995790481567,
+        1.9513561725616455,
+        -0.2533009946346283,
+        -1.1128652095794678,
+        0.7558857202529907,
+        -1.0569686889648438,
+        -0.29127490520477295,
+        -0.5515467524528503,
+        -1.5773495435714722,
+        1.2873198986053467,
+        -0.36441829800605774,
+        -0.22161449491977692,
+        -0.1613665074110031,
+        -1.09657621383667
+      ],
+      [
+        2.7258474826812744,
+        0.09528335183858871,
+        -0.2645092308521271,
+        0.15224775671958923,
+        0.37966176867485046,
+        -1.266640067100525,
+        -1.3302533626556396,
+        0.2352428138256073,
+        -1.3847336769104004,
+        -0.9199259281158447,
+        0.49911290407180786,
+        0.512569785118103,
+        -0.2932400107383728,
+        -0.15695318579673767,
+        0.24800381064414978,
+        -1.1958178281784058
+      ],
+      [
+        -2.437811851501465,
+        1.3706233501434326,
+        -0.6130828857421875,
+        0.04555630311369896,
+        1.3398782014846802,
+        0.5457714200019836,
+        1.275175929069519,
+        -0.5850625038146973,
+        0.169952854514122,
+        -0.33303090929985046,
+        -0.9604375958442688,
+        -0.4522644281387329,
+        0.9667981266975403,
+        0.15534047782421112,
+        -1.2079795598983765,
+        -0.16259858012199402
+      ],
+      [
+        0.04562314599752426,
+        0.3919294774532318,
+        0.410649836063385,
+        0.6469319462776184,
+        0.5774237513542175,
+        2.2630908489227295,
+        -0.712445080280304,
+        -1.1138416528701782,
+        -0.7111727595329285,
+        -0.23091602325439453,
+        1.602993130683899,
+        1.1184111833572388,
+        0.023472322151064873,
+        -0.9061961770057678,
+        0.5119935274124146,
+        -1.6013227701187134
+      ]
+    ]
+  ],
+  "changed_future_hidden_state": [
+    [
+      [
+        2.475024700164795,
+        -0.6283769011497498,
+        0.29217568039894104,
+        0.2642004191875458,
+        0.8736686706542969,
+        0.2726992070674896,
+        -0.6569352746009827,
+        -0.3821741044521332,
+        0.29276666045188904,
+        -0.5534896850585938,
+        0.042307864874601364,
+        -0.8595573306083679,
+        -1.6633203029632568,
+        0.8529935479164124,
+        0.2194073349237442,
+        -1.7804899215698242
+      ],
+      [
+        0.15898320078849792,
+        -1.4966493844985962,
+        1.3001396656036377,
+        -0.6201580166816711,
+        0.530611515045166,
+        1.8932693004608154,
+        -0.2186431735754013,
+        1.0169291496276855,
+        1.337486982345581,
+        -1.7301602363586426,
+        -0.36457934975624084,
+        0.9295175671577454,
+        -0.11727128177881241,
+        0.8034771084785461,
+        -0.2423841655254364,
+        -0.39675071835517883
+      ],
+      [
+        0.4805983901023865,
+        1.233636498451233,
+        0.10775107890367508,
+        0.4749114215373993,
+        -1.7554174661636353,
+        0.18694931268692017,
+        1.918078064918518,
+        -1.1883585453033447,
+        0.9721998572349548,
+        1.2768914699554443,
+        1.3323050737380981,
+        -0.10972718149423599,
+        0.01270999200642109,
+        0.5845822095870972,
+        -1.0266444683074951,
+        0.07985126227140427
+      ],
+      [
+        0.5115131735801697,
+        2.0750560760498047,
+        -0.36482152342796326,
+        0.36730054020881653,
+        1.0504040718078613,
+        0.812953531742096,
+        -0.08275329321622849,
+        -1.6894105672836304,
+        -0.9945475459098816,
+        0.27095890045166016,
+        1.745376706123352,
+        1.0257768630981445,
+        0.3360763192176819,
+        0.08110307157039642,
+        0.36621585488319397,
+        1.0482699871063232
+      ],
+      [
+        -1.4828557968139648,
+        -1.0546871423721313,
+        -0.9412181973457336,
+        1.9966119527816772,
+        -0.38986608386039734,
+        -0.889629065990448,
+        0.8169992566108704,
+        -0.804073691368103,
+        -0.3093498647212982,
+        -0.6190943717956543,
+        -1.5029290914535522,
+        1.2116988897323608,
+        -0.2647683024406433,
+        -0.44166991114616394,
+        -0.270818829536438,
+        -0.9805724620819092
+      ],
+      [
+        2.741715431213379,
+        0.028661800548434258,
+        -0.39848828315734863,
+        0.2750968039035797,
+        0.34070122241973877,
+        -1.1863961219787598,
+        -1.319670557975769,
+        0.3108786344528198,
+        -1.470852255821228,
+        -0.8779962658882141,
+        0.5133561491966248,
+        0.4926741123199463,
+        -0.25539135932922363,
+        -0.22750741243362427,
+        0.1821253001689911,
+        -1.1306731700897217
+      ],
+      [
+        -2.441521406173706,
+        1.3566219806671143,
+        -0.6639652848243713,
+        0.09307017922401428,
+        1.3075882196426392,
+        0.6092075705528259,
+        1.3082771301269531,
+        -0.5488526821136475,
+        0.13429860770702362,
+        -0.31727075576782227,
+        -0.9035346508026123,
+        -0.45030564069747925,
+        0.9753814935684204,
+        0.12409159541130066,
+        -1.2260557413101196,
+        -0.14233292639255524
+      ],
+      [
+        -1.3802008628845215,
+        1.8135160207748413,
+        0.6494446992874146,
+        0.08106948435306549,
+        -0.7716838121414185,
+        -0.500453531742096,
+        -1.1942366361618042,
+        1.1733472347259521,
+        -0.9451490044593811,
+        -0.6638289093971252,
+        0.38905319571495056,
+        0.3025127053260803,
+        1.7114239931106567,
+        0.5295724272727966,
+        0.970282256603241,
+        -0.986286997795105
+      ]
+    ]
+  ],
+  "padded_input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_position_ids": [
+    [
+      0,
+      1,
+      2,
+      3,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_hidden_state": [
+    [
+      [
+        2.348954200744629,
+        -0.09527663886547089,
+        0.19545263051986694,
+        0.464946448802948,
+        0.6779965162277222,
+        0.45914191007614136,
+        -1.3672893047332764,
+        -0.7023479342460632,
+        0.09788250923156738,
+        0.06207561865448952,
+        0.07718004286289215,
+        -0.7484263777732849,
+        -1.9141855239868164,
+        0.9653149843215942,
+        0.7345874309539795,
+        -1.1919121742248535
+      ],
+      [
+        0.43318870663642883,
+        -1.0824134349822998,
+        1.328692078590393,
+        -0.6072807908058167,
+        0.5671329498291016,
+        2.3261961936950684,
+        -0.8367037177085876,
+        0.6042765974998474,
+        1.2848491668701172,
+        -1.3204411268234253,
+        -0.3695427477359772,
+        1.067093014717102,
+        -0.2501853108406067,
+        0.9268195629119873,
+        0.03786185756325722,
+        -0.22839462757110596
+      ],
+      [
+        0.710109293460846,
+        1.3063181638717651,
+        0.05125481262803078,
+        0.05416928604245186,
+        -1.816381812095642,
+        0.155558779835701,
+        1.4213935136795044,
+        -1.4467324018478394,
+        1.0432040691375732,
+        1.3452603816986084,
+        1.4665547609329224,
+        -0.12673895061016083,
+        0.04358247295022011,
+        0.708381712436676,
+        -0.8235551714897156,
+        0.24225881695747375
+      ],
+      [
+        0.726507842540741,
+        2.035364866256714,
+        -0.10553140193223953,
+        -0.1327599436044693,
+        1.0338947772979736,
+        0.7242819666862488,
+        -0.39529672265052795,
+        -1.8069162368774414,
+        -0.8547156453132629,
+        0.2672673463821411,
+        1.7453638315200806,
+        0.9569552540779114,
+        0.14532488584518433,
+        0.29543718695640564,
+        0.4912334084510803,
+        1.070923089981079
+      ],
+      [
+        1.0984631776809692,
+        0.6658608317375183,
+        0.4536934494972229,
+        -1.111095905303955,
+        -0.07813029736280441,
+        -0.3534058630466461,
+        -2.846869468688965,
+        -0.3757239282131195,
+        0.330267995595932,
+        -0.2717861235141754,
+        -0.46070024371147156,
+        0.9907264113426208,
+        -0.41101986169815063,
+        0.8149014711380005,
+        1.0126690864562988,
+        -0.7273316979408264
+      ],
+      [
+        1.0984631776809692,
+        0.6658608317375183,
+        0.4536934494972229,
+        -1.111095905303955,
+        -0.07813029736280441,
+        -0.3534058630466461,
+        -2.846869468688965,
+        -0.3757239282131195,
+        0.330267995595932,
+        -0.2717861235141754,
+        -0.46070024371147156,
+        0.9907264113426208,
+        -0.41101986169815063,
+        0.8149014711380005,
+        1.0126690864562988,
+        -0.7273316979408264
+      ],
+      [
+        1.0984631776809692,
+        0.6658608317375183,
+        0.4536934494972229,
+        -1.111095905303955,
+        -0.07813029736280441,
+        -0.3534058630466461,
+        -2.846869468688965,
+        -0.3757239282131195,
+        0.330267995595932,
+        -0.2717861235141754,
+        -0.46070024371147156,
+        0.9907264113426208,
+        -0.41101986169815063,
+        0.8149014711380005,
+        1.0126690864562988,
+        -0.7273316979408264
+      ],
+      [
+        1.0984631776809692,
+        0.6658608317375183,
+        0.4536934494972229,
+        -1.111095905303955,
+        -0.07813029736280441,
+        -0.3534058630466461,
+        -2.846869468688965,
+        -0.3757239282131195,
+        0.330267995595932,
+        -0.2717861235141754,
+        -0.46070024371147156,
+        0.9907264113426208,
+        -0.41101986169815063,
+        0.8149014711380005,
+        1.0126690864562988,
+        -0.7273316979408264
+      ]
+    ]
+  ],
+  "compact_hidden_state": [
+    [
+      [
+        2.348954200744629,
+        -0.09527663886547089,
+        0.19545261561870575,
+        0.464946448802948,
+        0.6779964566230774,
+        0.4591417908668518,
+        -1.367289423942566,
+        -0.702347993850708,
+        0.0978824570775032,
+        0.06207560747861862,
+        0.07718005031347275,
+        -0.7484262585639954,
+        -1.9141855239868164,
+        0.9653149843215942,
+        0.7345874309539795,
+        -1.1919121742248535
+      ],
+      [
+        0.4331887662410736,
+        -1.0824134349822998,
+        1.328692078590393,
+        -0.6072808504104614,
+        0.5671330094337463,
+        2.3261961936950684,
+        -0.8367034792900085,
+        0.6042764782905579,
+        1.2848492860794067,
+        -1.3204411268234253,
+        -0.3695427179336548,
+        1.067093014717102,
+        -0.2501853406429291,
+        0.9268195629119873,
+        0.03786187991499901,
+        -0.22839465737342834
+      ],
+      [
+        0.7101092338562012,
+        1.3063180446624756,
+        0.05125480890274048,
+        0.05416933074593544,
+        -1.8163816928863525,
+        0.155558779835701,
+        1.4213933944702148,
+        -1.4467324018478394,
+        1.0432039499282837,
+        1.3452602624893188,
+        1.4665546417236328,
+        -0.12673893570899963,
+        0.04358244687318802,
+        0.7083816528320312,
+        -0.8235551118850708,
+        0.24225875735282898
+      ],
+      [
+        0.726507842540741,
+        2.035364866256714,
+        -0.10553137212991714,
+        -0.1327599436044693,
+        1.0338947772979736,
+        0.7242819666862488,
+        -0.39529675245285034,
+        -1.8069162368774414,
+        -0.8547155857086182,
+        0.2672673761844635,
+        1.7453638315200806,
+        0.9569552540779114,
+        0.14532488584518433,
+        0.29543718695640564,
+        0.4912334084510803,
+        1.0709229707717896
+      ]
+    ]
+  ]
+}
diff --git a/test/fixtures/embedding_models/tiny-random-qwen3_linear-causal/config.json b/test/fixtures/embedding_models/tiny-random-qwen3_linear-causal/config.json
new file mode 100644
index 00000000..afd5053b
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-qwen3_linear-causal/config.json
@@ -0,0 +1,40 @@
+{
+  "architectures": [
+    "Qwen3Model"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 8,
+  "hidden_act": "silu",
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 20,
+  "is_causal": true,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 4,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "factor": 2.0,
+    "rope_theta": 10000.0,
+    "rope_type": "linear",
+    "type": "linear"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.19.0.dev0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 32
+}
diff --git a/test/fixtures/embedding_models/tiny-random-qwen3_linear-causal/model.safetensors b/test/fixtures/embedding_models/tiny-random-qwen3_linear-causal/model.safetensors
new file mode 100644
index 00000000..43c49374
Binary files /dev/null and b/test/fixtures/embedding_models/tiny-random-qwen3_linear-causal/model.safetensors differ
diff --git a/test/fixtures/embedding_models/tiny-random-qwen3_linear-causal/reference.json b/test/fixtures/embedding_models/tiny-random-qwen3_linear-causal/reference.json
new file mode 100644
index 00000000..794e92f8
--- /dev/null
+++ b/test/fixtures/embedding_models/tiny-random-qwen3_linear-causal/reference.json
@@ -0,0 +1,585 @@
+{
+  "transformers_version": "5.19.0.dev0",
+  "transformers_commit": "3693f8d26311305e914735a6373fb03468d6aaa0",
+  "is_causal": true,
+  "input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8
+    ]
+  ],
+  "attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1
+    ]
+  ],
+  "hidden_state": [
+    [
+      [
+        -1.332697868347168,
+        0.7285035252571106,
+        0.2462701052427292,
+        0.9126808643341064,
+        0.49572375416755676,
+        -0.055270370095968246,
+        0.5712628960609436,
+        -0.10520800203084946,
+        -0.4197082221508026,
+        0.16006706655025482,
+        2.513075828552246,
+        0.5101444125175476,
+        -0.8519627451896667,
+        2.007155418395996,
+        -0.777197539806366,
+        0.19769108295440674
+      ],
+      [
+        -1.1066310405731201,
+        0.2999636232852936,
+        0.1991424709558487,
+        0.3156890273094177,
+        -0.5695977210998535,
+        0.2666531205177307,
+        0.717149555683136,
+        -0.29553404450416565,
+        1.3534818887710571,
+        -1.0769379138946533,
+        1.1365094184875488,
+        1.9484788179397583,
+        -0.9598391056060791,
+        -0.040897298604249954,
+        -0.707036018371582,
+        1.9969927072525024
+      ],
+      [
+        0.39589110016822815,
+        -1.3712940216064453,
+        0.988987922668457,
+        0.5564053654670715,
+        -0.9279597997665405,
+        0.989714503288269,
+        0.7368783354759216,
+        -0.4904135763645172,
+        -0.4365149438381195,
+        -0.20802313089370728,
+        1.819280982017517,
+        -2.12956166267395,
+        0.9164615869522095,
+        -0.7529182434082031,
+        -0.5852075815200806,
+        0.4306463599205017
+      ],
+      [
+        0.20449954271316528,
+        0.0890413224697113,
+        -0.773203432559967,
+        0.752077579498291,
+        -0.1044410914182663,
+        1.0855385065078735,
+        -0.28461599349975586,
+        -1.2774797677993774,
+        -0.7137797474861145,
+        -1.2971302270889282,
+        0.9671080708503723,
+        0.5762349963188171,
+        0.7520018815994263,
+        -1.6377841234207153,
+        0.5740512609481812,
+        -2.189971685409546
+      ],
+      [
+        0.559679388999939,
+        -1.5600589513778687,
+        -0.616054356098175,
+        -0.8498685359954834,
+        0.846074640750885,
+        -1.996099829673767,
+        0.03344755619764328,
+        -0.8699432015419006,
+        0.8233052492141724,
+        0.9942224025726318,
+        1.3975518941879272,
+        -0.2718188762664795,
+        1.1883363723754883,
+        -0.9908868670463562,
+        0.6362152695655823,
+        0.42185166478157043
+      ],
+      [
+        2.350285768508911,
+        -0.3711071312427521,
+        -0.7534564733505249,
+        0.5843968391418457,
+        -1.5700724124908447,
+        1.3491671085357666,
+        0.34152090549468994,
+        -1.5185502767562866,
+        0.684260904788971,
+        0.12176615744829178,
+        0.1520400047302246,
+        0.21613669395446777,
+        0.8942729830741882,
+        -0.6035752296447754,
+        -0.9133085012435913,
+        -0.3742775321006775
+      ],
+      [
+        0.5736373662948608,
+        -0.6041820645332336,
+        -0.8257227540016174,
+        0.102457694709301,
+        0.13089315593242645,
+        -0.5561263561248779,
+        0.7387528419494629,
+        -0.3921845555305481,
+        -2.482466697692871,
+        0.4435667097568512,
+        -1.0529035329818726,
+        -0.880914568901062,
+        -1.6230337619781494,
+        -0.9740861058235168,
+        1.2443904876708984,
+        0.4275036156177521
+      ],
+      [
+        -0.4798935055732727,
+        1.137915849685669,
+        0.14236591756343842,
+        -1.7746542692184448,
+        -0.02988196164369583,
+        0.15443821251392365,
+        0.46419161558151245,
+        -1.2310506105422974,
+        1.1666206121444702,
+        0.551112949848175,
+        1.4211466312408447,
+        0.4597647190093994,
+        2.011002540588379,
+        -1.1744412183761597,
+        0.3702611029148102,
+        0.24859091639518738
+      ]
+    ]
+  ],
+  "changed_future_hidden_state": [
+    [
+      [
+        -1.332697868347168,
+        0.7285035252571106,
+        0.2462701052427292,
+        0.9126808643341064,
+        0.49572375416755676,
+        -0.055270370095968246,
+        0.5712628960609436,
+        -0.10520800203084946,
+        -0.4197082221508026,
+        0.16006706655025482,
+        2.513075828552246,
+        0.5101444125175476,
+        -0.8519627451896667,
+        2.007155418395996,
+        -0.777197539806366,
+        0.19769108295440674
+      ],
+      [
+        -1.1066310405731201,
+        0.2999636232852936,
+        0.1991424709558487,
+        0.3156890273094177,
+        -0.5695977210998535,
+        0.2666531205177307,
+        0.717149555683136,
+        -0.29553404450416565,
+        1.3534818887710571,
+        -1.0769379138946533,
+        1.1365094184875488,
+        1.9484788179397583,
+        -0.9598391056060791,
+        -0.040897298604249954,
+        -0.707036018371582,
+        1.9969927072525024
+      ],
+      [
+        0.39589110016822815,
+        -1.3712940216064453,
+        0.988987922668457,
+        0.5564053654670715,
+        -0.9279597997665405,
+        0.989714503288269,
+        0.7368783354759216,
+        -0.4904135763645172,
+        -0.4365149438381195,
+        -0.20802313089370728,
+        1.819280982017517,
+        -2.12956166267395,
+        0.9164615869522095,
+        -0.7529182434082031,
+        -0.5852075815200806,
+        0.4306463599205017
+      ],
+      [
+        0.20449954271316528,
+        0.0890413224697113,
+        -0.773203432559967,
+        0.752077579498291,
+        -0.1044410914182663,
+        1.0855385065078735,
+        -0.28461599349975586,
+        -1.2774797677993774,
+        -0.7137797474861145,
+        -1.2971302270889282,
+        0.9671080708503723,
+        0.5762349963188171,
+        0.7520018815994263,
+        -1.6377841234207153,
+        0.5740512609481812,
+        -2.189971685409546
+      ],
+      [
+        0.559679388999939,
+        -1.5600589513778687,
+        -0.616054356098175,
+        -0.8498685359954834,
+        0.846074640750885,
+        -1.996099829673767,
+        0.03344755619764328,
+        -0.8699432015419006,
+        0.8233052492141724,
+        0.9942224025726318,
+        1.3975518941879272,
+        -0.2718188762664795,
+        1.1883363723754883,
+        -0.9908868670463562,
+        0.6362152695655823,
+        0.42185166478157043
+      ],
+      [
+        2.350285768508911,
+        -0.3711071312427521,
+        -0.7534564733505249,
+        0.5843968391418457,
+        -1.5700724124908447,
+        1.3491671085357666,
+        0.34152090549468994,
+        -1.5185502767562866,
+        0.684260904788971,
+        0.12176615744829178,
+        0.1520400047302246,
+        0.21613669395446777,
+        0.8942729830741882,
+        -0.6035752296447754,
+        -0.9133085012435913,
+        -0.3742775321006775
+      ],
+      [
+        0.5736373662948608,
+        -0.6041820645332336,
+        -0.8257227540016174,
+        0.102457694709301,
+        0.13089315593242645,
+        -0.5561263561248779,
+        0.7387528419494629,
+        -0.3921845555305481,
+        -2.482466697692871,
+        0.4435667097568512,
+        -1.0529035329818726,
+        -0.880914568901062,
+        -1.6230337619781494,
+        -0.9740861058235168,
+        1.2443904876708984,
+        0.4275036156177521
+      ],
+      [
+        0.20173311233520508,
+        -0.9515380263328552,
+        0.004179876763373613,
+        1.1135282516479492,
+        0.5165417790412903,
+        1.2093756198883057,
+        0.28589048981666565,
+        0.4498256742954254,
+        -0.8937991857528687,
+        -1.5571949481964111,
+        1.2252867221832275,
+        -0.8017433285713196,
+        0.490538090467453,
+        -0.44763776659965515,
+        -1.0989993810653687,
+        2.1681644916534424
+      ]
+    ]
+  ],
+  "padded_input_ids": [
+    [
+      1,
+      2,
+      3,
+      4,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_attention_mask": [
+    [
+      1,
+      1,
+      1,
+      1,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_position_ids": [
+    [
+      0,
+      1,
+      2,
+      3,
+      0,
+      0,
+      0,
+      0
+    ]
+  ],
+  "padded_hidden_state": [
+    [
+      [
+        -1.332697868347168,
+        0.7285035252571106,
+        0.2462701052427292,
+        0.9126808643341064,
+        0.49572375416755676,
+        -0.055270370095968246,
+        0.5712628960609436,
+        -0.10520800203084946,
+        -0.4197082221508026,
+        0.16006706655025482,
+        2.513075828552246,
+        0.5101444125175476,
+        -0.8519627451896667,
+        2.007155418395996,
+        -0.777197539806366,
+        0.19769108295440674
+      ],
+      [
+        -1.1066310405731201,
+        0.2999636232852936,
+        0.1991424709558487,
+        0.3156890273094177,
+        -0.5695977210998535,
+        0.2666531205177307,
+        0.717149555683136,
+        -0.29553404450416565,
+        1.3534818887710571,
+        -1.0769379138946533,
+        1.1365094184875488,
+        1.9484788179397583,
+        -0.9598391056060791,
+        -0.040897298604249954,
+        -0.707036018371582,
+        1.9969927072525024
+      ],
+      [
+        0.39589110016822815,
+        -1.3712940216064453,
+        0.988987922668457,
+        0.5564053654670715,
+        -0.9279597997665405,
+        0.989714503288269,
+        0.7368783354759216,
+        -0.4904135763645172,
+        -0.4365149438381195,
+        -0.20802313089370728,
+        1.819280982017517,
+        -2.12956166267395,
+        0.9164615869522095,
+        -0.7529182434082031,
+        -0.5852075815200806,
+        0.4306463599205017
+      ],
+      [
+        0.20449954271316528,
+        0.0890413224697113,
+        -0.773203432559967,
+        0.752077579498291,
+        -0.1044410914182663,
+        1.0855385065078735,
+        -0.28461599349975586,
+        -1.2774797677993774,
+        -0.7137797474861145,
+        -1.2971302270889282,
+        0.9671080708503723,
+        0.5762349963188171,
+        0.7520018815994263,
+        -1.6377841234207153,
+        0.5740512609481812,
+        -2.189971685409546
+      ],
+      [
+        -0.8472859263420105,
+        -0.5972305536270142,
+        -0.8438308835029602,
+        1.7743812799453735,
+        0.7173834443092346,
+        -1.123368501663208,
+        0.6549981236457825,
+        -0.7230342626571655,
+        0.09419386088848114,
+        -0.4039069712162018,
+        1.8323605060577393,
+        -0.30406713485717773,
+        -1.653549075126648,
+        -0.8194564580917358,
+        0.6959991455078125,
+        -0.2258853167295456
+      ],
+      [
+        -0.8472859263420105,
+        -0.5972305536270142,
+        -0.8438308835029602,
+        1.7743812799453735,
+        0.7173834443092346,
+        -1.123368501663208,
+        0.6549981236457825,
+        -0.7230342626571655,
+        0.09419386088848114,
+        -0.4039069712162018,
+        1.8323605060577393,
+        -0.30406713485717773,
+        -1.653549075126648,
+        -0.8194564580917358,
+        0.6959991455078125,
+        -0.2258853167295456
+      ],
+      [
+        -0.8472859263420105,
+        -0.5972305536270142,
+        -0.8438308835029602,
+        1.7743812799453735,
+        0.7173834443092346,
+        -1.123368501663208,
+        0.6549981236457825,
+        -0.7230342626571655,
+        0.09419386088848114,
+        -0.4039069712162018,
+        1.8323605060577393,
+        -0.30406713485717773,
+        -1.653549075126648,
+        -0.8194564580917358,
+        0.6959991455078125,
+        -0.2258853167295456
+      ],
+      [
+        -0.8472859263420105,
+        -0.5972305536270142,
+        -0.8438308835029602,
+        1.7743812799453735,
+        0.7173834443092346,
+        -1.123368501663208,
+        0.6549981236457825,
+        -0.7230342626571655,
+        0.09419386088848114,
+        -0.4039069712162018,
+        1.8323605060577393,
+        -0.30406713485717773,
+        -1.653549075126648,
+        -0.8194564580917358,
+        0.6959991455078125,
+        -0.2258853167295456
+      ]
+    ]
+  ],
+  "compact_hidden_state": [
+    [
+      [
+        -1.332697868347168,
+        0.7285035252571106,
+        0.2462701052427292,
+        0.9126808643341064,
+        0.49572375416755676,
+        -0.055270370095968246,
+        0.5712628960609436,
+        -0.10520800203084946,
+        -0.4197082221508026,
+        0.16006706655025482,
+        2.513075828552246,
+        0.5101444125175476,
+        -0.8519627451896667,
+        2.007155418395996,
+        -0.777197539806366,
+        0.19769108295440674
+      ],
+      [
+        -1.1066310405731201,
+        0.2999635934829712,
+        0.19914251565933228,
+        0.3156890273094177,
+        -0.569597601890564,
+        0.2666531503200531,
+        0.717149555683136,
+        -0.29553407430648804,
+        1.3534820079803467,
+        -1.0769380331039429,
+        1.1365094184875488,
+        1.9484788179397583,
+        -0.9598391056060791,
+        -0.040897272527217865,
+        -0.7070360779762268,
+        1.9969927072525024
+      ],
+      [
+        0.3958910405635834,
+        -1.3712940216064453,
+        0.9889881014823914,
+        0.5564053654670715,
+        -0.9279597997665405,
+        0.9897145628929138,
+        0.7368783950805664,
+        -0.4904135763645172,
+        -0.4365150034427643,
+        -0.20802317559719086,
+        1.819280982017517,
+        -2.12956166267395,
+        0.9164614677429199,
+        -0.7529183030128479,
+        -0.5852077007293701,
+        0.4306463897228241
+      ],
+      [
+        0.2044994831085205,
+        0.08904127776622772,
+        -0.773203432559967,
+        0.752077579498291,
+        -0.10444121062755585,
+        1.0855385065078735,
+        -0.2846159338951111,
+        -1.2774797677993774,
+        -0.7137797474861145,
+        -1.2971302270889282,
+        0.9671080708503723,
+        0.5762349367141724,
+        0.7520018219947815,
+        -1.6377841234207153,
+        0.5740513205528259,
+        -2.189971685409546
+      ]
+    ]
+  ]
+}