diff --git a/scripts/generate_wrappers.py b/scripts/generate_wrappers.py
index ef54e4073..9b0256146 100644
--- a/scripts/generate_wrappers.py
+++ b/scripts/generate_wrappers.py
@@ -605,7 +605,6 @@ def _generate_call(op_name, call, method=True):
             )
             py_args = _generate_py_args(call)
             py_args_str = f"{py_args}, " if py_args else ""
-            default_impl_index = _default_impl_index_expr(call)
 
             return (
                 f'  m.def("{op_name}", []({params}) {{\n'
@@ -981,8 +980,7 @@ def _append_optional_params(prefix, params):
     symbol_name = _op_symbol_name(operator.name)
     op_type = _op_cpp_type(operator.name)
     declarations = [
-        f"std::vector<std::size_t> ActiveImplementationIndicesFor"
-        f"{symbol_name}(Device::Type dev_type);"
+        f"std::vector<std::size_t> ActiveImplementationIndicesFor{symbol_name}(Device::Type dev_type);"
     ]
     definitions = [
         f"""std::vector<std::size_t> ActiveImplementationIndicesFor{symbol_name}(Device::Type dev_type) {{
@@ -1127,10 +1125,7 @@ def _is_optional_tensor(arg):
         if arg.spelling in optional_non_tensor_params:
             return False
 
-        if arg.spelling in optional_tensor_params:
-            return True
-
-        return False
+        return arg.spelling in optional_tensor_params
 
     def _is_vector_tensor(arg):
         if arg.spelling in vector_tensor_params:
@@ -1667,33 +1662,20 @@ def _dispatch_gen_batch_size():
 
     if use_monolithic_bindings:
         op_includes = "\n".join(op_includes)
-        ops_source = f"""#include <pybind11/pybind11.h>
-
-// Generated with `INFINI_OPS_MONOLITHIC_BINDINGS=1`.
+        binding_preamble = f"""// Generated with `INFINI_OPS_MONOLITHIC_BINDINGS=1`.
 {op_includes}
-
-#include "tuning.h"
-
-namespace infini::ops {{
-
-PYBIND11_MODULE(ops, m) {{
-  const char* tuning_path = std::getenv("INFINI_OPS_TUNING_PATH");
-  if (!tuning_path) {{
-    tuning_path = "tuning.json";
-  }}
-  infini::ops::TuningManager::Instance().LoadTuningCache(tuning_path);
-{textwrap.indent(bind_func_calls, _INDENTATION)}
-}}
-
-}}  // namespace infini::ops
 """
+        bind_func_declarations = ""
     else:
+        binding_preamble = ""
         bind_func_declarations = "\n".join(
             f"void {bind_func_name}(pybind11::module& m);"
             for bind_func_name in bind_func_names
         )
-        ops_source = f"""#include <pybind11/pybind11.h>
 
+    ops_source = f"""#include <pybind11/pybind11.h>
+
+{binding_preamble}
 #include "tuning.h"
 
 namespace infini::ops {{
@@ -1701,11 +1683,7 @@ def _dispatch_gen_batch_size():
 {bind_func_declarations}
 
 PYBIND11_MODULE(ops, m) {{
-  const char* tuning_path = std::getenv("INFINI_OPS_TUNING_PATH");
-  if (!tuning_path) {{
-    tuning_path = "tuning.json";
-  }}
-  infini::ops::TuningManager::Instance().LoadTuningCache(tuning_path);
+  TuningManager::Instance().InitializeFromEnvironment();
 {textwrap.indent(bind_func_calls, _INDENTATION)}
 }}
 
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 213163f91..c6e8bd292 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -35,12 +35,28 @@ endfunction()
 
 include(GNUInstallDirs)
 
+find_package(nlohmann_json 3.12.0 CONFIG QUIET)
+if(NOT TARGET nlohmann_json::nlohmann_json)
+    if(POLICY CMP0135)
+        cmake_policy(SET CMP0135 NEW)
+    endif()
+    include(FetchContent)
+    FetchContent_Declare(nlohmann_json
+        URL https://github.com/nlohmann/json/releases/download/v3.12.0/json.tar.xz
+        URL_HASH SHA256=42f6e95cad6ec532fd372391373363b62a14af6d771056dbfc86160e6dfff7aa
+    )
+    FetchContent_MakeAvailable(nlohmann_json)
+endif()
+
 file(GLOB BASE_SRCS CONFIGURE_DEPENDS "*.cc")
 list(FILTER BASE_SRCS EXCLUDE REGEX ".*tensor\\.cc$")
 
 target_sources(infiniops PRIVATE ${BASE_SRCS})
 
-target_link_libraries(infiniops PUBLIC infinirt)
+target_link_libraries(infiniops
+    PUBLIC infinirt
+    PRIVATE nlohmann_json::nlohmann_json
+)
 
 set(INFINI_RT_INCLUDE_FLAGS "")
 foreach(_include_dir IN LISTS INFINI_RT_INCLUDE_DIRS)
diff --git a/src/config.h b/src/config.h
index 5c4bfa71a..69179d83f 100644
--- a/src/config.h
+++ b/src/config.h
@@ -2,23 +2,24 @@
 #define INFINI_OPS_CONFIG_H_
 
 #include <cstddef>
+#include <optional>
 
 namespace infini::ops {
 
 class Config {
  public:
-  std::size_t implementation_index() const { return implementation_index_; }
+  std::size_t implementation_index() const {
+    return implementation_index_.value_or(0);
+  }
 
   void set_implementation_index(std::size_t implementation_index) {
     implementation_index_ = implementation_index;
-    auto_select_ = false;
   }
 
-  bool auto_select() const { return auto_select_; }
+  bool auto_select() const { return !implementation_index_.has_value(); }
 
  private:
-  std::size_t implementation_index_{0};
-  bool auto_select_{true};
+  std::optional<std::size_t> implementation_index_;
 };
 
 }  // namespace infini::ops
diff --git a/src/operator.h b/src/operator.h
index 8d5eb0334..57ad2f51b 100644
--- a/src/operator.h
+++ b/src/operator.h
@@ -4,12 +4,12 @@
 #include <algorithm>
 #include <cassert>
 #include <chrono>
-#include <cstdlib>
 #include <iostream>
 #include <limits>
 #include <memory>
 #include <optional>
 #include <string>
+#include <string_view>
 #include <tuple>
 #include <type_traits>
 #include <unordered_map>
@@ -22,7 +22,6 @@
 #include "runtime.h"
 #include "tensor.h"
 #include "tuning.h"
-#include "tuning_utils.h"
 
 namespace infini::ops::detail {
 
@@ -78,6 +77,35 @@ std::vector<std::size_t> ListToVector(List<values...>) {
   return {static_cast<std::size_t>(values)...};
 }
 
+template <typename Key>
+std::string ExtractOperatorName() {
+#if defined(__GNUC__) || defined(__clang__)
+  std::string_view signature = __PRETTY_FUNCTION__;
+  auto key_position = signature.find("Key = ");
+  if (key_position == std::string_view::npos) return "UnknownOp";
+
+  key_position += 6;
+  auto end_position = signature.find_first_of("]>;", key_position);
+  std::string full_name{
+      signature.substr(key_position, end_position - key_position)};
+#elif defined(_MSC_VER)
+  std::string_view signature = __FUNCSIG__;
+  auto key_position = signature.find("Key=");
+  if (key_position == std::string_view::npos) return "UnknownOp";
+
+  key_position += 4;
+  auto end_position = signature.find_first_of("]>,", key_position);
+  std::string full_name{
+      signature.substr(key_position, end_position - key_position)};
+#else
+  return "UnknownOp";
+#endif
+
+  auto last_colon = full_name.rfind("::");
+  return last_colon == std::string::npos ? full_name
+                                         : full_name.substr(last_colon + 2);
+}
+
 template <typename ValueType, auto... values>
 bool ListContains(ValueType value, List<values...>) {
   return ((value == static_cast<ValueType>(values)) || ...);
@@ -96,6 +124,21 @@ inline void SyncDevice(Device::Type dev_type) {
       "SyncDevice");
 }
 
+inline Device::Type FirstDeviceType() { return Device::Type::kCount; }
+
+template <typename First, typename... Rest>
+Device::Type FirstDeviceType(const First& first, const Rest&... rest) {
+  if constexpr (std::is_same_v<std::decay_t<First>, Tensor>) {
+    return first.device().type();
+  } else if constexpr (std::is_same_v<std::decay_t<First>,
+                                      std::vector<Tensor>>) {
+    return first.empty() ? FirstDeviceType(rest...)
+                         : first.front().device().type();
+  } else {
+    return FirstDeviceType(rest...);
+  }
+}
+
 template <typename TensorLike, typename = void>
 class IsTensorLike : public std::false_type {};
 
@@ -179,7 +222,7 @@ Config ResolveConfig(const Config& config, Device::Type dev_type,
 
 template <typename Key, typename... Args>
 Config ResolveConfigOnline(const Handle& handle, const Config& config,
-                           const Args&... args);
+                           Device::Type dev_type, const Args&... args);
 
 template <typename Key, Device::Type kDev>
 struct ActiveImplementations;
@@ -228,8 +271,8 @@ class Operator : public OperatorBase {
                                         const Tensor tensor, Args&&... args) {
     Config resolved =
         ResolveConfig<Key>(config, tensor.device().type(), tensor, args...);
-    return MakeWithDevice(resolved, tensor.device().type(), tensor,
-                          std::forward<Args>(args)...);
+    return MakeResolved(resolved, tensor.device().type(), tensor,
+                        std::forward<Args>(args)...);
   }
 
   template <typename... Args>
@@ -245,8 +288,8 @@ class Operator : public OperatorBase {
 
     Config resolved = ResolveConfig<Key>(
         config, tensors.front().device().type(), tensors, args...);
-    return MakeWithDevice(resolved, tensors.front().device().type(), tensors,
-                          std::forward<Args>(args)...);
+    return MakeResolved(resolved, tensors.front().device().type(), tensors,
+                        std::forward<Args>(args)...);
   }
 
   template <typename... Args>
@@ -268,15 +311,22 @@ class Operator : public OperatorBase {
       generation = cache_generation_;
     }
 
+    const auto dev_type = detail::FirstDeviceType(args...);
+    assert(dev_type != Device::Type::kCount &&
+           "operator call requires at least one tensor argument");
+
     const Config effective_config =
-        ResolveConfigOnline<Key>(handle, config, args...);
+        ResolveConfigOnline<Key>(handle, config, dev_type, args...);
 
     auto key = CacheKeyBuilder<Key>{}(effective_config, args...);
 
     auto it{cache.find(key)};
 
     if (it == cache.end()) {
-      it = cache.emplace(std::move(key), Make(effective_config, args...)).first;
+      it = cache
+               .emplace(std::move(key),
+                        MakeResolved(effective_config, dev_type, args...))
+               .first;
     }
 
     auto& op{it->second};
@@ -345,7 +395,7 @@ class Operator : public OperatorBase {
   }
 
   template <typename... Args>
-  static std::unique_ptr<Operator> MakeWithDevice(
+  static std::unique_ptr<Operator> MakeResolved(
       const Config& config, Device::Type dispatch_device_type, Args&&... args) {
     std::unique_ptr<Operator> op_ptr;
     auto cache_args = std::forward_as_tuple(args...);
@@ -433,37 +483,35 @@ struct ActiveImplementations {
 template <typename Key, typename... Args>
 Config ResolveConfig(const Config& config, Device::Type dev_type,
                      const Args&... args) {
-  if (config.auto_select()) {
-    auto indices = Operator<Key>::active_implementation_indices(dev_type);
-    if (!indices.empty()) {
-      auto signature = TuningSignature::Build(args...);
-
-      auto op_name = detail::ExtractOperatorName<Key>();
-      auto tuned_index =
-          TuningManager::Instance().Lookup(op_name, dev_type, signature);
-
-      Config resolved = config;
-      if (tuned_index.has_value()) {
-        bool is_valid = std::find(indices.begin(), indices.end(),
-                                  *tuned_index) != indices.end();
-        if (is_valid) {
-          resolved.set_implementation_index(*tuned_index);
-        } else {
-          std::cerr << "[Tuning] Warning: tuned implementation " << *tuned_index
-                    << " for " << op_name << " on "
-                    << Device::StringFromType(dev_type)
-                    << " is not available (compiled indices:";
-          for (auto idx : indices) std::cerr << " " << idx;
-          std::cerr << "), falling back to " << indices.front() << std::endl;
-          resolved.set_implementation_index(indices.front());
-        }
-      } else {
-        resolved.set_implementation_index(indices.front());
-      }
-      return resolved;
+  if (!config.auto_select()) return config;
+
+  auto indices = Operator<Key>::active_implementation_indices(dev_type);
+  if (indices.empty()) return config;
+
+  auto signature = TuningSignature::Build(args...);
+  auto op_name = detail::ExtractOperatorName<Key>();
+  auto tuned_index =
+      TuningManager::Instance().Lookup(op_name, dev_type, signature);
+  auto chosen = indices.front();
+
+  if (tuned_index.has_value()) {
+    bool is_valid = std::find(indices.begin(), indices.end(), *tuned_index) !=
+                    indices.end();
+    if (is_valid) {
+      chosen = *tuned_index;
+    } else {
+      std::cerr << "[Tuning] Warning: tuned implementation " << *tuned_index
+                << " for " << op_name << " on "
+                << Device::StringFromType(dev_type)
+                << " is not available (compiled indices:";
+      for (auto idx : indices) std::cerr << " " << idx;
+      std::cerr << "), falling back to " << chosen << std::endl;
     }
   }
-  return config;
+
+  Config resolved = config;
+  resolved.set_implementation_index(chosen);
+  return resolved;
 }
 
 template <typename Key, typename... Args>
@@ -473,12 +521,10 @@ double BenchmarkImplementation(const Handle& handle, Device::Type dev_type,
   fixed.set_implementation_index(impl_index);
 
   auto op = Operator<Key>::Make(fixed, args...);
-  if (!op) {
-    return std::numeric_limits<double>::infinity();
-  }
 
-  const int warmup = detail::EnvInt("INFINI_OPS_TUNING_WARMUP", 1);
-  const int repeat = detail::EnvInt("INFINI_OPS_TUNING_REPEAT", 5);
+  const auto& tuning = TuningManager::Instance();
+  const int warmup = tuning.warmup_count();
+  const int repeat = tuning.repeat_count();
 
   for (int i = 0; i < warmup; ++i) {
     (*op)(handle, args...);
@@ -499,57 +545,52 @@ double BenchmarkImplementation(const Handle& handle, Device::Type dev_type,
 
 template <typename Key, typename... Args>
 Config ResolveConfigOnline(const Handle& handle, const Config& config,
-                           const Args&... args) {
-  if (config.auto_select() && TuningManager::Instance().IsEnabled()) {
-    Device::Type dev_type = detail::FirstDeviceType(args...);
-    auto indices = Operator<Key>::active_implementation_indices(dev_type);
-
-    if (!indices.empty()) {
-      auto signature = TuningSignature::Build(args...);
-      auto op_name = detail::ExtractOperatorName<Key>();
-
-      auto tuned =
-          TuningManager::Instance().Lookup(op_name, dev_type, signature);
-
-      std::size_t chosen;
-      if (tuned.has_value() &&
-          std::find(indices.begin(), indices.end(), *tuned) != indices.end()) {
-        chosen = *tuned;
-      } else {
-        if (indices.size() == 1) {
-          chosen = indices.front();
-          TuningManager::Instance().Record(op_name, dev_type, signature,
-                                           chosen);
-          std::cout << "[Tuning] " << op_name << " on "
-                    << Device::StringFromType(dev_type)
-                    << ": single impl, chose index " << chosen << std::endl;
-        } else {
-          chosen = indices.front();
-          double best_time = std::numeric_limits<double>::infinity();
-          for (auto idx : indices) {
-            double t =
-                BenchmarkImplementation<Key>(handle, dev_type, idx, args...);
-            if (t < best_time) {
-              best_time = t;
-              chosen = idx;
-            }
-          }
-          TuningManager::Instance().Record(op_name, dev_type, signature,
-                                           chosen);
-          std::cout << "[Tuning] " << op_name << " on "
-                    << Device::StringFromType(dev_type) << ": benchmarked "
-                    << indices.size() << " impls, chose index " << chosen
-                    << " (" << best_time * 1e6 << " us)" << std::endl;
-        }
-      }
+                           Device::Type dev_type, const Args&... args) {
+  if (!config.auto_select()) return config;
 
-      Config resolved = config;
-      resolved.set_implementation_index(chosen);
-      return resolved;
+  auto& tuning = TuningManager::Instance();
+  if (!tuning.IsEnabled()) {
+    return ResolveConfig<Key>(config, dev_type, args...);
+  }
+
+  auto indices = Operator<Key>::active_implementation_indices(dev_type);
+  if (indices.empty()) return config;
+
+  auto signature = TuningSignature::Build(args...);
+  auto op_name = detail::ExtractOperatorName<Key>();
+  auto tuned = tuning.Lookup(op_name, dev_type, signature);
+  std::size_t chosen;
+
+  if (tuned.has_value() &&
+      std::find(indices.begin(), indices.end(), *tuned) != indices.end()) {
+    chosen = *tuned;
+  } else if (indices.size() == 1) {
+    chosen = indices.front();
+    tuning.Record(op_name, dev_type, signature, chosen);
+    std::cout << "[Tuning] " << op_name << " on "
+              << Device::StringFromType(dev_type)
+              << ": single impl, chose index " << chosen << std::endl;
+  } else {
+    chosen = indices.front();
+    double best_time = std::numeric_limits<double>::infinity();
+    for (auto idx : indices) {
+      double time =
+          BenchmarkImplementation<Key>(handle, dev_type, idx, args...);
+      if (time < best_time) {
+        best_time = time;
+        chosen = idx;
+      }
     }
+    tuning.Record(op_name, dev_type, signature, chosen);
+    std::cout << "[Tuning] " << op_name << " on "
+              << Device::StringFromType(dev_type) << ": benchmarked "
+              << indices.size() << " impls, chose index " << chosen << " ("
+              << best_time * 1e6 << " us)" << std::endl;
   }
-  (void)handle;
-  return config;
+
+  Config resolved = config;
+  resolved.set_implementation_index(chosen);
+  return resolved;
 }
 
 }  // namespace infini::ops
diff --git a/src/tuning.cc b/src/tuning.cc
index 8c5c4a672..8d7b246db 100644
--- a/src/tuning.cc
+++ b/src/tuning.cc
@@ -1,73 +1,109 @@
 #include "tuning.h"
 
+#include <cstdlib>
 #include <fstream>
 #include <iostream>
-#include <sstream>
+#include <nlohmann/json.hpp>
+#include <string_view>
+#include <utility>
+
+namespace infini::ops {
 
 namespace {
 
-void SkipWhitespace(std::istream& in) {
-  while (in && std::isspace(in.peek())) {
-    in.get();
-  }
+using Json = nlohmann::json;
+
+constexpr int kTuningCacheVersion = 1;
+constexpr char kDefaultTuningPath[] = "tuning.json";
+
+int EnvInt(const char* name, int fallback) {
+  const char* value = std::getenv(name);
+  if (!value || !*value) return fallback;
+
+  int parsed = std::atoi(value);
+  return parsed > 0 ? parsed : fallback;
 }
 
-std::string ParseString(std::istream& in) {
-  SkipWhitespace(in);
-  if (in.get() != '"') return "";
-  std::string result;
-  while (in) {
-    char c = in.get();
-    if (c == '"') break;
-    if (c == '\\') {
-      c = in.get();
-    }
-    result += c;
-  }
-  return result;
+const Json* FindMember(const Json& object, const char* name) {
+  if (!object.is_object()) return nullptr;
+
+  auto iterator = object.find(name);
+  return iterator == object.end() ? nullptr : &*iterator;
 }
 
-double ParseNumber(std::istream& in) {
-  SkipWhitespace(in);
-  double val = 0;
-  in >> val;
-  return val;
+bool IsInteger(const Json& value) {
+  return value.is_number_integer() || value.is_number_unsigned();
 }
 
-int64_t ParseInteger(std::istream& in) {
-  SkipWhitespace(in);
-  int64_t val = 0;
-  in >> val;
-  return val;
+template <auto... device_types>
+std::optional<Device::Type> DeviceTypeFromString(std::string_view name,
+                                                 List<device_types...>) {
+  const Device::Type types[]{device_types...};
+  for (auto type : types) {
+    if (name == Device::StringFromType(type)) return type;
+  }
+  return std::nullopt;
 }
 
-void SkipTo(std::istream& in, char target) {
-  while (in && in.get() != target) {
+Json SignatureToJson(const TuningSignature& signature) {
+  Json tensors = Json::array();
+  for (const auto& tensor : signature.tensors) {
+    tensors.push_back(
+        {{"shape", tensor.shape}, {"dtype", static_cast<int>(tensor.dtype)}});
   }
+
+  return {{"tensors", std::move(tensors)}, {"scalars", signature.scalars}};
 }
 
-std::string NextKey(std::istream& in) {
-  SkipWhitespace(in);
-  if (in.peek() == '}' || in.peek() == ']') return "";
-  if (in.peek() == ',') in.get();
-  SkipWhitespace(in);
-  if (in.peek() == '"') {
-    auto key = ParseString(in);
-    SkipTo(in, ':');
-    return key;
+std::optional<TuningSignature> SignatureFromJson(const Json& value) {
+  const Json* tensors = FindMember(value, "tensors");
+  const Json* scalars = FindMember(value, "scalars");
+  if (!tensors || !tensors->is_array() || !scalars || !scalars->is_array()) {
+    return std::nullopt;
   }
-  return "";
+
+  TuningSignature parsed;
+  for (const auto& tensor : *tensors) {
+    const Json* shape = FindMember(tensor, "shape");
+    const Json* dtype = FindMember(tensor, "dtype");
+    if (!shape || !shape->is_array() || !dtype || !IsInteger(*dtype)) {
+      return std::nullopt;
+    }
+
+    TuningSignature::TensorSig tensor_signature;
+    for (const auto& dimension : *shape) {
+      if (!IsInteger(dimension)) return std::nullopt;
+
+      tensor_signature.shape.push_back(dimension.get<int64_t>());
+    }
+    tensor_signature.dtype = static_cast<DataType>(dtype->get<int64_t>());
+    parsed.tensors.push_back(std::move(tensor_signature));
+  }
+
+  for (const auto& scalar : *scalars) {
+    if (!scalar.is_number()) return std::nullopt;
+
+    parsed.scalars.push_back(scalar.get<double>());
+  }
+
+  return parsed;
 }
 
 }  // namespace
 
-namespace infini::ops {
-
 TuningManager& TuningManager::Instance() {
   static TuningManager instance;
   return instance;
 }
 
+void TuningManager::InitializeFromEnvironment() {
+  warmup_count_ = EnvInt("INFINI_OPS_TUNING_WARMUP", kDefaultWarmupCount);
+  repeat_count_ = EnvInt("INFINI_OPS_TUNING_REPEAT", kDefaultRepeatCount);
+
+  const char* path = std::getenv("INFINI_OPS_TUNING_PATH");
+  LoadTuningCache(path && *path ? path : kDefaultTuningPath);
+}
+
 void TuningManager::LoadTuningCache(const std::string& json_path) {
   std::lock_guard<std::mutex> lock(mutex_);
 
@@ -75,147 +111,51 @@ void TuningManager::LoadTuningCache(const std::string& json_path) {
   enabled_ = true;
 
   std::ifstream file(json_path);
-  if (!file.is_open()) {
+  if (!file.is_open()) return;
+
+  Json root = Json::parse(file, nullptr, false);
+  const Json* version = FindMember(root, "version");
+  const Json* entries = FindMember(root, "entries");
+  if (root.is_discarded() || !version || !IsInteger(*version) || !entries ||
+      !entries->is_array()) {
+    std::cerr << "[TuningManager] Warning: failed to parse " << json_path
+              << ", starting with an empty cache" << std::endl;
+    cache_.clear();
+    return;
+  }
+
+  if (version->get<int>() != kTuningCacheVersion) {
+    std::cerr << "[TuningManager] Warning: tuning.json version "
+              << version->get<int>() << " not supported (expected "
+              << kTuningCacheVersion << ")" << std::endl;
     return;
   }
 
-  try {
-    std::stringstream buffer;
-    buffer << file.rdbuf();
-    std::istringstream in(buffer.str());
-
-    SkipTo(in, '{');
-    std::string key;
-    while ((key = NextKey(in)) != "") {
-      if (key == "version") {
-        int version = static_cast<int>(ParseInteger(in));
-        if (version != 1) {
-          std::cerr << "[TuningManager] Warning: tuning.json version "
-                    << version << " not supported (expected 1)" << std::endl;
-          return;
-        }
-      } else if (key == "entries") {
-        SkipTo(in, '[');
-        SkipWhitespace(in);
-        while (in && in.peek() != ']') {
-          SkipTo(in, '{');
-          std::string op_name;
-          Device::Type device = Device::Type::kCount;
-          TuningSignature sig;
-          std::size_t best_impl = 0;
-
-          while ((key = NextKey(in)) != "") {
-            if (key == "operator") {
-              op_name = ParseString(in);
-            } else if (key == "device") {
-              std::string dev_str = ParseString(in);
-              if (dev_str == "cpu")
-                device = Device::Type::kCpu;
-              else if (dev_str == "nvidia")
-                device = Device::Type::kNvidia;
-              else if (dev_str == "cambricon")
-                device = Device::Type::kCambricon;
-              else if (dev_str == "ascend")
-                device = Device::Type::kAscend;
-              else if (dev_str == "metax")
-                device = Device::Type::kMetax;
-              else if (dev_str == "moore")
-                device = Device::Type::kMoore;
-              else if (dev_str == "iluvatar")
-                device = Device::Type::kIluvatar;
-              else if (dev_str == "hygon")
-                device = Device::Type::kHygon;
-            } else if (key == "signature") {
-              SkipTo(in, '{');
-              while ((key = NextKey(in)) != "") {
-                if (key == "tensors") {
-                  SkipTo(in, '[');
-                  SkipWhitespace(in);
-                  while (in && in.peek() != ']') {
-                    SkipTo(in, '{');
-                    TuningSignature::TensorSig tsig;
-                    while ((key = NextKey(in)) != "") {
-                      if (key == "shape") {
-                        SkipTo(in, '[');
-                        SkipWhitespace(in);
-                        while (in && in.peek() != ']') {
-                          tsig.shape.push_back(ParseInteger(in));
-                          SkipWhitespace(in);
-                          if (in.peek() == ',') in.get();
-                          SkipWhitespace(in);
-                        }
-                        if (in.peek() == ']') in.get();
-                      } else if (key == "dtype") {
-                        tsig.dtype = static_cast<DataType>(ParseInteger(in));
-                      } else {
-                        SkipTo(in, ',');
-                      }
-                    }
-                    if (in.peek() == '}') in.get();
-                    sig.tensors.push_back(tsig);
-                    SkipWhitespace(in);
-                    if (in.peek() == ',') in.get();
-                    SkipWhitespace(in);
-                  }
-                  if (in.peek() == ']') in.get();
-                } else if (key == "scalars") {
-                  SkipTo(in, '[');
-                  SkipWhitespace(in);
-                  while (in && in.peek() != ']') {
-                    sig.scalars.push_back(ParseNumber(in));
-                    SkipWhitespace(in);
-                    if (in.peek() == ',') in.get();
-                    SkipWhitespace(in);
-                  }
-                  if (in.peek() == ']') in.get();
-                } else {
-                  SkipTo(in, ',');
-                }
-              }
-              if (in.peek() == '}') in.get();
-            } else if (key == "best_implementation") {
-              best_impl = static_cast<std::size_t>(ParseInteger(in));
-            } else if (key == "metadata") {
-              int depth = 0;
-              SkipWhitespace(in);
-              char c = in.get();
-              if (c == '{') depth = 1;
-              while (depth > 0 && in) {
-                c = in.get();
-                if (c == '{')
-                  depth++;
-                else if (c == '}')
-                  depth--;
-              }
-            } else {
-              SkipTo(in, ',');
-            }
-          }
-
-          if (in.peek() == '}') in.get();
-
-          if (!op_name.empty() && device != Device::Type::kCount) {
-            CacheKey cache_key{op_name, device, sig};
-            cache_[cache_key] = best_impl;
-          }
-
-          SkipWhitespace(in);
-          if (in.peek() == ',') in.get();
-          SkipWhitespace(in);
-        }
-      } else {
-        SkipTo(in, ',');
-      }
+  for (const auto& entry : *entries) {
+    const Json* operator_name = FindMember(entry, "operator");
+    const Json* device_name = FindMember(entry, "device");
+    const Json* signature_json = FindMember(entry, "signature");
+    const Json* best_implementation = FindMember(entry, "best_implementation");
+    if (!operator_name || !operator_name->is_string() || !device_name ||
+        !device_name->is_string() || !signature_json || !best_implementation ||
+        !IsInteger(*best_implementation)) {
+      continue;
     }
 
-    std::cout << "[TuningManager] Loaded " << cache_.size()
-              << " tuning entries from " << json_path << std::endl;
+    auto device = DeviceTypeFromString(
+        device_name->get_ref<const std::string&>(), AllDeviceTypes{});
+    auto signature = SignatureFromJson(*signature_json);
+    if (!device.has_value() || !signature.has_value()) {
+      continue;
+    }
 
-  } catch (...) {
-    std::cerr << "[TuningManager] Warning: failed to parse " << json_path
-              << ", starting with an empty cache" << std::endl;
-    cache_.clear();
+    CacheKey key{operator_name->get_ref<const std::string&>(), *device,
+                 std::move(*signature)};
+    cache_[std::move(key)] = best_implementation->get<std::size_t>();
   }
+
+  std::cout << "[TuningManager] Loaded " << cache_.size()
+            << " tuning entries from " << json_path << std::endl;
 }
 
 std::optional<std::size_t> TuningManager::Lookup(
@@ -225,11 +165,10 @@ std::optional<std::size_t> TuningManager::Lookup(
 
   std::lock_guard<std::mutex> lock(mutex_);
   CacheKey key{operator_name, device, signature};
-  auto it = cache_.find(key);
-  if (it != cache_.end()) {
-    return it->second;
-  }
-  return std::nullopt;
+  auto iterator = cache_.find(key);
+  if (iterator == cache_.end()) return std::nullopt;
+
+  return iterator->second;
 }
 
 void TuningManager::Record(const std::string& operator_name,
@@ -252,43 +191,17 @@ void TuningManager::FlushToDiskLocked() const {
     return;
   }
 
-  out << "{\n";
-  out << "  \"version\": 1,\n";
-  out << "  \"entries\": [\n";
-
-  std::size_t entry_index = 0;
-  for (const auto& [key, best_impl] : cache_) {
-    out << "    {\n";
-    out << "      \"operator\": \"" << key.operator_name << "\",\n";
-    out << "      \"device\": \"" << Device::StringFromType(key.device)
-        << "\",\n";
-    out << "      \"signature\": {\n";
-
-    out << "        \"tensors\": [";
-    for (std::size_t i = 0; i < key.signature.tensors.size(); ++i) {
-      const auto& t = key.signature.tensors[i];
-      out << (i == 0 ? "\n" : ",\n");
-      out << "          {\"shape\": [";
-      for (std::size_t d = 0; d < t.shape.size(); ++d) {
-        out << (d == 0 ? "" : ", ") << t.shape[d];
-      }
-      out << "], \"dtype\": " << static_cast<int>(t.dtype) << "}";
-    }
-    out << (key.signature.tensors.empty() ? "" : "\n        ") << "],\n";
-
-    out << "        \"scalars\": [";
-    for (std::size_t i = 0; i < key.signature.scalars.size(); ++i) {
-      out << (i == 0 ? "" : ", ") << key.signature.scalars[i];
-    }
-    out << "]\n";
-
-    out << "      },\n";
-    out << "      \"best_implementation\": " << best_impl << "\n";
-    out << "    }" << (++entry_index < cache_.size() ? "," : "") << "\n";
+  Json entries = Json::array();
+  for (const auto& [key, best_implementation] : cache_) {
+    entries.push_back(
+        {{"operator", key.operator_name},
+         {"device", std::string{Device::StringFromType(key.device)}},
+         {"signature", SignatureToJson(key.signature)},
+         {"best_implementation", best_implementation}});
   }
 
-  out << "  ]\n";
-  out << "}\n";
+  Json root{{"version", kTuningCacheVersion}, {"entries", std::move(entries)}};
+  out << root.dump(2) << '\n';
 }
 
 }  // namespace infini::ops
diff --git a/src/tuning.h b/src/tuning.h
index 8c1b8c123..0e12b3e91 100644
--- a/src/tuning.h
+++ b/src/tuning.h
@@ -3,6 +3,7 @@
 
 #include <cstddef>
 #include <cstdint>
+#include <functional>
 #include <mutex>
 #include <optional>
 #include <string>
@@ -16,6 +17,15 @@
 
 namespace infini::ops {
 
+namespace detail {
+
+template <typename T>
+void CombineTuningHash(std::size_t& hash, const T& value) {
+  hash ^= std::hash<T>{}(value) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
+}
+
+}  // namespace detail
+
 struct TuningSignature {
   struct TensorSig {
     std::vector<int64_t> shape;
@@ -41,18 +51,17 @@ struct TuningSignature {
   }
 
   std::size_t Hash() const {
-    std::size_t h = 0;
+    std::size_t hash = 0;
     for (const auto& t : tensors) {
-      for (auto dim : t.shape) {
-        h ^= std::hash<int64_t>{}(dim) + 0x9e3779b9 + (h << 6) + (h >> 2);
+      for (auto dimension : t.shape) {
+        detail::CombineTuningHash(hash, dimension);
       }
-      h ^= std::hash<int>{}(static_cast<int>(t.dtype)) + 0x9e3779b9 + (h << 6) +
-           (h >> 2);
+      detail::CombineTuningHash(hash, static_cast<int>(t.dtype));
     }
     for (auto s : scalars) {
-      h ^= std::hash<double>{}(s) + 0x9e3779b9 + (h << 6) + (h >> 2);
+      detail::CombineTuningHash(hash, s);
     }
-    return h;
+    return hash;
   }
 
  private:
@@ -93,25 +102,12 @@ struct TuningSignature {
   }
 };
 
-}  // namespace infini::ops
-
-namespace std {
-
-template <>
-struct hash<infini::ops::TuningSignature> {
-  std::size_t operator()(const infini::ops::TuningSignature& sig) const {
-    return sig.Hash();
-  }
-};
-
-}  // namespace std
-
-namespace infini::ops {
-
 class TuningManager {
  public:
   static TuningManager& Instance();
 
+  void InitializeFromEnvironment();
+
   void LoadTuningCache(const std::string& json_path);
 
   std::optional<std::size_t> Lookup(const std::string& operator_name,
@@ -123,6 +119,10 @@ class TuningManager {
 
   bool IsEnabled() const { return enabled_; }
 
+  int warmup_count() const { return warmup_count_; }
+
+  int repeat_count() const { return repeat_count_; }
+
  private:
   TuningManager() = default;
 
@@ -130,6 +130,10 @@ class TuningManager {
 
   TuningManager& operator=(const TuningManager&) = delete;
 
+  static constexpr int kDefaultWarmupCount = 1;
+
+  static constexpr int kDefaultRepeatCount = 5;
+
   struct CacheKey {
     std::string operator_name;
     Device::Type device;
@@ -143,11 +147,11 @@ class TuningManager {
 
   struct CacheKeyHash {
     std::size_t operator()(const CacheKey& key) const {
-      std::size_t h = std::hash<std::string>{}(key.operator_name);
-      h ^= std::hash<int>{}(static_cast<int>(key.device)) + 0x9e3779b9 +
-           (h << 6) + (h >> 2);
-      h ^= key.signature.Hash() + 0x9e3779b9 + (h << 6) + (h >> 2);
-      return h;
+      std::size_t hash = 0;
+      detail::CombineTuningHash(hash, key.operator_name);
+      detail::CombineTuningHash(hash, static_cast<int>(key.device));
+      detail::CombineTuningHash(hash, key.signature.Hash());
+      return hash;
     }
   };
 
@@ -157,7 +161,11 @@ class TuningManager {
 
   bool enabled_{false};
 
-  std::string json_path_{"tuning.json"};
+  std::string json_path_;
+
+  int warmup_count_{kDefaultWarmupCount};
+
+  int repeat_count_{kDefaultRepeatCount};
 
   mutable std::mutex mutex_;
 };
diff --git a/src/tuning_utils.h b/src/tuning_utils.h
deleted file mode 100644
index f1d55e522..000000000
--- a/src/tuning_utils.h
+++ /dev/null
@@ -1,91 +0,0 @@
-#ifndef INFINI_OPS_TUNING_UTILS_H_
-#define INFINI_OPS_TUNING_UTILS_H_
-
-#include <cstdlib>
-#include <string>
-#include <string_view>
-#include <type_traits>
-#include <vector>
-
-#include "device.h"
-#include "tensor.h"
-
-namespace infini::ops {
-
-namespace detail {
-
-template <typename Key>
-std::string ExtractOperatorName() {
-#if defined(__GNUC__) || defined(__clang__)
-  std::string_view sig = __PRETTY_FUNCTION__;
-
-  auto key_pos = sig.find("Key = ");
-  if (key_pos == std::string_view::npos) return "UnknownOp";
-
-  key_pos += 6;
-  auto end_pos = sig.find_first_of("]>;", key_pos);
-  std::string full_name(sig.substr(key_pos, end_pos - key_pos));
-
-  auto last_colon = full_name.rfind("::");
-  if (last_colon != std::string::npos) {
-    return full_name.substr(last_colon + 2);
-  }
-  return full_name;
-#elif defined(_MSC_VER)
-  std::string_view sig = __FUNCSIG__;
-  auto key_pos = sig.find("Key=");
-  if (key_pos == std::string_view::npos) return "UnknownOp";
-  key_pos += 4;
-  auto end_pos = sig.find_first_of("]>,", key_pos);
-  std::string full_name(sig.substr(key_pos, end_pos - key_pos));
-  auto last_colon = full_name.rfind("::");
-  if (last_colon != std::string::npos) {
-    return full_name.substr(last_colon + 2);
-  }
-  return full_name;
-#else
-  return "UnknownOp";
-#endif
-}
-
-inline int EnvInt(const char* name, int fallback) {
-  const char* v = std::getenv(name);
-  if (!v || !*v) return fallback;
-  int parsed = std::atoi(v);
-  return parsed > 0 ? parsed : fallback;
-}
-
-inline Device::Type FirstDeviceTypeHelper(bool& found) {
-  found = false;
-  return Device::Type::kCount;
-}
-
-template <typename First, typename... Rest>
-Device::Type FirstDeviceTypeHelper(bool& found, const First& first,
-                                   const Rest&... rest) {
-  if constexpr (std::is_same_v<std::decay_t<First>, Tensor>) {
-    found = true;
-    return first.device().type();
-  } else if constexpr (std::is_same_v<std::decay_t<First>,
-                                      std::vector<Tensor>>) {
-    if (!first.empty()) {
-      found = true;
-      return first.front().device().type();
-    }
-    return FirstDeviceTypeHelper(found, rest...);
-  } else {
-    return FirstDeviceTypeHelper(found, rest...);
-  }
-}
-
-template <typename... Args>
-Device::Type FirstDeviceType(const Args&... args) {
-  bool found = false;
-  return FirstDeviceTypeHelper(found, args...);
-}
-
-}  // namespace detail
-
-}  // namespace infini::ops
-
-#endif  // INFINI_OPS_TUNING_UTILS_H_
diff --git a/tests/test_cpp_api.py b/tests/test_cpp_api.py
index 02933b8bf..9fb7ea86d 100644
--- a/tests/test_cpp_api.py
+++ b/tests/test_cpp_api.py
@@ -1,3 +1,4 @@
+import json
 import os
 import subprocess
 import textwrap
@@ -7,45 +8,55 @@
 
 
 def test_cpp_operator_call_instantiation_smoke(tmp_path):
-    install_prefix = _install_prefix()
-    include_dir = install_prefix / "include"
-    library_dir = _library_dir(install_prefix)
-    source = tmp_path / "add_smoke.cc"
-    binary = tmp_path / "add_smoke"
-    source.write_text(_ADD_SMOKE_SOURCE)
-
-    _run(
-        [
-            _compiler("CXX", "c++"),
-            "-std=c++17",
-            "-Werror",
-            f"-I{include_dir}",
-            str(source),
-            f"-L{library_dir}",
-            "-linfiniops",
-            "-linfinirt",
-            f"-Wl,-rpath,{library_dir}",
-            "-o",
-            str(binary),
-        ]
-    )
+    binary = _compile_cpp(tmp_path, "add_smoke", _ADD_SMOKE_SOURCE)
     _run([str(binary)])
 
 
 def test_cpp_returning_call_smoke(tmp_path):
+    binary = _compile_cpp(tmp_path, "add_return_smoke", _ADD_RETURN_SMOKE_SOURCE)
+    _run([str(binary)])
+
+
+def test_tuning_cache_round_trip(tmp_path):
+    binary = _compile_cpp(tmp_path, "tuning_cache", _TUNING_CACHE_SOURCE)
+    cache_path = tmp_path / "tuning.json"
+    environment = os.environ.copy()
+    environment.update(
+        {
+            "INFINI_OPS_TUNING_PATH": str(cache_path),
+            "INFINI_OPS_TUNING_WARMUP": "2",
+            "INFINI_OPS_TUNING_REPEAT": "3",
+        }
+    )
+
+    _run([str(binary), "initialize", str(cache_path)], env=environment)
+
+    cache = json.loads(cache_path.read_text())
+    assert cache["version"] == 1
+    assert cache["entries"][0]["best_implementation"] == 7
+
+    _run([str(binary), "lookup", str(cache_path)])
+
+    cache_path.write_text("{")
+    _run([str(binary), "miss", str(cache_path)])
+
+
+def _compile_cpp(tmp_path, name, source_text):
     install_prefix = _install_prefix()
-    include_dir = install_prefix / "include"
+    include_dirs = [install_prefix / "include"]
+    if infinirt_root := os.environ.get("INFINI_RT_ROOT"):
+        include_dirs.append(Path(infinirt_root) / "include")
     library_dir = _library_dir(install_prefix)
-    source = tmp_path / "add_return_smoke.cc"
-    binary = tmp_path / "add_return_smoke"
-    source.write_text(_ADD_RETURN_SMOKE_SOURCE)
+    source = tmp_path / f"{name}.cc"
+    binary = tmp_path / name
+    source.write_text(source_text)
 
     _run(
         [
             _compiler("CXX", "c++"),
             "-std=c++17",
             "-Werror",
-            f"-I{include_dir}",
+            *(f"-I{include_dir}" for include_dir in include_dirs),
             str(source),
             f"-L{library_dir}",
             "-linfiniops",
@@ -55,7 +66,8 @@ def test_cpp_returning_call_smoke(tmp_path):
             str(binary),
         ]
     )
-    _run([str(binary)])
+
+    return binary
 
 
 def _install_prefix():
@@ -68,12 +80,14 @@ def _install_prefix():
 
 
 def _library_dir(prefix):
-    for name in ("lib", "lib64"):
-        library_dir = prefix / name
-        if (library_dir / "libinfiniops.so").exists():
+    for library_dir in (prefix, prefix / "lib", prefix / "lib64"):
+        if all(
+            (library_dir / name).exists()
+            for name in ("libinfiniops.so", "libinfinirt.so")
+        ):
             return library_dir
 
-    pytest.skip(f"`libinfiniops.so` was not found under `{prefix}`.")
+    pytest.skip(f"InfiniOps and InfiniRT libraries were not found under `{prefix}`.")
 
 
 def _compiler(env_name, default):
@@ -85,16 +99,59 @@ def _compiler(env_name, default):
     return compiler
 
 
-def _run(command):
+def _run(command, *, env=None):
     try:
-        subprocess.run(command, check=True, text=True, capture_output=True)
+        subprocess.run(command, check=True, text=True, capture_output=True, env=env)
     except FileNotFoundError as error:
         pytest.skip(f"`{command[0]}` is not available: {error}")
     except subprocess.CalledProcessError as error:
-        output = "\n".join((error.stdout, error.stderr)).strip()
+        output = f"{error.stdout}\n{error.stderr}".strip()
         raise AssertionError(output) from error
 
 
+_TUNING_CACHE_SOURCE = textwrap.dedent(
+    r"""
+    #include <tuning.h>
+
+    #include <string>
+
+    int main(int argc, char** argv) {
+      if (argc != 3) {
+        return 2;
+      }
+
+      infini::ops::TuningSignature signature;
+      signature.tensors.push_back(
+          {{2, 3}, infini::ops::DataType::kFloat32});
+      signature.scalars.push_back(1.5);
+
+      auto& manager = infini::ops::TuningManager::Instance();
+      const std::string mode{argv[1]};
+
+      if (mode == "initialize") {
+        manager.InitializeFromEnvironment();
+        if (manager.warmup_count() != 2 || manager.repeat_count() != 3) {
+          return 1;
+        }
+        manager.Record("Add", infini::ops::Device::Type::kCpu, signature, 7);
+        return 0;
+      }
+
+      manager.LoadTuningCache(argv[2]);
+      auto implementation = manager.Lookup(
+          "Add", infini::ops::Device::Type::kCpu, signature);
+      if (mode == "lookup") {
+        return implementation == std::optional<std::size_t>{7} ? 0 : 1;
+      }
+      if (mode == "miss") {
+        return implementation.has_value() ? 1 : 0;
+      }
+      return 2;
+    }
+    """
+).lstrip()
+
+
 _ADD_SMOKE_SOURCE = textwrap.dedent(
     r"""
     #include <infini/ops.h>
diff --git a/tests/test_generate_wrappers.py b/tests/test_generate_wrappers.py
index 9f3385789..56b9833a8 100644
--- a/tests/test_generate_wrappers.py
+++ b/tests/test_generate_wrappers.py
@@ -164,15 +164,16 @@ class Mul {
     text = module._generate_pybind11(operator)
 
     assert "std::size_t DefaultImplementationIndexForMul" in text
-    # Constructor still uses DefaultImplementationIndex directly
     assert (
         "config.set_implementation_index("
         "DefaultImplementationIndexForMul(DeviceFromPybind11Handle(input).type()))"
     ) in text
     assert "std::optional<std::size_t> implementation_index" in text
-    # Free function now uses has_value() to support auto-tuning
-    assert "if (implementation_index.has_value())" in text
-    assert "config.set_implementation_index(*implementation_index)" in text
+    assert (
+        "if (implementation_index.has_value()) {\n"
+        "      config.set_implementation_index(*implementation_index);\n"
+        "    }"
+    ) in text
     assert 'py::arg("implementation_index") = py::none()' in text