From 0ae1859ae90fd6a03bee5b7c3f555924bb1f96ff Mon Sep 17 00:00:00 2001 From: Ahmed Youssef Date: Mon, 9 Nov 2026 22:19:24 -0800 Subject: [PATCH 1/2] fix(ifeval): count characters, not bytes, in tolerance check --- llm/IFEval-cpp/common.h | 28 ++++++++++++++++++++++++---- llm/IFEval-cpp/types.h | 7 ++++--- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/llm/IFEval-cpp/common.h b/llm/IFEval-cpp/common.h index d05e858..3d6ad0e 100644 --- a/llm/IFEval-cpp/common.h +++ b/llm/IFEval-cpp/common.h @@ -58,18 +58,38 @@ inline bool contains_string(const std::string& text, return h.find(n) != std::string::npos; } +// Python's len() counts characters; count UTF-8 lead bytes to match it. +inline size_t utf8_len(const std::string& s) { + size_t n = 0; + for (unsigned char c : s) n += (c & 0xC0) != 0x80; + return n; +} + +// Byte offset where character n starts (s.size() if s is shorter). +inline size_t utf8_offset(const std::string& s, size_t n) { + size_t i = 0; + for (; i < s.size(); ++i) { + if ((static_cast(s[i]) & 0xC0) == 0x80) continue; + if (n == 0) break; + --n; + } + return i; +} + inline bool ends_with(const std::string& s, const std::string& suf, unsigned threshold) { - if (s.size() < suf.size()) return false; - std::string a = tolower(s.substr(s.size() - (suf.size() + threshold))); + size_t ls = utf8_len(s), lsuf = utf8_len(suf), win = lsuf + threshold; + if (ls < lsuf) return false; + std::string a = tolower(s.substr(utf8_offset(s, ls > win ? ls - win : 0))); std::string b = tolower(suf); return threshold == 0 ? a == b : contains_string(a, b); } inline bool starts_with(const std::string& s, const std::string& prf, unsigned threshold) { - if (s.size() < prf.size()) return false; - std::string a = tolower(s.substr(0, prf.size() + threshold)); + size_t lprf = utf8_len(prf); + if (utf8_len(s) < lprf) return false; + std::string a = tolower(s.substr(0, utf8_offset(s, lprf + threshold))); std::string b = tolower(prf); return threshold == 0 ? a == b : contains_string(a, b); } diff --git a/llm/IFEval-cpp/types.h b/llm/IFEval-cpp/types.h index 7274455..3a6e801 100644 --- a/llm/IFEval-cpp/types.h +++ b/llm/IFEval-cpp/types.h @@ -229,12 +229,13 @@ class ConstrainedResponse : public Instruction { const std::string aMaybe = "My answer is maybe."; const unsigned sizeThreshold = 3; virtual bool verify_(const std::string& resp) const override { + const size_t n = utf8_len(resp); return (resp.find(aYes) != std::string::npos && - resp.size() <= sizeThreshold + aYes.size()) || + n <= sizeThreshold + aYes.size()) || (resp.find(aNo) != std::string::npos && - resp.size() <= sizeThreshold + aNo.size()) || + n <= sizeThreshold + aNo.size()) || (resp.find(aMaybe) != std::string::npos && - resp.size() <= sizeThreshold + aMaybe.size()); + n <= sizeThreshold + aMaybe.size()); } }; From 55d6129b3424e00918f226b7156b496b75478b25 Mon Sep 17 00:00:00 2001 From: Ahmed Youssef Date: Mon, 9 Nov 2026 22:20:02 -0800 Subject: [PATCH 2/2] fix(ifeval): strip whitespace like the Python reference --- llm/IFEval-cpp/common.h | 3 ++- llm/IFEval-cpp/types.h | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/llm/IFEval-cpp/common.h b/llm/IFEval-cpp/common.h index 3d6ad0e..dd48dce 100644 --- a/llm/IFEval-cpp/common.h +++ b/llm/IFEval-cpp/common.h @@ -222,9 +222,10 @@ inline std::array transform_response(const std::string& resp) { std::array out{}; for (int mask = 0; mask < 8; ++mask) { std::string t = resp; - if (mask & 0b001) t = remove_font_modifiers(t); if (mask & 0b010) t = remove_first_line(t); if (mask & 0b100) t = remove_last_line(t); + if (mask & 0b110) t = trim(t); // python strips after removing a line + if (mask & 0b001) t = remove_font_modifiers(t); out[mask] = std::move(t); } return out; diff --git a/llm/IFEval-cpp/types.h b/llm/IFEval-cpp/types.h index 3a6e801..c525244 100644 --- a/llm/IFEval-cpp/types.h +++ b/llm/IFEval-cpp/types.h @@ -46,12 +46,13 @@ class Instruction { virtual constexpr InstructionGroup Group() = 0; bool IsFollowed(const std::string& resp, bool loose = false) const { + // Blank text never counts as following (matches the python reference) // For strict checks, just verify the response itself - if (!loose) return verify_(resp); + if (!loose) return !trim(resp).empty() && verify_(resp); auto transformations = transform_response(resp); for (std::string transformation : transformations) { - if (verify_(transformation)) return true; + if (!trim(transformation).empty() && verify_(transformation)) return true; } return false; } @@ -906,7 +907,8 @@ class Quotation : public Instruction { private: virtual bool verify_(const std::string& resp) const override { - return resp.size() >= 2 && resp.front() == '"' && resp.back() == '"'; + const std::string t = trim(resp); + return t.size() >= 2 && t.front() == '"' && t.back() == '"'; } };