diff --git a/llm/IFEval-cpp/common.h b/llm/IFEval-cpp/common.h index d05e858..dd48dce 100644 --- a/llm/IFEval-cpp/common.h +++ b/llm/IFEval-cpp/common.h @@ -58,18 +58,38 @@ inline bool contains_string(const std::string& text, return h.find(n) != std::string::npos; } +// Python's len() counts characters; count UTF-8 lead bytes to match it. +inline size_t utf8_len(const std::string& s) { + size_t n = 0; + for (unsigned char c : s) n += (c & 0xC0) != 0x80; + return n; +} + +// Byte offset where character n starts (s.size() if s is shorter). +inline size_t utf8_offset(const std::string& s, size_t n) { + size_t i = 0; + for (; i < s.size(); ++i) { + if ((static_cast(s[i]) & 0xC0) == 0x80) continue; + if (n == 0) break; + --n; + } + return i; +} + inline bool ends_with(const std::string& s, const std::string& suf, unsigned threshold) { - if (s.size() < suf.size()) return false; - std::string a = tolower(s.substr(s.size() - (suf.size() + threshold))); + size_t ls = utf8_len(s), lsuf = utf8_len(suf), win = lsuf + threshold; + if (ls < lsuf) return false; + std::string a = tolower(s.substr(utf8_offset(s, ls > win ? ls - win : 0))); std::string b = tolower(suf); return threshold == 0 ? a == b : contains_string(a, b); } inline bool starts_with(const std::string& s, const std::string& prf, unsigned threshold) { - if (s.size() < prf.size()) return false; - std::string a = tolower(s.substr(0, prf.size() + threshold)); + size_t lprf = utf8_len(prf); + if (utf8_len(s) < lprf) return false; + std::string a = tolower(s.substr(0, utf8_offset(s, lprf + threshold))); std::string b = tolower(prf); return threshold == 0 ? a == b : contains_string(a, b); } @@ -202,9 +222,10 @@ inline std::array transform_response(const std::string& resp) { std::array out{}; for (int mask = 0; mask < 8; ++mask) { std::string t = resp; - if (mask & 0b001) t = remove_font_modifiers(t); if (mask & 0b010) t = remove_first_line(t); if (mask & 0b100) t = remove_last_line(t); + if (mask & 0b110) t = trim(t); // python strips after removing a line + if (mask & 0b001) t = remove_font_modifiers(t); out[mask] = std::move(t); } return out; diff --git a/llm/IFEval-cpp/types.h b/llm/IFEval-cpp/types.h index 7274455..c525244 100644 --- a/llm/IFEval-cpp/types.h +++ b/llm/IFEval-cpp/types.h @@ -46,12 +46,13 @@ class Instruction { virtual constexpr InstructionGroup Group() = 0; bool IsFollowed(const std::string& resp, bool loose = false) const { + // Blank text never counts as following (matches the python reference) // For strict checks, just verify the response itself - if (!loose) return verify_(resp); + if (!loose) return !trim(resp).empty() && verify_(resp); auto transformations = transform_response(resp); for (std::string transformation : transformations) { - if (verify_(transformation)) return true; + if (!trim(transformation).empty() && verify_(transformation)) return true; } return false; } @@ -229,12 +230,13 @@ class ConstrainedResponse : public Instruction { const std::string aMaybe = "My answer is maybe."; const unsigned sizeThreshold = 3; virtual bool verify_(const std::string& resp) const override { + const size_t n = utf8_len(resp); return (resp.find(aYes) != std::string::npos && - resp.size() <= sizeThreshold + aYes.size()) || + n <= sizeThreshold + aYes.size()) || (resp.find(aNo) != std::string::npos && - resp.size() <= sizeThreshold + aNo.size()) || + n <= sizeThreshold + aNo.size()) || (resp.find(aMaybe) != std::string::npos && - resp.size() <= sizeThreshold + aMaybe.size()); + n <= sizeThreshold + aMaybe.size()); } }; @@ -905,7 +907,8 @@ class Quotation : public Instruction { private: virtual bool verify_(const std::string& resp) const override { - return resp.size() >= 2 && resp.front() == '"' && resp.back() == '"'; + const std::string t = trim(resp); + return t.size() >= 2 && t.front() == '"' && t.back() == '"'; } };