From 04d900c5357f5bb591358d4ef070c195330dc7ec Mon Sep 17 00:00:00 2001 From: Farook Date: Tue, 10 Nov 2026 03:53:49 +0300 Subject: [PATCH] remove special tokens and count characters in ifeval --- llm/IFEval-cpp/common.h | 29 ++++++++++++--- llm/IFEval-cpp/ifeval.cc | 35 ++++++++++++++++++- llm/IFEval-cpp/types.h | 7 ++-- .../evaluation_lib.py | 24 ++++++++++++- 4 files changed, 85 insertions(+), 10 deletions(-) diff --git a/llm/IFEval-cpp/common.h b/llm/IFEval-cpp/common.h index d05e858..bf4af17 100644 --- a/llm/IFEval-cpp/common.h +++ b/llm/IFEval-cpp/common.h @@ -58,18 +58,36 @@ inline bool contains_string(const std::string& text, return h.find(n) != std::string::npos; } +inline size_t utf8_len(const std::string& s) { + size_t n = 0; + for (unsigned char c : s) n += (c & 0xC0) != 0x80; + return n; +} + +inline size_t utf8_offset(const std::string& s, size_t n) { + size_t i = 0; + for (; i < s.size(); ++i) { + if ((static_cast(s[i]) & 0xC0) == 0x80) continue; + if (n == 0) break; + --n; + } + return i; +} + inline bool ends_with(const std::string& s, const std::string& suf, unsigned threshold) { - if (s.size() < suf.size()) return false; - std::string a = tolower(s.substr(s.size() - (suf.size() + threshold))); + size_t ls = utf8_len(s), lsuf = utf8_len(suf), win = lsuf + threshold; + if (ls < lsuf) return false; + std::string a = tolower(s.substr(utf8_offset(s, ls > win ? ls - win : 0))); std::string b = tolower(suf); return threshold == 0 ? a == b : contains_string(a, b); } inline bool starts_with(const std::string& s, const std::string& prf, unsigned threshold) { - if (s.size() < prf.size()) return false; - std::string a = tolower(s.substr(0, prf.size() + threshold)); + size_t lprf = utf8_len(prf); + if (utf8_len(s) < lprf) return false; + std::string a = tolower(s.substr(0, utf8_offset(s, lprf + threshold))); std::string b = tolower(prf); return threshold == 0 ? a == b : contains_string(a, b); } @@ -162,7 +180,8 @@ inline std::string remove_font_modifiers(const std::string& s) { // skip emphasis/strong/strike/escape chars as long as they're not preceeded // by an escape character - if ((c == '*' || c == '_' || c == '~' || c == '\\') && s[i - 1] != '\\') + if ((c == '*' || c == '_' || c == '~' || c == '\\') && + (i == 0 || s[i - 1] != '\\')) continue; // remove heading markers (#) at line starts diff --git a/llm/IFEval-cpp/ifeval.cc b/llm/IFEval-cpp/ifeval.cc index ad57731..ccb1023 100644 --- a/llm/IFEval-cpp/ifeval.cc +++ b/llm/IFEval-cpp/ifeval.cc @@ -6,6 +6,38 @@ namespace mlperf { namespace mobile { +namespace { + +std::string remove_special_tokens(const std::string& s) { + std::string out; + out.reserve(s.size()); + std::size_t i = 0; + while (i < s.size()) { + if (s[i] == '<') { + std::size_t j = i + 1; + while (j < s.size() && s[j] != '<' && s[j] != '>' && + !std::isspace(static_cast(s[j]))) + ++j; + if (j < s.size() && s[j] == '>') { + std::string piece = s.substr(i, j - i + 1); + if (piece == "" || piece == "" || piece == "" || + piece == "" || + (piece.size() > 3 && + (piece.compare(0, 2, "<|") == 0 || + piece.compare(piece.size() - 2, 2, "|>") == 0))) { + i = j + 1; + continue; + } + } + } + out.push_back(s[i]); + ++i; + } + return out; +} + +} // namespace + IFEval::IFEval(const std::string& input_tfrecord) { @@ -21,7 +53,8 @@ IFEval::IFEval(const std::string& input_tfrecord) auto record = input[i]; int key = record["key"].as_i(); std::string prompt = record["prompt"].as_string(); - std::string response = record["response"].as_string(); + std::string response = + remove_special_tokens(record["response"].as_string()); auto instructions = BuildInstructions(record); //std::string input_formatted = FormatLlamaUserPrompt(prompt); diff --git a/llm/IFEval-cpp/types.h b/llm/IFEval-cpp/types.h index 7274455..3a6e801 100644 --- a/llm/IFEval-cpp/types.h +++ b/llm/IFEval-cpp/types.h @@ -229,12 +229,13 @@ class ConstrainedResponse : public Instruction { const std::string aMaybe = "My answer is maybe."; const unsigned sizeThreshold = 3; virtual bool verify_(const std::string& resp) const override { + const size_t n = utf8_len(resp); return (resp.find(aYes) != std::string::npos && - resp.size() <= sizeThreshold + aYes.size()) || + n <= sizeThreshold + aYes.size()) || (resp.find(aNo) != std::string::npos && - resp.size() <= sizeThreshold + aNo.size()) || + n <= sizeThreshold + aNo.size()) || (resp.find(aMaybe) != std::string::npos && - resp.size() <= sizeThreshold + aMaybe.size()); + n <= sizeThreshold + aMaybe.size()); } }; diff --git a/llm/instruction_following_eval/evaluation_lib.py b/llm/instruction_following_eval/evaluation_lib.py index 333a196..3e8bdab 100644 --- a/llm/instruction_following_eval/evaluation_lib.py +++ b/llm/instruction_following_eval/evaluation_lib.py @@ -168,13 +168,35 @@ def test_instruction_following_loose( ) +def remove_special_tokens(s): + out = [] + i = 0 + while i < len(s): + if s[i] == "<": + j = i + 1 + while (j < len(s) and s[j] != "<" and s[j] != ">" and + s[j] not in " \t\n\v\f\r"): + j += 1 + if j < len(s) and s[j] == ">": + piece = s[i:j + 1] + if (piece == "" or piece == "" or piece == "" or + piece == "" or + (len(piece) > 3 and + (piece.startswith("<|") or piece.endswith("|>")))): + i = j + 1 + continue + out.append(s[i]) + i += 1 + return "".join(out) + + def read_prompt_to_response_dict(input_jsonl_filename): """Creates dictionary matching prompt and response.""" return_dict = {} with open(input_jsonl_filename, "r") as f: for l in f: example = json.loads(l) - return_dict[example["prompt"]] = example["response"] + return_dict[example["prompt"]] = remove_special_tokens(example["response"]) return return_dict