Remove special tokens and count characters in ifeval

llm/IFEval-cpp/common.h

@@ -58,18 +58,36 @@ inline bool contains_string(const std::string& text,
   return h.find(n) != std::string::npos;
 }
 
+inline size_t utf8_len(const std::string& s) {
+  size_t n = 0;
+  for (unsigned char c : s) n += (c & 0xC0) != 0x80;
+  return n;
+}
+
+inline size_t utf8_offset(const std::string& s, size_t n) {
+  size_t i = 0;
+  for (; i < s.size(); ++i) {
+    if ((static_cast<unsigned char>(s[i]) & 0xC0) == 0x80) continue;
+    if (n == 0) break;
+    --n;
+  }
+  return i;
+}
+
 inline bool ends_with(const std::string& s, const std::string& suf,
                       unsigned threshold) {
-  if (s.size() < suf.size()) return false;
-  std::string a = tolower(s.substr(s.size() - (suf.size() + threshold)));
+  size_t ls = utf8_len(s), lsuf = utf8_len(suf), win = lsuf + threshold;
+  if (ls < lsuf) return false;
+  std::string a = tolower(s.substr(utf8_offset(s, ls > win ? ls - win : 0)));
   std::string b = tolower(suf);
   return threshold == 0 ? a == b : contains_string(a, b);
 }
 
 inline bool starts_with(const std::string& s, const std::string& prf,
                         unsigned threshold) {
-  if (s.size() < prf.size()) return false;
-  std::string a = tolower(s.substr(0, prf.size() + threshold));
+  size_t lprf = utf8_len(prf);
+  if (utf8_len(s) < lprf) return false;
+  std::string a = tolower(s.substr(0, utf8_offset(s, lprf + threshold)));
   std::string b = tolower(prf);
   return threshold == 0 ? a == b : contains_string(a, b);
 }
@@ -162,7 +180,8 @@ inline std::string remove_font_modifiers(const std::string& s) {
 
     // skip emphasis/strong/strike/escape chars as long as they're not preceeded
     // by an escape character
-    if ((c == '*' || c == '_' || c == '~' || c == '\\') && s[i - 1] != '\\')
+    if ((c == '*' || c == '_' || c == '~' || c == '\\') &&
+        (i == 0 || s[i - 1] != '\\'))
       continue;
 
     // remove heading markers (#) at line starts

llm/IFEval-cpp/ifeval.cc

@@ -6,6 +6,38 @@
 namespace mlperf {
 namespace mobile {
 
+namespace {
+
+std::string remove_special_tokens(const std::string& s) {
+  std::string out;
+  out.reserve(s.size());
+  std::size_t i = 0;
+  while (i < s.size()) {
+    if (s[i] == '<') {
+      std::size_t j = i + 1;
+      while (j < s.size() && s[j] != '<' && s[j] != '>' &&
+             !std::isspace(static_cast<unsigned char>(s[j])))
+        ++j;
+      if (j < s.size() && s[j] == '>') {
+        std::string piece = s.substr(i, j - i + 1);
+        if (piece == "<pad>" || piece == "<eos>" || piece == "<bos>" ||
+            piece == "<mask>" ||
+            (piece.size() > 3 &&
+             (piece.compare(0, 2, "<|") == 0 ||
+              piece.compare(piece.size() - 2, 2, "|>") == 0))) {
+          i = j + 1;
+          continue;
+        }
+      }
+    }
+    out.push_back(s[i]);
+    ++i;
+  }
+  return out;
+}
+
+}  // namespace
+
 IFEval::IFEval(const std::string& input_tfrecord)
    {
 
@@ -21,7 +53,8 @@ IFEval::IFEval(const std::string& input_tfrecord)
     auto record = input[i];
     int key = record["key"].as_i();
     std::string prompt = record["prompt"].as_string();
-    std::string response = record["response"].as_string();
+    std::string response =
+        remove_special_tokens(record["response"].as_string());
     auto instructions = BuildInstructions(record);
 
     //std::string input_formatted = FormatLlamaUserPrompt(prompt);

llm/IFEval-cpp/types.h

@@ -229,12 +229,13 @@ class ConstrainedResponse : public Instruction {
   const std::string aMaybe = "My answer is maybe.";
   const unsigned sizeThreshold = 3;
   virtual bool verify_(const std::string& resp) const override {
+    const size_t n = utf8_len(resp);
     return (resp.find(aYes) != std::string::npos &&
-            resp.size() <= sizeThreshold + aYes.size()) ||
+            n <= sizeThreshold + aYes.size()) ||
            (resp.find(aNo) != std::string::npos &&
-            resp.size() <= sizeThreshold + aNo.size()) ||
+            n <= sizeThreshold + aNo.size()) ||
            (resp.find(aMaybe) != std::string::npos &&
-            resp.size() <= sizeThreshold + aMaybe.size());
+            n <= sizeThreshold + aMaybe.size());
   }
 };
 

llm/instruction_following_eval/evaluation_lib.py

@@ -168,13 +168,35 @@ def test_instruction_following_loose(
   )
 
 
+def remove_special_tokens(s):
+  out = []
+  i = 0
+  while i < len(s):
+    if s[i] == "<":
+      j = i + 1
+      while (j < len(s) and s[j] != "<" and s[j] != ">" and
+             s[j] not in " \t\n\v\f\r"):
+        j += 1
+      if j < len(s) and s[j] == ">":
+        piece = s[i:j + 1]
+        if (piece == "<pad>" or piece == "<eos>" or piece == "<bos>" or
+            piece == "<mask>" or
+            (len(piece) > 3 and
+             (piece.startswith("<|") or piece.endswith("|>")))):
+          i = j + 1
+          continue
+    out.append(s[i])
+    i += 1
+  return "".join(out)
+
+
 def read_prompt_to_response_dict(input_jsonl_filename):
   """Creates dictionary matching prompt and response."""
   return_dict = {}
   with open(input_jsonl_filename, "r") as f:
     for l in f:
       example = json.loads(l)
-      return_dict[example["prompt"]] = example["response"]
+      return_dict[example["prompt"]] = remove_special_tokens(example["response"])
   return return_dict