feat: PE2 + SGR prompt optimization methods

.gitignore

@@ -123,6 +123,7 @@ celerybeat.pid
 
 # Environments
 .env
+.envrc
 .venv
 env/
 venv/
@@ -175,7 +176,7 @@ src/solutions/SPELL/outputs/*
 src/solutions/evo/self_evo/data/*
 src/solutions/SPELL/data/*
 
-# Let's fix some directory for run logs
+.DS_Store
+.worktrees/
 run_logs/*
-
-coolprompt_test/*
+coolprompt_test/*

coolprompt/evaluator/ifeval_checkers.py

@@ -0,0 +1,186 @@
+"""Verifiable checkers for a subset of IFEval instructions.
+
+Each checker returns True iff the model `response` satisfies the
+instruction described by its `kwargs`. Implements a focused
+subset of the IFEval (Zhou et al., 2023) instruction registry;
+the loader filters the dataset to these ids so every evaluated
+prompt is checkable.
+"""
+
+import json
+import re
+
+
+def _word_count(text: str) -> int:
+    return len(re.findall(r"\b\w+\b", text))
+
+
+def _sentence_count(text: str) -> int:
+    parts = re.split(r"[.!?]+", text)
+    return len([p for p in parts if p.strip()])
+
+
+def _check_relation(
+    value: int, threshold: int, relation: str
+) -> bool:
+    if relation == "at least":
+        return value >= threshold
+    if relation == "at most":
+        return value <= threshold
+    if relation == "less than":
+        return value < threshold
+    if relation == "exactly":
+        return value == threshold
+    return False
+
+
+def _number_words(resp, kw):
+    return _check_relation(
+        _word_count(resp),
+        int(kw.get("num_words", 0)),
+        kw.get("relation", "at least"),
+    )
+
+
+def _number_sentences(resp, kw):
+    return _check_relation(
+        _sentence_count(resp),
+        int(kw.get("num_sentences", 0)),
+        kw.get("relation", "at least"),
+    )
+
+
+def _keyword_existence(resp, kw):
+    # Substring (not word-boundary) match is intentional, matching
+    # the IFEval reference impl; differs from _keyword_frequency.
+    low = resp.lower()
+    return all(k.lower() in low for k in kw.get("keywords", []))
+
+
+def _keyword_frequency(resp, kw):
+    word = str(kw.get("keyword", "")).lower()
+    count = len(
+        re.findall(
+            r"\b" + re.escape(word) + r"\b", resp.lower()
+        )
+    )
+    return _check_relation(
+        count,
+        int(kw.get("frequency", 0)),
+        kw.get("relation", "at least"),
+    )
+
+
+def _forbidden_words(resp, kw):
+    low = resp.lower()
+    return all(
+        re.search(
+            r"\b" + re.escape(w.lower()) + r"\b", low
+        ) is None
+        for w in kw.get("forbidden_words", [])
+    )
+
+
+def _all_lowercase(resp, kw):
+    letters = [c for c in resp if c.isalpha()]
+    return bool(letters) and all(c.islower() for c in letters)
+
+
+def _all_uppercase(resp, kw):
+    letters = [c for c in resp if c.isalpha()]
+    return bool(letters) and all(c.isupper() for c in letters)
+
+
+def _number_bullets(resp, kw):
+    bullets = re.findall(
+        r"^\s*[\*\-]\s+", resp, re.MULTILINE
+    )
+    return len(bullets) == int(kw.get("num_bullets", 0))
+
+
+def _json_format(resp, kw):
+    text = resp.strip()
+    text = re.sub(r"^```(json)?|```$", "", text).strip()
+    try:
+        json.loads(text)
+        return True
+    except (ValueError, TypeError):
+        return False
+
+
+def _number_highlighted(resp, kw):
+    highlights = re.findall(
+        r"(?<!\*)\*(?!\*)[^\*\n]+?(?<!\*)\*(?!\*)", resp
+    )
+    return len(highlights) >= int(
+        kw.get("num_highlights", 0)
+    )
+
+
+def _title(resp, kw):
+    return re.search(r"<<[^>\n]+>>", resp) is not None
+
+
+def _end_checker(resp, kw):
+    phrase = str(kw.get("end_phrase", "")).strip().lower()
+    return resp.strip().lower().endswith(phrase)
+
+
+def _quotation(resp, kw):
+    t = resp.strip()
+    return len(t) >= 2 and t.startswith('"') and t.endswith('"')
+
+
+def _postscript(resp, kw):
+    marker = str(
+        kw.get("postscript_marker", "P.S.")
+    ).lower()
+    return marker in resp.lower()
+
+
+def _placeholders(resp, kw):
+    found = re.findall(r"\[[^\]\n]+\](?!\()", resp)
+    return len(found) >= int(
+        kw.get("num_placeholders", 0)
+    )
+
+
+# instruction id -> checker function
+_CHECKERS = {
+    "length_constraints:number_words": _number_words,
+    "length_constraints:number_sentences": _number_sentences,
+    "keywords:existence": _keyword_existence,
+    "keywords:frequency": _keyword_frequency,
+    "keywords:forbidden_words": _forbidden_words,
+    "change_case:english_lowercase": _all_lowercase,
+    "change_case:english_capital": _all_uppercase,
+    "detectable_format:number_bullet_lists": _number_bullets,
+    "detectable_format:json_format": _json_format,
+    "detectable_format:number_highlighted_sections": (
+        _number_highlighted
+    ),
+    "detectable_format:title": _title,
+    "startend:end_checker": _end_checker,
+    "startend:quotation": _quotation,
+    "detectable_content:postscript": _postscript,
+    "detectable_content:number_placeholders": _placeholders,
+}
+
+SUPPORTED_INSTRUCTIONS = frozenset(_CHECKERS.keys())
+
+
+def check_instruction(
+    instruction_id: str, response: str, kwargs: dict
+) -> bool:
+    """Return True iff `response` satisfies the instruction.
+
+    Unknown instruction ids return False (treated as a failed
+    constraint), so unsupported rows must be filtered upstream.
+    """
+    checker = _CHECKERS.get(instruction_id)
+    if checker is None:
+        return False
+    try:
+        return bool(checker(response, kwargs or {}))
+    except (ValueError, TypeError, re.error):
+        return False

coolprompt/evaluator/metrics.py

@@ -1,3 +1,5 @@
+import json
+import re
 from abc import ABC, abstractmethod
 import re
 from typing import Optional, Dict, List, Tuple
@@ -26,6 +28,7 @@
     FLUENCY_TEMPLATE,
     RELEVANCE_TEMPLATE,
 )
+from coolprompt.evaluator.ifeval_checkers import check_instruction
 
 
 class HFEvaluateMetric(ABC):
@@ -570,6 +573,82 @@ def parse_output(self, output: str) -> str:
         return extract_number_from_text(extracted)
 
 
+class IFEvalMetric(GenerationMetric):
+    """IFEval prompt-level strict accuracy.
+
+    `targets` are JSON specs:
+        {"instruction_id_list": [...], "kwargs": [{...}, ...]}
+    A prompt scores 1.0 iff every listed instruction is
+    satisfied by the raw model output; the metric returns the
+    mean over prompts. Unlike other generation metrics, this
+    scores the raw output (no <ans> tag extraction).
+    """
+
+    @staticmethod
+    def _get_name():
+        return "ifeval"
+
+    def __init__(self):
+        super().__init__()
+
+    def compute(self, outputs, targets, dataset=None):
+        return self._compute_raw(
+            list(map(str, outputs)),
+            list(map(str, targets)),
+            dataset,
+        )
+
+    def _compute_raw(self, outputs, targets, dataset):
+        per_prompt = []
+        for output, spec_json in zip(outputs, targets):
+            try:
+                spec = json.loads(spec_json)
+            except (ValueError, TypeError):
+                per_prompt.append(0.0)
+                continue
+            ids = spec.get("instruction_id_list", [])
+            kw_list = spec.get("kwargs", [{}] * len(ids))
+            satisfied = all(
+                check_instruction(iid, output, kw or {})
+                for iid, kw in zip(ids, kw_list)
+            )
+            per_prompt.append(
+                1.0 if satisfied and ids else 0.0
+            )
+        return float(mean(per_prompt)) if per_prompt else 0.0
+
+    def failure_breakdown(self, outputs, targets):
+        """Per-instruction failure summary for the diagnosis.
+
+        Returns a string like
+        'change_case:english_lowercase failed 3/5; ...' listing
+        only the constraints that failed at least once, or None
+        if every constraint passed / targets are unparseable.
+        """
+        fails = {}
+        totals = {}
+        for output, spec_json in zip(
+            map(str, outputs), map(str, targets)
+        ):
+            try:
+                spec = json.loads(spec_json)
+            except (ValueError, TypeError):
+                continue
+            ids = spec.get("instruction_id_list", [])
+            kw_list = spec.get("kwargs", [{}] * len(ids))
+            for iid, kw in zip(ids, kw_list):
+                totals[iid] = totals.get(iid, 0) + 1
+                if not check_instruction(iid, output, kw or {}):
+                    fails[iid] = fails.get(iid, 0) + 1
+        if not fails:
+            return None
+        parts = [
+            f"{iid} failed {fails[iid]}/{totals[iid]}"
+            for iid in sorted(fails, key=lambda k: -fails[k])
+        ]
+        return "; ".join(parts)
+
+
 def define_lang(outputs, targets):
     langs = [detect_language(target) for target in targets]
     return max(set(langs), key=langs.count)

coolprompt/evaluator/portfolio.py

@@ -0,0 +1,18 @@
+"""Best-of selection across optimizer methods (portfolio)."""
+
+
+def select_portfolio(results):
+    """Pick the method whose prompt scored highest on val.
+
+    Args:
+        results: dict mapping method name -> (prompt, val_score).
+            Insertion order breaks ties (first listed wins).
+
+    Returns:
+        (method, prompt, val_score) of the winner.
+    """
+    best = None
+    for method, (prompt, score) in results.items():
+        if best is None or score > best[2]:
+            best = (method, prompt, score)
+    return best

coolprompt/optimizer/ape/__init__.py

@@ -0,0 +1,6 @@
+from coolprompt.optimizer.ape.run import ape_optimizer, APEMethod
+
+__all__ = [
+    'ape_optimizer',
+    'APEMethod',
+]

coolprompt/optimizer/ape/proposer.py

@@ -0,0 +1,66 @@
+"""APE proposer: generates prompt variations via paraphrasing."""
+
+from langchain_core.language_models.base import BaseLanguageModel
+
+from coolprompt.optimizer.pe2.node import Node
+from coolprompt.utils.parsing import (
+    extract_answer,
+    get_model_answer_extracted,
+)
+from coolprompt.utils.prompt_templates.ape_templates import (
+    APE_PARAPHRASE_TEMPLATE,
+)
+
+
+class APEProposer:
+    """Generates prompt variations by paraphrasing.
+
+    Unlike PE2's proposer, APE ignores failure examples
+    and simply paraphrases the current prompt.
+
+    Args:
+        model (BaseLanguageModel): LLM for paraphrasing.
+        prompt_max_tokens (int): Max token budget hint
+            for the new prompt.
+    """
+
+    def __init__(
+        self,
+        model: BaseLanguageModel,
+        prompt_max_tokens: int = 300,
+    ) -> None:
+        self.model = model
+        self.prompt_max_tokens = prompt_max_tokens
+
+    def propose(
+        self,
+        node: Node,
+        examples_str: str,
+        full_template: str,
+        batch_size: int,
+    ) -> tuple[str, str]:
+        """Proposes a paraphrased prompt variation.
+
+        Args:
+            node (Node): Current beam node whose prompt
+                is being varied.
+            examples_str (str): Ignored by APE.
+            full_template (str): Ignored by APE.
+            batch_size (int): Ignored by APE.
+
+        Returns:
+            tuple[str, str]: (new_prompt, "paraphrase").
+        """
+        prompt = APE_PARAPHRASE_TEMPLATE.format(
+            prompt=node.prompt,
+            max_tokens=self.prompt_max_tokens,
+        )
+        answer = get_model_answer_extracted(self.model, prompt)
+
+        new_prompt = extract_answer(
+            answer,
+            ("<prompt>", "</prompt>"),
+            format_mismatch_label=node.prompt,
+        )
+
+        return new_prompt.strip(), "paraphrase"