Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 14 additions & 4 deletions src/repo2rlenv/pipelines/code_instruct.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@
task_fingerprints,
)
from repo2rlenv.pipelines.base import PipelineResult
from repo2rlenv.sources import blob_reference_url
from repo2rlenv.spec.input import GenerationInput, PipelineName
from repo2rlenv.spec.options import CodeInstructOptions

Expand Down Expand Up @@ -492,15 +493,24 @@ def _build_task(self, seed: Seed, parsed: ParsedTask, *, test_filename: str) ->
)
dockerfile = build_code_instruct_dockerfile(image_ref)

reference = blob_reference_url(
self.input.repo.source_kind,
owner,
name,
self.input.repo.ref,
seed.relative_path,
seed.start_line,
seed.end_line,
)

repo2env = {
"pipeline": "code_instruct",
"pipeline_version": "0.6.2",
"repo": f"{owner}/{name}",
"ref": self.input.repo.ref,
"reference": (
f"https://github.com/{owner}/{name}/blob/{self.input.repo.ref}/"
f"{seed.relative_path}#L{seed.start_line}-L{seed.end_line}"
),
# Omitted, not None, for a local checkout: TOML has no null and
# tomli_w rejects it outright.
**({"reference": reference} if reference is not None else {}),
"source_access": self.input.repo.access,
"built_at": datetime.now(UTC).isoformat(),
"synthesis_llm": self.input.llm.qualified_name,
Expand Down
18 changes: 14 additions & 4 deletions src/repo2rlenv/pipelines/equivalence_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
from repo2rlenv.pipelines._function_extractor import FunctionCandidate, walk_repo
from repo2rlenv.pipelines._oss_instruct import check_equivalence_test_strength
from repo2rlenv.pipelines.base import PipelineResult
from repo2rlenv.sources import blob_reference_url
from repo2rlenv.spec.input import GenerationInput, PipelineName
from repo2rlenv.spec.options import EquivalenceTestsOptions

Expand Down Expand Up @@ -657,15 +658,24 @@ def _build_task(

instruction = _build_instruction(cand)

reference = blob_reference_url(
self.input.repo.source_kind,
owner,
name,
self.input.repo.ref,
cand.relative_path,
cand.lineno,
cand.end_lineno,
)

repo2env = {
"pipeline": "equivalence_tests",
"pipeline_version": "0.7.1",
"repo": f"{owner}/{name}",
"ref": self.input.repo.ref,
"reference": (
f"https://github.com/{owner}/{name}/blob/{self.input.repo.ref}/"
f"{cand.relative_path}#L{cand.lineno}-L{cand.end_lineno}"
),
# Omitted, not None, for a local checkout: TOML has no null and
# tomli_w rejects it outright.
**({"reference": reference} if reference is not None else {}),
"source_access": self.input.repo.access,
"built_at": datetime.now(UTC).isoformat(),
"synthesis_llm": self.input.llm.qualified_name,
Expand Down
22 changes: 22 additions & 0 deletions src/repo2rlenv/sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,3 +67,25 @@ def detect_source_kind(url: str) -> SourceKind:

def capabilities_for(kind: SourceKind) -> frozenset[Capability]:
return _CAPABILITIES[kind]


def blob_reference_url(
kind: SourceKind, owner: str, name: str, ref: str, path: str, start_line: int, end_line: int
) -> str | None:
"""Browsable link to a line range in ``path`` at ``ref``, or ``None`` for
a local checkout (no host to link to, so callers must omit rather than
null the field, since TOML has no null).

GitHub and GitLab differ in both the path segment and the line-range
anchor: GitHub is ``/blob/`` with ``#L{start}-L{end}``; GitLab's
canonical form is ``/-/blob/`` with ``#L{start}-{end}`` (no second
``L``).
"""
if kind is SourceKind.LOCAL:
return None
host = "gitlab.com" if kind is SourceKind.GITLAB else "github.com"
blob_path = "-/blob" if kind is SourceKind.GITLAB else "blob"
anchor = (
f"L{start_line}-{end_line}" if kind is SourceKind.GITLAB else f"L{start_line}-L{end_line}"
)
return f"https://{host}/{owner}/{name}/{blob_path}/{ref}/{path}#{anchor}"
85 changes: 85 additions & 0 deletions tests/test_pipeline_code_instruct.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@
import pytest

from repo2rlenv.pipelines._eval_script import all_tests_passed as _all_tests_passed
from repo2rlenv.pipelines._oss_instruct import ParsedTask, Seed
from repo2rlenv.pipelines.code_instruct import (
CodeInstructPipeline,
build_code_instruct_dockerfile,
make_solution_diff,
)
from repo2rlenv.sources import SourceKind
from repo2rlenv.spec.options import CodeInstructOptions

# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -123,3 +125,86 @@ def test_code_instruct_options_defaults():
assert opts.seed_max_loc == 200
assert opts.require_test_fails_without_oracle is True
assert opts.require_test_passes_with_oracle is True


# ---------------------------------------------------------------------------
# _build_task: reference URL must follow source_kind, not always github.com
# ---------------------------------------------------------------------------


def _stub_pipeline_for_build_task(source_kind=SourceKind.GITHUB):
"""A pipeline instance with just enough scaffolding to call `_build_task`."""
from types import SimpleNamespace
from unittest.mock import MagicMock

pipe = CodeInstructPipeline.__new__(CodeInstructPipeline)
pipe._llm_cost_usd = 0.0
pipe.bootstrap = SimpleNamespace(
image_tag="local/r2e-bootstrap/o__r:abc",
image_digest="local/r2e-bootstrap/o__r:abc",
pushed_to_registry=False,
language=SimpleNamespace(value="python"),
)
pipe.input = MagicMock()
pipe.input.repo.owner_name = ("o", "r")
pipe.input.repo.ref = "main"
pipe.input.repo.access = "auto"
pipe.input.repo.source_kind = source_kind
pipe.input.output.org = "default"
pipe.input.llm.qualified_name = "test-provider/test-model"
pipe._progress_cb = None
return pipe


def _seed_and_parsed():
seed = Seed(
relative_path="src/calc.py",
start_line=10,
end_line=20,
text="def add(x, y):\n return x + y\n",
)
parsed = ParsedTask(
problem="Implement add(x, y).",
test_code="def test_add():\n assert add(1, 2) == 3\n",
solution_code="def add(x, y):\n return x + y\n",
)
return seed, parsed


@pytest.mark.parametrize(
("source_kind", "expected_reference"),
[
(SourceKind.GITHUB, "https://github.com/o/r/blob/main/src/calc.py#L10-L20"),
(SourceKind.GITLAB, "https://gitlab.com/o/r/-/blob/main/src/calc.py#L10-20"),
(SourceKind.LOCAL, None),
],
)
def test_build_task_reference_host_matches_source(source_kind, expected_reference):
"""The 'reference' provenance URL must point at the seed's actual host,
not always github.com: code_instruct sets no required_capabilities and
runs on any source (sources.py), and a GitLab- or local-sourced task
previously got a github.com link that does not resolve."""
pipe = _stub_pipeline_for_build_task(source_kind=source_kind)
seed, parsed = _seed_and_parsed()
task = pipe._build_task(seed, parsed, test_filename="test_r2e_deadbeef.py")
if expected_reference is None:
# TOML has no null, so a local checkout must drop the key, not null it.
assert "reference" not in task.repo2env
else:
assert task.repo2env["reference"] == expected_reference


def test_build_task_local_source_writes_a_loadable_task_toml(tmp_path):
"""A None reference would crash tomli_w.dumps (TOML has no null) inside
write_harbor_task. Drive the real emitter, not just the dict, so a
regression here fails loudly instead of only at push time."""
import tomllib

from repo2rlenv.emitter.harbor import write_harbor_task

pipe = _stub_pipeline_for_build_task(source_kind=SourceKind.LOCAL)
seed, parsed = _seed_and_parsed()
task = pipe._build_task(seed, parsed, test_filename="test_r2e_deadbeef.py")
task_path = write_harbor_task(task, tmp_path)
loaded = tomllib.loads((task_path / "task.toml").read_text())
assert "reference" not in loaded["metadata"]["repo2env"]
74 changes: 74 additions & 0 deletions tests/test_pipeline_equivalence_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
build_equivalence_dockerfile,
uses_both_names,
)
from repo2rlenv.sources import SourceKind
from repo2rlenv.spec.options import EquivalenceTestsOptions


Expand Down Expand Up @@ -285,3 +286,76 @@ def test_equivalence_tests_options_defaults():
assert opts.max_loc == 60
assert opts.require_test_fails_with_stub is True
assert opts.require_test_passes_with_oracle is True


# ---------------------------------------------------------------------------
# _build_task: reference URL must follow source_kind, not always github.com
# ---------------------------------------------------------------------------


def _stub_pipeline_for_build_task(source_kind=SourceKind.GITHUB):
"""A pipeline instance with just enough scaffolding to call `_build_task`."""
from types import SimpleNamespace
from unittest.mock import MagicMock

pipe = EquivalenceTestsPipeline.__new__(EquivalenceTestsPipeline)
pipe._llm_cost_usd = 0.0
pipe.bootstrap = SimpleNamespace(
image_tag="local/r2e-bootstrap/o__r:abc",
image_digest="local/r2e-bootstrap/o__r:abc",
pushed_to_registry=False,
language=SimpleNamespace(value="python"),
)
pipe.input = MagicMock()
pipe.input.repo.owner_name = ("o", "r")
pipe.input.repo.ref = "main"
pipe.input.repo.access = "auto"
pipe.input.repo.source_kind = source_kind
pipe.input.output.org = "default"
pipe.input.llm.qualified_name = "test-provider/test-model"
pipe._progress_cb = None
return pipe


@pytest.mark.parametrize(
("source_kind", "expected_reference"),
[
(SourceKind.GITHUB, "https://github.com/o/r/blob/main/src/calc.py#L1-L2"),
(SourceKind.GITLAB, "https://gitlab.com/o/r/-/blob/main/src/calc.py#L1-2"),
(SourceKind.LOCAL, None),
],
)
def test_build_task_reference_host_matches_source(source_kind, expected_reference):
"""The 'reference' provenance URL must point at the candidate's actual
host, not always github.com: equivalence_tests sets no
required_capabilities and runs on any source (sources.py), and a
GitLab- or local-sourced task previously got a github.com link that
does not resolve."""
pipe = _stub_pipeline_for_build_task(source_kind=source_kind)
cand = _candidate()
task = pipe._build_task(
cand, "def test_add():\n assert add(1, 2) == 3\n", test_filename="test_r2e_deadbeef.py"
)
if expected_reference is None:
# TOML has no null, so a local checkout must drop the key, not null it.
assert "reference" not in task.repo2env
else:
assert task.repo2env["reference"] == expected_reference


def test_build_task_local_source_writes_a_loadable_task_toml(tmp_path):
"""A None reference would crash tomli_w.dumps (TOML has no null) inside
write_harbor_task. Drive the real emitter, not just the dict, so a
regression here fails loudly instead of only at push time."""
import tomllib

from repo2rlenv.emitter.harbor import write_harbor_task

pipe = _stub_pipeline_for_build_task(source_kind=SourceKind.LOCAL)
cand = _candidate()
task = pipe._build_task(
cand, "def test_add():\n assert add(1, 2) == 3\n", test_filename="test_r2e_deadbeef.py"
)
task_path = write_harbor_task(task, tmp_path)
loaded = tomllib.loads((task_path / "task.toml").read_text())
assert "reference" not in loaded["metadata"]["repo2env"]