From e6ce453d7c7fd05a7412aa05f364e5f8acc0b057 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Fri, 10 Jul 2026 20:08:06 +0800 Subject: [PATCH 1/2] Add raw Pine Docker backtest harness --- .dockerignore | 23 +++ .github/workflows/ci.yml | 13 ++ .gitmodules | 6 + CONTRIBUTING.md | 9 + README.md | 54 +++++- cpp/providers/README.md | 9 + docker/Dockerfile | 63 +++++++ docker/entrypoint.py | 170 ++++++++++++++++++ pyproject.toml | 1 + src/pineforge_data/__init__.py | 10 ++ src/pineforge_data/cli/backtest.py | 40 ++++- src/pineforge_data/docker_runtime.py | 249 +++++++++++++++++++++++++++ tests/fixtures/sma_cross.pine | 11 ++ tests/test_backtest.py | 25 ++- tests/test_docker_integration.py | 82 +++++++++ tests/test_docker_runtime.py | 40 +++++ vendor/pineforge-codegen-oss | 1 + vendor/pineforge-engine | 1 + 18 files changed, 791 insertions(+), 16 deletions(-) create mode 100644 .dockerignore create mode 100644 .gitmodules create mode 100644 cpp/providers/README.md create mode 100644 docker/Dockerfile create mode 100644 docker/entrypoint.py create mode 100644 src/pineforge_data/docker_runtime.py create mode 100644 tests/fixtures/sma_cross.pine create mode 100644 tests/test_docker_integration.py create mode 100644 tests/test_docker_runtime.py create mode 160000 vendor/pineforge-codegen-oss create mode 160000 vendor/pineforge-engine diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..49d24c8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,23 @@ +.git +.github +.venv +build +dist +**/__pycache__ +**/*.pyc +tests +vendor/pineforge-engine/.git +vendor/pineforge-engine/build +vendor/pineforge-engine/benchmarks +vendor/pineforge-engine/corpus +vendor/pineforge-engine/docs +vendor/pineforge-engine/scripts +vendor/pineforge-engine/tests +vendor/pineforge-engine/tutorial +vendor/pineforge-codegen-oss/.git +vendor/pineforge-codegen-oss/.github +vendor/pineforge-codegen-oss/docs +vendor/pineforge-codegen-oss/gate +vendor/pineforge-codegen-oss/npm +vendor/pineforge-codegen-oss/scripts +vendor/pineforge-codegen-oss/tests diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c9d753..1eb159f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,3 +28,16 @@ jobs: - run: pytest - if: matrix.python-version == '3.13' run: python -m build + + docker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + submodules: true + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + cache: pip + - run: python -m pip install -e '.[dev]' + - run: PINEFORGE_DOCKER_TEST=1 pytest tests/test_docker_integration.py diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..2a67fe7 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "vendor/pineforge-codegen-oss"] + path = vendor/pineforge-codegen-oss + url = https://github.com/pineforge-4pass/pineforge-codegen-oss.git +[submodule "vendor/pineforge-engine"] + path = vendor/pineforge-engine + url = https://github.com/pineforge-4pass/pineforge-engine.git diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index afa323a..82a23cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,6 +3,14 @@ Community data providers are the reason this repository exists. Keep the core contracts small and put vendor behavior behind provider modules. +Docker and initialized codegen/engine submodules are required for raw-Pine +integration work: + +```bash +git submodule update --init +docker version +``` + ## Provider checklist - Implement one or more protocols from `pineforge_data.providers`. @@ -31,4 +39,5 @@ ruff check . mypy src pytest python -m build +PINEFORGE_DOCKER_TEST=1 pytest tests/test_docker_integration.py ``` diff --git a/README.md b/README.md index 81222e7..01d772c 100644 --- a/README.md +++ b/README.md @@ -74,13 +74,36 @@ adapter emits `start_sequence + 1` next. ## Direct backtest harness -`pineforge-backtest` fetches confirmed OHLCV through a data provider, packs the -normalized bars into the PineForge C ABI, and calls a compiled strategy library -directly. It does not create an intermediate CSV. +The public backtest input is raw PineScript v6. `pineforge-backtest` fetches +confirmed OHLCV through a data provider and runs this pinned pipeline: + +```text +raw .pine + provider OHLCV + ↓ read-only mount +Docker: Python codegen → C++ strategy → pineforge-engine → JSON report +``` + +Docker is a prerequisite. A host C++ compiler and a precompiled strategy +library are not required. + +Clone with both pinned runtime dependencies: ```bash -pineforge-backtest \ - --strategy /path/to/strategy.so \ +git clone --recurse-submodules https://github.com/pineforge-4pass/pineforge-data.git +cd pineforge-data +python3 -m venv .venv +.venv/bin/pip install -e '.[ccxt]' +``` + +For an existing checkout: + +```bash +git submodule update --init +``` + +```bash +.venv/bin/pineforge-backtest \ + --pine strategy.pine \ --exchange kraken \ --symbol BTC/USD \ --timeframe 15m \ @@ -90,6 +113,14 @@ pineforge-backtest \ --pretty ``` +The first invocation builds a local image tagged from the container source and +the exact codegen and engine submodule commits. Later invocations reuse it. +Pass `--rebuild-image` to force a rebuild or `--no-image-build` to require a +prebuilt local image. +Compilation and execution run as a non-root user with networking disabled, all +Linux capabilities dropped, a read-only root filesystem, and only a read-only +temporary input mount. + The JSON report contains data provenance, processed-bar counts, every closed trade, all/long/short trade statistics, equity statistics, security-feed diagnostics, optional trace values, and the complete equity curve. Unix @@ -99,11 +130,17 @@ Use `--provider-config config.json` for CCXT constructor options and `--strategy-params inputs.json` for Pine input overrides. The provider config file may contain credentials, so keep it outside version control. +The report records the Pine source hash, generated C++ hash, transpile and +compile timings, and the exact codegen and engine commits. The OSS codegen is +source-available under its own PolyForm Noncommercial license and supplemental +terms; review `vendor/pineforge-codegen-oss/LICENSE` before distribution or +commercial use. The engine remains Apache-2.0. + Provider implementations are organized by their strongest supported runtime. The current Python bucket contains CCXT and the harness; native low-latency -providers will live in the C++ bucket. Both buckets must emit the same -normalized records, but an individual provider does not need implementations -in both languages. +providers live under `cpp/providers`. Both buckets must emit the same normalized +records, but an individual provider does not need implementations in both +languages. ## Development @@ -113,6 +150,7 @@ python3 -m venv .venv .venv/bin/ruff check . .venv/bin/mypy src .venv/bin/pytest +PINEFORGE_DOCKER_TEST=1 .venv/bin/pytest tests/test_docker_integration.py ``` ## Provider boundary diff --git a/cpp/providers/README.md b/cpp/providers/README.md new file mode 100644 index 0000000..8b191a5 --- /dev/null +++ b/cpp/providers/README.md @@ -0,0 +1,9 @@ +# C++ providers + +This directory is the native provider bucket for feeds that have a supported +C++ SDK or a measured latency/throughput requirement. + +A native provider must normalize into the same bar and trade field contract as +the Python package and use shared conformance fixtures. Providers do not need a +duplicate Python implementation, and provider-specific types must not leak into +`pineforge-engine`. diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..719d8d1 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,63 @@ +# syntax=docker/dockerfile:1@sha256:87999aa3d42bdc6bea60565083ee17e86d1f3339802f543c0d03998580f9cb89 + +FROM debian:bookworm-slim@sha256:60eac759739651111db372c07be67863818726f754804b8707c90979bda511df AS engine-builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + cmake \ + g++ \ + libeigen3-dev \ + ninja-build \ + && rm -rf /var/lib/apt/lists/* + +COPY vendor/pineforge-engine /src/pineforge-engine + +RUN cmake -G Ninja -S /src/pineforge-engine -B /tmp/pineforge-build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/pineforge \ + -DPINEFORGE_BUILD_TESTS=OFF \ + -DPINEFORGE_BUILD_TUTORIAL=OFF \ + -DPINEFORGE_BUILD_CORPUS_STRATEGIES=OFF \ + && cmake --build /tmp/pineforge-build --target pineforge -j2 \ + && cmake --install /tmp/pineforge-build + +FROM python:3.13-slim-bookworm@sha256:fcbd8dfc2605ba7c2eca646846c5e892b2931e41f6227985154a596f26ab8ed7 + +ARG ENGINE_SHA=unknown +ARG CODEGEN_SHA=unknown +ARG DATA_SOURCE_DIGEST=unknown + +LABEL org.opencontainers.image.title="pineforge-data-backtest" \ + org.opencontainers.image.description="Raw PineScript and provider OHLCV to PineForge report" \ + org.opencontainers.image.source="https://github.com/pineforge-4pass/pineforge-data" \ + org.opencontainers.image.licenses="Apache-2.0 AND LicenseRef-PolyForm-Noncommercial-1.0.0" \ + io.pineforge.engine.commit="${ENGINE_SHA}" \ + io.pineforge.codegen.commit="${CODEGEN_SHA}" \ + io.pineforge.data.source-digest="${DATA_SOURCE_DIGEST}" + +RUN apt-get update && apt-get install -y --no-install-recommends \ + g++ \ + libeigen3-dev \ + && rm -rf /var/lib/apt/lists/* /usr/share/doc /usr/share/man + +COPY --from=engine-builder /opt/pineforge /opt/pineforge +COPY vendor/pineforge-codegen-oss/pineforge_codegen /opt/pineforge-codegen/pineforge_codegen +COPY vendor/pineforge-codegen-oss/VERSION /opt/pineforge-codegen/VERSION +COPY src/pineforge_data /opt/pineforge-data/pineforge_data +COPY docker/entrypoint.py /opt/pineforge/bin/pineforge-data-backtest +COPY LICENSE /opt/licenses/pineforge-data.LICENSE +COPY vendor/pineforge-engine/LICENSE /opt/licenses/pineforge-engine.LICENSE +COPY vendor/pineforge-codegen-oss/LICENSE /opt/licenses/pineforge-codegen.LICENSE + +RUN printf '%s\n' "${ENGINE_SHA}" > /opt/pineforge/ENGINE_COMMIT \ + && printf '%s\n' "${CODEGEN_SHA}" > /opt/pineforge/CODEGEN_COMMIT \ + && printf '%s\n' "${DATA_SOURCE_DIGEST}" > /opt/pineforge/DATA_SOURCE_DIGEST \ + && chmod +x /opt/pineforge/bin/pineforge-data-backtest \ + && useradd --system --uid 10001 --shell /usr/sbin/nologin pineforge + +ENV PYTHONPATH=/opt/pineforge-data:/opt/pineforge-codegen \ + PINEFORGE_PREFIX=/opt/pineforge \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +USER 10001 +ENTRYPOINT ["/opt/pineforge/bin/pineforge-data-backtest"] diff --git a/docker/entrypoint.py b/docker/entrypoint.py new file mode 100644 index 0000000..5d9b8fd --- /dev/null +++ b/docker/entrypoint.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Container entrypoint: raw Pine + normalized OHLCV -> JSON backtest report.""" + +from __future__ import annotations + +import csv +import hashlib +import json +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import cast + +from pineforge_codegen import transpile +from pineforge_codegen.errors import CompileError + +from pineforge_data import ( + BacktestOptions, + Bar, + Instrument, + MagnifierDistribution, + PineForgeBacktestRunner, +) +from pineforge_data.backtest import JsonValue + +WORK = Path("/work") +PREFIX = Path(os.environ.get("PINEFORGE_PREFIX", "/opt/pineforge")) + + +def _load_request() -> dict[str, object]: + value = json.loads((WORK / "request.json").read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError("request.json must contain an object") + return cast(dict[str, object], value) + + +def _load_bars(path: Path, instrument: Instrument, source: str) -> list[Bar]: + bars: list[Bar] = [] + with path.open(newline="", encoding="utf-8") as handle: + for row in csv.DictReader(handle): + bars.append( + Bar( + instrument, + int(row["timestamp"]), + float(row["open"]), + float(row["high"]), + float(row["low"]), + float(row["close"]), + float(row["volume"]), + source, + ) + ) + return bars + + +def _read_commit(name: str) -> str: + return (PREFIX / name).read_text(encoding="utf-8").strip() + + +def _compile(generated: Path, strategy_library: Path) -> float: + command = [ + "g++", + "-std=c++17", + "-O2", + "-ffp-contract=off", + "-fPIC", + "-shared", + f"-I{PREFIX / 'include'}", + "-I/usr/include/eigen3", + str(generated), + "-Wl,--whole-archive", + str(PREFIX / "lib/libpineforge.a"), + "-Wl,--no-whole-archive", + "-o", + str(strategy_library), + ] + started = time.perf_counter() + completed = subprocess.run(command, text=True, capture_output=True, check=False) + elapsed = time.perf_counter() - started + if completed.returncode != 0: + detail = completed.stderr.strip() or completed.stdout.strip() or "unknown compiler error" + raise RuntimeError(f"strategy compilation failed:\n{detail}") + return elapsed + + +def main() -> int: + pine_path = WORK / "strategy.pine" + ohlcv_path = WORK / "ohlcv.csv" + if not pine_path.is_file() or not ohlcv_path.is_file(): + print( + "error: /work must contain strategy.pine, ohlcv.csv, and request.json", + file=sys.stderr, + ) + return 2 + + try: + request = _load_request() + instrument_data = cast(dict[str, object], request["instrument"]) + options_data = cast(dict[str, object], request["options"]) + instrument = Instrument( + symbol=str(instrument_data["symbol"]), + venue=str(instrument_data.get("venue", "")), + timezone=str(instrument_data.get("timezone", "UTC")), + session=str(instrument_data.get("session", "24x7")), + volume_unit=str(instrument_data.get("volume_unit", "base")), + ) + bars = _load_bars(ohlcv_path, instrument, str(request.get("source", "provider"))) + options = BacktestOptions( + input_timeframe=str(options_data.get("input_timeframe", "")), + script_timeframe=str(options_data.get("script_timeframe", "")), + bar_magnifier=bool(options_data.get("bar_magnifier", False)), + magnifier_samples=int(options_data.get("magnifier_samples", 4)), + magnifier_distribution=MagnifierDistribution( + int(options_data.get("magnifier_distribution", 3)) + ), + trace_enabled=bool(options_data.get("trace_enabled", False)), + chart_timezone=cast(str | None, options_data.get("chart_timezone")), + ) + strategy_params = cast(dict[str, JsonValue], request.get("strategy_params", {})) + pine_source = pine_path.read_text(encoding="utf-8") + + with tempfile.TemporaryDirectory(prefix="pineforge-") as temporary: + temp = Path(temporary) + generated = temp / "strategy.cpp" + strategy_library = temp / "strategy.so" + transpile_started = time.perf_counter() + generated_source = transpile(pine_source, filename=pine_path.name) + transpile_seconds = time.perf_counter() - transpile_started + generated.write_text(generated_source, encoding="utf-8") + compile_seconds = _compile(generated, strategy_library) + report = PineForgeBacktestRunner.load(strategy_library).run( + bars, + instrument=instrument, + options=options, + strategy_params=strategy_params, + ) + + output: dict[str, object] = { + "runtime": { + "data_source_digest": _read_commit("DATA_SOURCE_DIGEST"), + "engine_commit": _read_commit("ENGINE_COMMIT"), + "codegen_commit": _read_commit("CODEGEN_COMMIT"), + "codegen_version": ( + Path("/opt/pineforge-codegen/VERSION").read_text(encoding="utf-8").strip() + ), + }, + "transpile": { + "seconds": round(transpile_seconds, 6), + "pine_sha256": hashlib.sha256(pine_source.encode()).hexdigest(), + "generated_cpp_sha256": hashlib.sha256(generated_source.encode()).hexdigest(), + "generated_cpp_bytes": len(generated_source.encode()), + }, + "compile": {"seconds": round(compile_seconds, 6)}, + "backtest": report.to_dict(), + } + print(json.dumps(output, separators=(",", ":"), allow_nan=False)) + return 0 + except CompileError as exc: + print(f"error: PineScript transpilation failed: {exc}", file=sys.stderr) + return 5 + except Exception as exc: + print(f"error: {exc}", file=sys.stderr) + return 4 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 3859e66..7abea11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ testpaths = ["tests"] pythonpath = ["src"] [tool.ruff] +extend-exclude = ["vendor"] line-length = 100 target-version = "py311" diff --git a/src/pineforge_data/__init__.py b/src/pineforge_data/__init__.py index 563cf8e..91f8b10 100644 --- a/src/pineforge_data/__init__.py +++ b/src/pineforge_data/__init__.py @@ -7,6 +7,12 @@ MagnifierDistribution, PineForgeBacktestRunner, ) +from .docker_runtime import ( + DockerBacktestRuntime, + DockerExecutionError, + DockerPrerequisiteError, + discover_repository_root, +) from .engine import EngineStreamSink, PfBar, PfTradeTick, pack_bars, pack_trade_ticks from .errors import EngineStreamError from .models import Bar, Instrument, MacroObservation, TradeTick @@ -32,6 +38,9 @@ "CcxtDependencyError", "CcxtError", "CcxtProvider", + "DockerBacktestRuntime", + "DockerExecutionError", + "DockerPrerequisiteError", "EngineBacktestError", "EngineStreamError", "EngineStreamSink", @@ -47,6 +56,7 @@ "PineForgeBacktestRunner", "TradeSubscription", "TradeTick", + "discover_repository_root", "pack_bars", "pack_trade_ticks", ] diff --git a/src/pineforge_data/cli/backtest.py b/src/pineforge_data/cli/backtest.py index 318e05c..4233579 100644 --- a/src/pineforge_data/cli/backtest.py +++ b/src/pineforge_data/cli/backtest.py @@ -11,7 +11,8 @@ from pathlib import Path from typing import cast -from ..backtest import BacktestOptions, JsonValue, PineForgeBacktestRunner +from ..backtest import BacktestOptions, JsonValue +from ..docker_runtime import DockerBacktestRuntime, discover_repository_root from ..models import Instrument from ..providers import CcxtProvider from ..requests import BarRequest @@ -64,9 +65,9 @@ def _load_json_object(path: Path | None) -> dict[str, JsonValue]: def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="pineforge-backtest", - description="Feed provider OHLCV directly into a compiled PineForge strategy", + description="Transpile raw PineScript and backtest provider OHLCV in Docker", ) - parser.add_argument("--strategy", type=Path, required=True, help="compiled strategy library") + parser.add_argument("--pine", type=Path, required=True, help="raw PineScript v6 strategy") parser.add_argument("--provider", choices=("ccxt",), default="ccxt") parser.add_argument("--exchange", required=True, help="CCXT exchange id, such as kraken") parser.add_argument("--symbol", required=True, help="CCXT unified symbol, such as BTC/USD") @@ -86,6 +87,14 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--bar-magnifier", action="store_true") parser.add_argument("--magnifier-samples", type=int, default=4) parser.add_argument("--trace", action="store_true") + parser.add_argument("--image", help="Docker image override; default tag follows submodule pins") + parser.add_argument("--repository-root", type=Path, help="pineforge-data checkout path") + parser.add_argument("--rebuild-image", action="store_true") + parser.add_argument( + "--no-image-build", + action="store_true", + help="fail instead of building when the pinned image is absent", + ) parser.add_argument("--output", type=Path, help="report path; defaults to stdout") parser.add_argument("--pretty", action="store_true", help="pretty-print report JSON") return parser @@ -124,13 +133,27 @@ async def run_harness(args: argparse.Namespace) -> dict[str, JsonValue]: trace_enabled=args.trace, chart_timezone=args.timezone, ) - runner = PineForgeBacktestRunner.load(args.strategy) - report = runner.run( + pine_path = args.pine.expanduser().resolve() + if not pine_path.is_file(): + raise FileNotFoundError(f"PineScript file not found: {pine_path}") + repository_root = discover_repository_root(args.repository_root) + runtime = DockerBacktestRuntime( + repository_root, + image=args.image, + rebuild=args.rebuild_image, + build_if_missing=not args.no_image_build, + ) + container = runtime.run( + pine_path.read_text(encoding="utf-8"), bars, instrument=instrument, + source=provider_name, options=options, strategy_params=strategy_params, ) + required_sections = ("runtime", "transpile", "compile", "backtest") + if any(section not in container for section in required_sections): + raise RuntimeError("Docker report is missing a required section") return { "provider": { "name": provider_name, @@ -146,11 +169,14 @@ async def run_harness(args: argparse.Namespace) -> dict[str, JsonValue]: "bars": len(bars), }, "strategy": { - "library": str(args.strategy.expanduser().resolve()), + "pine": str(pine_path), "input_timeframe": engine_timeframe, "script_timeframe": options.script_timeframe, }, - "backtest": report.to_dict(), + "runtime": cast(JsonValue, container["runtime"]), + "transpile": cast(JsonValue, container["transpile"]), + "compile": cast(JsonValue, container["compile"]), + "backtest": cast(JsonValue, container["backtest"]), } diff --git a/src/pineforge_data/docker_runtime.py b/src/pineforge_data/docker_runtime.py new file mode 100644 index 0000000..370e453 --- /dev/null +++ b/src/pineforge_data/docker_runtime.py @@ -0,0 +1,249 @@ +"""Docker boundary for raw PineScript compilation and engine execution.""" + +from __future__ import annotations + +import csv +import hashlib +import json +import os +import shutil +import subprocess +import tempfile +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path + +from .backtest import BacktestOptions, JsonValue +from .models import Bar, Instrument + + +class DockerPrerequisiteError(RuntimeError): + """Docker or an initialized source submodule is unavailable.""" + + +class DockerExecutionError(RuntimeError): + """The isolated transpile, compile, or backtest command failed.""" + + +def discover_repository_root(start: Path | None = None) -> Path: + """Find a checkout containing the Dockerfile and both pinned submodules.""" + + configured = os.environ.get("PINEFORGE_DATA_ROOT") + origin = Path(configured).expanduser() if configured else start + if origin is None: + origin = Path(__file__).resolve() + candidates = [origin, *origin.parents] + for candidate in candidates: + if (candidate / "docker/Dockerfile").is_file() and (candidate / ".gitmodules").is_file(): + return candidate.resolve() + raise DockerPrerequisiteError( + "pineforge-data checkout not found; set PINEFORGE_DATA_ROOT to a cloned repository" + ) + + +def _completed_error(completed: subprocess.CompletedProcess[str]) -> str: + return completed.stderr.strip() or completed.stdout.strip() or "command failed without output" + + +@dataclass(slots=True) +class DockerBacktestRuntime: + """Build and execute the pinned raw-Pine backtest image.""" + + repository_root: Path + image: str | None = None + rebuild: bool = False + build_if_missing: bool = True + + def _submodule_commit(self, relative_path: str) -> str: + path = self.repository_root / relative_path + if not (path / ".git").exists() or not any(path.iterdir()): + raise DockerPrerequisiteError( + f"missing submodule {relative_path}; run `git submodule update --init`" + ) + completed = subprocess.run( + ["git", "-C", str(path), "rev-parse", "HEAD"], + text=True, + capture_output=True, + check=False, + ) + if completed.returncode != 0: + raise DockerPrerequisiteError(_completed_error(completed)) + return completed.stdout.strip() + + def _pins(self) -> tuple[str, str]: + engine = self._submodule_commit("vendor/pineforge-engine") + codegen = self._submodule_commit("vendor/pineforge-codegen-oss") + return engine, codegen + + def _source_digest(self) -> str: + digest = hashlib.sha256() + paths = [ + self.repository_root / "docker/Dockerfile", + self.repository_root / "docker/entrypoint.py", + *sorted((self.repository_root / "src/pineforge_data").rglob("*.py")), + ] + for path in paths: + relative = path.relative_to(self.repository_root).as_posix() + digest.update(relative.encode()) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + def resolved_image(self) -> str: + """Return the explicit image or a tag derived from both submodule pins.""" + + if self.image: + return self.image + engine, codegen = self._pins() + source = self._source_digest() + return f"pineforge-data-backtest:d{source[:12]}-e{engine[:12]}-c{codegen[:12]}" + + def _check_docker(self) -> None: + if shutil.which("docker") is None: + raise DockerPrerequisiteError( + "Docker is required but the `docker` command was not found" + ) + completed = subprocess.run( + ["docker", "info", "--format", "{{.ServerVersion}}"], + text=True, + capture_output=True, + check=False, + ) + if completed.returncode != 0: + raise DockerPrerequisiteError( + f"Docker daemon is unavailable: {_completed_error(completed)}" + ) + + def _image_exists(self, image: str) -> bool: + completed = subprocess.run( + ["docker", "image", "inspect", image], + text=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + return completed.returncode == 0 + + def ensure_image(self) -> str: + """Check Docker and build the pin-addressed image when necessary.""" + + self._check_docker() + engine, codegen = self._pins() + source = self._source_digest() + image = self.resolved_image() + if not self.rebuild and self._image_exists(image): + return image + if not self.build_if_missing: + raise DockerPrerequisiteError(f"Docker image is not available locally: {image}") + command = [ + "docker", + "build", + "--file", + str(self.repository_root / "docker/Dockerfile"), + "--tag", + image, + "--build-arg", + f"ENGINE_SHA={engine}", + "--build-arg", + f"CODEGEN_SHA={codegen}", + "--build-arg", + f"DATA_SOURCE_DIGEST={source}", + str(self.repository_root), + ] + completed = subprocess.run(command, text=True, check=False) + if completed.returncode != 0: + raise DockerExecutionError( + f"Docker image build failed with exit {completed.returncode}" + ) + return image + + def run( + self, + pine_source: str, + bars: Sequence[Bar], + *, + instrument: Instrument, + source: str, + options: BacktestOptions, + strategy_params: Mapping[str, JsonValue] | None = None, + ) -> dict[str, object]: + """Run raw Pine and normalized OHLCV inside the isolated image.""" + + if not pine_source.strip(): + raise ValueError("PineScript source must not be empty") + if not bars: + raise ValueError("bars must not be empty") + image = self.ensure_image() + with tempfile.TemporaryDirectory(prefix="pineforge-data-") as temporary: + workspace = Path(temporary) + (workspace / "strategy.pine").write_text(pine_source, encoding="utf-8") + with (workspace / "ohlcv.csv").open("w", newline="", encoding="utf-8") as handle: + writer = csv.writer(handle) + writer.writerow(("timestamp", "open", "high", "low", "close", "volume")) + for bar in bars: + writer.writerow( + ( + bar.timestamp_ms, + bar.open, + bar.high, + bar.low, + bar.close, + bar.volume, + ) + ) + request = { + "instrument": { + "symbol": instrument.symbol, + "venue": instrument.venue, + "timezone": instrument.timezone, + "session": instrument.session, + "volume_unit": instrument.volume_unit, + }, + "source": source, + "options": { + "input_timeframe": options.input_timeframe, + "script_timeframe": options.script_timeframe, + "bar_magnifier": options.bar_magnifier, + "magnifier_samples": options.magnifier_samples, + "magnifier_distribution": int(options.magnifier_distribution), + "trace_enabled": options.trace_enabled, + "chart_timezone": options.chart_timezone, + }, + "strategy_params": dict(strategy_params or {}), + } + (workspace / "request.json").write_text( + json.dumps(request, separators=(",", ":"), allow_nan=False), + encoding="utf-8", + ) + completed = subprocess.run( + [ + "docker", + "run", + "--rm", + "--network", + "none", + "--read-only", + "--tmpfs", + "/tmp:rw,exec,nosuid,nodev,size=512m", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--mount", + f"type=bind,src={workspace},dst=/work,readonly", + image, + ], + text=True, + capture_output=True, + check=False, + ) + if completed.returncode != 0: + raise DockerExecutionError(_completed_error(completed)) + try: + payload = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise DockerExecutionError("container returned invalid JSON") from exc + if not isinstance(payload, dict): + raise DockerExecutionError("container report must be a JSON object") + return payload diff --git a/tests/fixtures/sma_cross.pine b/tests/fixtures/sma_cross.pine new file mode 100644 index 0000000..f9dd87a --- /dev/null +++ b/tests/fixtures/sma_cross.pine @@ -0,0 +1,11 @@ +//@version=6 +strategy("Docker SMA cross", initial_capital=10000) + +fast = ta.sma(close, 2) +slow = ta.sma(close, 4) + +if ta.crossover(fast, slow) + strategy.entry("Long", strategy.long) + +if ta.crossunder(fast, slow) + strategy.close("Long") diff --git a/tests/test_backtest.py b/tests/test_backtest.py index 06bbfc0..bb680ff 100644 --- a/tests/test_backtest.py +++ b/tests/test_backtest.py @@ -3,6 +3,7 @@ import ctypes import json from math import nan +from pathlib import Path from typing import cast import pytest @@ -15,7 +16,7 @@ PineForgeBacktestRunner, ) from pineforge_data.backtest import _PfEquityPoint, _PfReport, _PfTrade -from pineforge_data.cli.backtest import ccxt_timeframe_to_pine, parse_timestamp +from pineforge_data.cli.backtest import build_parser, ccxt_timeframe_to_pine, parse_timestamp class FakeFunction: @@ -143,3 +144,25 @@ def test_ccxt_timeframe_conversion(ccxt: str, pine: str) -> None: def test_timestamp_parser_accepts_unix_ms_and_iso_8601() -> None: assert parse_timestamp("1000") == 1_000 assert parse_timestamp("1970-01-01T00:00:01Z") == 1_000 + + +def test_cli_requires_raw_pine_instead_of_shared_library() -> None: + args = build_parser().parse_args( + [ + "--pine", + "strategy.pine", + "--exchange", + "kraken", + "--symbol", + "BTC/USD", + "--timeframe", + "15m", + "--start", + "1000", + "--end", + "2000", + ] + ) + + assert args.pine == Path("strategy.pine") + assert not hasattr(args, "strategy") diff --git a/tests/test_docker_integration.py b/tests/test_docker_integration.py new file mode 100644 index 0000000..8972d99 --- /dev/null +++ b/tests/test_docker_integration.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import os +from pathlib import Path +from typing import cast + +import pytest + +from pineforge_data import ( + BacktestOptions, + Bar, + DockerBacktestRuntime, + Instrument, + discover_repository_root, +) + +pytestmark = pytest.mark.skipif( + os.environ.get("PINEFORGE_DOCKER_TEST") != "1", + reason="set PINEFORGE_DOCKER_TEST=1 to build and exercise the pinned image", +) + + +def test_raw_pine_transpiles_compiles_and_backtests_in_docker() -> None: + root = discover_repository_root(Path(__file__).resolve()) + pine = (root / "tests/fixtures/sma_cross.pine").read_text(encoding="utf-8") + instrument = Instrument("TEST/USD", venue="fixture") + closes = [ + 10, + 11, + 12, + 13, + 12, + 11, + 10, + 9, + 10, + 11, + 12, + 13, + 12, + 11, + 10, + 9, + 10, + 11, + 12, + 13, + 12, + 11, + 10, + 9, + ] + bars = [ + Bar( + instrument, + 1_700_000_000_000 + index * 60_000, + float(close), + float(close + 1), + float(close - 1), + float(close), + 100.0, + "fixture", + ) + for index, close in enumerate(closes) + ] + result = DockerBacktestRuntime(root).run( + pine, + bars, + instrument=instrument, + source="fixture", + options=BacktestOptions(input_timeframe="1", script_timeframe="1"), + ) + + runtime = cast(dict[str, object], result["runtime"]) + backtest = cast(dict[str, object], result["backtest"]) + summary = cast(dict[str, object], backtest["summary"]) + transpile = cast(dict[str, object], result["transpile"]) + assert runtime["engine_commit"] == "9734d48ce32ed61c0a1d0285166276f110e9afaa" + assert runtime["codegen_commit"] == "9aa99e4f0d4734cbfd4a01e4ceb058774580f1ee" + assert len(cast(str, runtime["data_source_digest"])) == 64 + assert summary["input_bars_processed"] == len(bars) + assert int(cast(int, transpile["generated_cpp_bytes"])) > 1_000 diff --git a/tests/test_docker_runtime.py b/tests/test_docker_runtime.py new file mode 100644 index 0000000..a1f20e5 --- /dev/null +++ b/tests/test_docker_runtime.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from pineforge_data import DockerBacktestRuntime, DockerPrerequisiteError + + +def test_image_tag_is_derived_from_both_pins(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + DockerBacktestRuntime, + "_pins", + lambda _self: ("a" * 40, "b" * 40), + ) + monkeypatch.setattr(DockerBacktestRuntime, "_source_digest", lambda _self: "c" * 64) + runtime = DockerBacktestRuntime(Path("/tmp/repository")) + + assert runtime.resolved_image() == ( + f"pineforge-data-backtest:d{'c' * 12}-e{'a' * 12}-c{'b' * 12}" + ) + + +def test_explicit_image_does_not_replace_pin_validation(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + DockerBacktestRuntime, + "_pins", + lambda _self: ("a" * 40, "b" * 40), + ) + monkeypatch.setattr(DockerBacktestRuntime, "_source_digest", lambda _self: "c" * 64) + runtime = DockerBacktestRuntime(Path("/tmp/repository"), image="example/image:test") + + assert runtime.resolved_image() == "example/image:test" + + +def test_missing_submodule_has_actionable_error(tmp_path: Path) -> None: + runtime = DockerBacktestRuntime(tmp_path) + + with pytest.raises(DockerPrerequisiteError, match="git submodule update --init"): + runtime._submodule_commit("vendor/pineforge-engine") diff --git a/vendor/pineforge-codegen-oss b/vendor/pineforge-codegen-oss new file mode 160000 index 0000000..9aa99e4 --- /dev/null +++ b/vendor/pineforge-codegen-oss @@ -0,0 +1 @@ +Subproject commit 9aa99e4f0d4734cbfd4a01e4ceb058774580f1ee diff --git a/vendor/pineforge-engine b/vendor/pineforge-engine new file mode 160000 index 0000000..9734d48 --- /dev/null +++ b/vendor/pineforge-engine @@ -0,0 +1 @@ +Subproject commit 9734d48ce32ed61c0a1d0285166276f110e9afaa From 57a1bf371293b30aa73cb2cf7542b05c77e68e0d Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Fri, 10 Jul 2026 20:10:52 +0800 Subject: [PATCH 2/2] Fix Docker bind mount permissions --- src/pineforge_data/docker_runtime.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/pineforge_data/docker_runtime.py b/src/pineforge_data/docker_runtime.py index 370e453..b90025b 100644 --- a/src/pineforge_data/docker_runtime.py +++ b/src/pineforge_data/docker_runtime.py @@ -212,10 +212,18 @@ def run( }, "strategy_params": dict(strategy_params or {}), } - (workspace / "request.json").write_text( + request_path = workspace / "request.json" + request_path.write_text( json.dumps(request, separators=(",", ":"), allow_nan=False), encoding="utf-8", ) + for input_path in ( + workspace / "strategy.pine", + workspace / "ohlcv.csv", + request_path, + ): + input_path.chmod(0o644) + workspace.chmod(0o755) completed = subprocess.run( [ "docker",