From 47e53a3b03684ce8068cbcea76796fa40ed831e1 Mon Sep 17 00:00:00 2001 From: Wyatt Childers Date: Wed, 7 Oct 2026 16:26:42 +0000 Subject: [PATCH] Add replay to the edg-docker-test tool [] --- dev_tools/bin/edg-docker-test | 117 ++++++++++++++++++++++++---------- dev_tools/bin/edgy | 36 +++++++++-- 2 files changed, 117 insertions(+), 36 deletions(-) diff --git a/dev_tools/bin/edg-docker-test b/dev_tools/bin/edg-docker-test index 3f88acf400..3d891bd955 100755 --- a/dev_tools/bin/edg-docker-test +++ b/dev_tools/bin/edg-docker-test @@ -17,14 +17,17 @@ import edgdocker import edgtest import edgutil -from datetime import datetime from typing import Any, Dict, List, Optional -from pathlib import Path, PurePosixPath +from pathlib import Path, PurePath, PurePosixPath from edgutil import eprint TOOL_PATH_EDG_EXEC = edgutil.RequiredToolPath('edg-exec') +# Name of the file (within the temporary directory) edgy writes its run +# directory to. +RUN_DIR_FILE_PATH = PurePath('edgy-run-dir.txt') + BUILTIN_POSTURES = [ 'day', 'night', @@ -67,6 +70,13 @@ def parse_args() -> argparse.Namespace: action = 'store_true', help = 'record tests' ) + parser.add_argument( + '--replay', + dest = 'replay', + choices = ['all-failing', 'output-mismatch', 'regress', 'unexpected'], + default = None, + help = 'rerun tests from the last run (see edgy --help)' + ) parser.add_argument( '--discard-test-results', dest = 'discard_test_results', @@ -115,7 +125,8 @@ class TestRunPosture: record_tests: bool, slow_tests: bool, meticulous: bool, test_filters: str, acknowledg_ui_host: Optional[str], - acknowledg_api_host: Optional[str]) -> None: + acknowledg_api_host: Optional[str], + replay_mode: Optional[str] = None) -> None: self.builds = builds self.keep_test_results = keep_test_results self.test_timeout_seconds = test_timeout_seconds @@ -125,6 +136,7 @@ class TestRunPosture: self.test_filters = test_filters self.acknowledg_ui_host = acknowledg_ui_host self.acknowledg_api_host = acknowledg_api_host + self.replay_mode = replay_mode def _configure_day_posture(args: argparse.Namespace) -> TestRunPosture: return TestRunPosture( @@ -256,6 +268,10 @@ def load_posture_file(file_path: Path, acknowledg_api_host = config_file_dict.get( 'acknowledg_api_host', default_posture.acknowledg_api_host + ), + replay_mode = config_file_dict.get( + 'replay_mode', + default_posture.replay_mode ) ) @@ -292,6 +308,8 @@ def _configure_posture(args: argparse.Namespace) -> TestRunPosture: posture.test_timeout_seconds = args.timeout_seconds if args.discard_test_results: posture.keep_test_results = False + if args.replay is not None: + posture.replay_mode = args.replay if args.non_interactive: posture.acknowledg_api_host = None posture.acknowledg_ui_host = None @@ -374,9 +392,6 @@ def update_build(src_dir: Path, posture: TestRunPosture, *, if build_process.returncode != 0: sys.exit(11) -def generate_run_timestamp() -> str: - return datetime.now().strftime(edgutil.RUN_TIMESTAMP_FORMAT) - def _build_edgy_test_config_arg(posture: TestRunPosture) -> str: all_configs = set() @@ -385,12 +400,8 @@ def _build_edgy_test_config_arg(posture: TestRunPosture) -> str: return ','.join(all_configs) -def _build_edgy_args(posture: TestRunPosture, *, - run_timestamp: str, verbose: bool) -> str: - arg_str = ( - f"--run-timestamp={run_timestamp} " - f"--config=\"{_build_edgy_test_config_arg(posture)}\"" - ) +def _build_edgy_args(posture: TestRunPosture, *, verbose: bool) -> str: + arg_str = f"--config=\"{_build_edgy_test_config_arg(posture)}\"" if posture.test_timeout_seconds != 0: arg_str += f" --timeout={posture.test_timeout_seconds}" @@ -400,17 +411,29 @@ def _build_edgy_args(posture: TestRunPosture, *, # This works because each run is in its own container, so we can safely # "claim" /tmp/edgy-run within the container. It will be both unique and # auto-deleted upon the contianer terminating. - arg_str += ' --runs-dir=/tmp/edgy-run' + arg_str += ' --output-directory=/tmp/edgy-run' if posture.slow_tests: arg_str += ' -A' if posture.meticulous: arg_str += ' --meticulous' if verbose: arg_str += ' --verbose' + if posture.replay_mode is not None: + arg_str += f" --replay={posture.replay_mode}" if len(posture.test_filters) != 0: arg_str += f" {posture.test_filters}" return arg_str +def as_docker_tmp_dir(tmp_dir: Path, src_dir: Path, + target_path: PurePath) -> PurePosixPath: + '''Map ``tmp_dir / target_path`` on the host to its absolute path inside + the container, where ``src_dir`` is mounted at /edg/workspace. + ''' + host_path = tmp_dir / target_path + return PurePosixPath( + '/', 'edg', 'workspace', host_path.relative_to(src_dir).as_posix() + ) + def _write_prefixed_env_context(tmp_dir: Path, src_dir: Path, posture: TestRunPosture) -> Path: '''Write prefix_source directives for every build/test config. @@ -426,12 +449,15 @@ def _write_prefixed_env_context(tmp_dir: Path, src_dir: Path, env_file = str(PurePosixPath(build.build_path, 'environment.sh')) # The EDG_BASE environment variable is prefixed through this system # by a file generated by edg-docker-test itself. - generated_env_file = tmp_dir / f"build-{idx}.sh" + generated_env_name = PurePath(f"build-{idx}.sh") + generated_env_file = tmp_dir / generated_env_name with open(generated_env_file, 'w') as gen_file_handle: base_path = PurePosixPath('/', 'edg', 'workspace', build.edg_base) gen_file_handle.write(f"export EDG_BASE=\"{base_path}\"\n") - docker_test_env_file = ( - generated_env_file.relative_to(src_dir).as_posix() + docker_test_env_file = as_docker_tmp_dir( + tmp_dir, + src_dir, + generated_env_name ) for test_config in build.test_configs: @@ -444,18 +470,38 @@ def _write_prefixed_env_context(tmp_dir: Path, src_dir: Path, def _write_loader_invocation(tmp_dir: Path, src_dir: Path, context_file: Path, invocation_name: str, exec_cmd: str, *, dry_run: bool, - verbose: bool) -> str: + verbose: bool, + extra_env: Optional[Dict[str, str]] = None + ) -> str: '''Combine the shared prefix context with an exec directive. - Returns the path to the invocation file relative to ``src_dir`` (as used - inside the container under /edg/workspace). + Any ``extra_env`` variables are written to a generated environment file + that the invocation sources before the exec directive. + + Returns the path to the invocation file as seen inside the container. ''' invocation_file = tmp_dir / invocation_name shutil.copyfile(context_file, invocation_file) with open(invocation_file, 'a') as file_handle: + if extra_env: + extra_env_name = PurePath( + f"extra-{PurePath(invocation_name).stem}-env.sh" + ) + extra_env_file = tmp_dir / extra_env_name + with open(extra_env_file, 'w') as env_file_handle: + for key, value in extra_env.items(): + env_file_handle.write(f"export {key}=\"{value}\"\n") + docker_extra_env_file = as_docker_tmp_dir( + tmp_dir, + src_dir, + extra_env_name + ) + file_handle.write(f"source {docker_extra_env_file}\n") file_handle.write(f"exec {exec_cmd}\n") - docker_invocation_file = invocation_file.relative_to(src_dir).as_posix() + docker_invocation_file = str( + as_docker_tmp_dir(tmp_dir, src_dir, PurePath(invocation_name)) + ) if dry_run or verbose: print(f"Wrote executable directives file ({docker_invocation_file}):") with open(invocation_file) as file_handle: @@ -465,15 +511,13 @@ def _write_loader_invocation(tmp_dir: Path, src_dir: Path, return docker_invocation_file def run_tests(tmp_dir: Path, src_dir: Path, posture: TestRunPosture, *, - context_file: Path, run_timestamp: str, dry_run: bool, + context_file: Path, dry_run: bool, verbose: bool) -> int: print('Running tests...', flush = True) - edgy_args = _build_edgy_args( - posture, - run_timestamp = run_timestamp, - verbose = verbose - ) + edgy_args = _build_edgy_args(posture, verbose = verbose) + # edgy writes the directory it chose (a new run or a replay) to this file. + docker_run_dir_file = as_docker_tmp_dir(tmp_dir, src_dir, RUN_DIR_FILE_PATH) docker_invocation_file = _write_loader_invocation( tmp_dir, src_dir, @@ -481,7 +525,10 @@ def run_tests(tmp_dir: Path, src_dir: Path, posture: TestRunPosture, *, 'edgy.edgivk', f"edgy {edgy_args}", dry_run = dry_run, - verbose = verbose + verbose = verbose, + extra_env = { + 'EDGY_RUN_DIR_FILE': str(docker_run_dir_file) + } ) cmd_args = [ @@ -511,7 +558,7 @@ def _is_posture_acknowledg_enabled(posture: TestRunPosture) -> bool: posture.keep_test_results) def run_acknowledg(tmp_dir: Path, src_dir: Path, posture: TestRunPosture, *, - context_file: Path, run_timestamp: str, dry_run: bool, + context_file: Path, run_dir: str, dry_run: bool, verbose: bool) -> None: print('Opening AcknowlEDG...', flush = True) @@ -520,7 +567,7 @@ def run_acknowledg(tmp_dir: Path, src_dir: Path, posture: TestRunPosture, *, src_dir, context_file, 'edgy-review.edgivk', - f"edgy-review --root='/edg/workspace/tests/runs/{run_timestamp}'", + f"edgy-review --root='{run_dir}'", dry_run = dry_run, verbose = verbose ) @@ -598,13 +645,11 @@ def main() -> None: tmp_dir = Path(tmp_dir_str) context_file = _write_prefixed_env_context(tmp_dir, src_dir, run_posture) - test_run_timestamp = generate_run_timestamp() test_exit_code = run_tests( tmp_dir, src_dir, run_posture, context_file = context_file, - run_timestamp = test_run_timestamp, dry_run = args.dry_run, verbose = args.verbose ) @@ -617,12 +662,20 @@ def main() -> None: (test_exit_code == 0 or test_exit_code == 5) and _is_posture_acknowledg_enabled(run_posture) ): + if args.dry_run: + test_run_dir = '' + elif (tmp_dir / RUN_DIR_FILE_PATH).exists(): + test_run_dir = (tmp_dir / RUN_DIR_FILE_PATH).read_text().strip() + else: + # edgy ran no tests (e.g. nothing to replay), so there's no output. + eprint('edgy did not report a run directory, skipping AcknowlEDG') + sys.exit(0 if test_exit_code == 0 else 20) run_acknowledg( tmp_dir, src_dir, run_posture, context_file = context_file, - run_timestamp = test_run_timestamp, + run_dir = test_run_dir, dry_run = args.dry_run, verbose = args.verbose ) diff --git a/dev_tools/bin/edgy b/dev_tools/bin/edgy index 6058130b7f..b2c429ca09 100755 --- a/dev_tools/bin/edgy +++ b/dev_tools/bin/edgy @@ -902,7 +902,9 @@ def get_latest_replay_number(replay_dir: Path) -> int: def get_run_dir(test_runs_dir: Path, args: argparse.Namespace, test_suites: List[TestSuiteConfig]) -> Path: - if is_replay_run(args): + if args.output_directory is not None: + return args.output_directory + elif is_replay_run(args): # This is guaranteed to exist as the checks on test counts prior to calling # this function ensure that at least one test from the last run is being # replay (so there must be a last run). @@ -933,6 +935,12 @@ def prepare_for_test_run(test_runs_dir: Path, args: argparse.Namespace, run_dir.mkdir(parents = True) + # Let a wrapping tool (e.g. edg-docker-test) learn exactly which directory + # this invocation is using, even for replays, without racing other runs. + run_dir_file = os.environ.get('EDGY_RUN_DIR_FILE') + if run_dir_file: + Path(run_dir_file).write_text(f"{run_dir}\n") + setup_cache = RunSetupCache(run_dir) for test_suite in test_suites: @@ -949,7 +957,12 @@ def prepare_for_test_run(test_runs_dir: Path, args: argparse.Namespace, # Set the test suite run as the current run test_suite.set_current_run(test_suite_run) - except FileExistsError: + except FileExistsError as ex: + if args.output_directory is not None: + # Retrying can't help, as an explicit output directory is never bumped. + edgutil.eprint(f"Path already exists: {ex.filename}") + sys.exit(1) + # This can happen if edgy is rapidly invoked, just wait a bit and retry (to # bump the directory name). time.sleep(.5) @@ -2176,6 +2189,16 @@ def parse_args() -> argparse.Namespace: type = Path, help = 'override the directory where tests runs are stored' ) + parser.add_argument( + '--output-directory', + dest = 'output_directory', + type = Path, + help = ( + 'store the output of this run in exactly this directory, in place of a ' + 'new run or replay directory (replays still read the previous run from ' + 'the runs directory, and no old runs are cleaned up)' + ) + ) parser.add_argument( '--run-timestamp', dest = 'test_run_timestamp', @@ -2262,7 +2285,11 @@ def parse_args() -> argparse.Namespace: '--disp-run-dir', dest = 'disp_run_dir', action = 'store_true', - help = 'print the directory where the run output is being stored' + help = ( + 'print the directory where the run output is being stored (if ' + 'scripting, prefer setting EDGY_RUN_DIR_FILE to have the run directory ' + 'path written there)' + ) ) parser.add_argument( '-W', @@ -2432,7 +2459,8 @@ def main() -> None: generate_logs(args, test_suites, tests, skipped_tests) maybe_dump_performance_info(args, tests, multi_config_run = multi_config_run) - try_cleanup_test_runs(test_runs_dir, args) + if args.output_directory is None: + try_cleanup_test_runs(test_runs_dir, args) sys.exit(exit_code) main()