diff --git a/README.md b/README.md index 73f02ddd..926fd094 100644 --- a/README.md +++ b/README.md @@ -720,4 +720,13 @@ php reprint.phar --state-dir=DIR --fs-root=DIR [options] * `flat-docroot` — Reassemble pulled files into a standard WordPress directory layout using symlinks. Useful when the source site has a non-standard layout (e.g. WP Cloud with ABSPATH separate from wp-content). * `apply-runtime` — Generates server configuration files (`runtime.php`, `start.sh` or `nginx.conf`) from preflight data. See [Step 6](#step-6--generate-runtime-configuration). -All commands except `preflight-assert` support `--abort` to abort the current sync and exit. For `files-pull`, this clears sync progress but keeps the remote index and downloaded files — the next run performs a delta sync. For `db-pull` and `db-index`, it clears the output file so the next run starts from scratch. Interrupted commands automatically resume from the last saved cursor. +The resumable commands `pull`, `pull-files`, `pull-db`, `files-pull`, +`files-index`, `db-pull`, `db-index`, and `db-apply` support `--abort`. An +unfinished import belongs to the command that started it; only that exact +command may resume or abort it. If another command is requested, the error +names the owning command and gives its exact resume and abort invocations. +Abort ignores options used only to perform normal work. File aborts retain +downloaded files and `pull/remote-index.jsonl`, so the next file pull computes a +delta. Database download aborts discard their SQL, table-index, domain, buffer, +and statistics files, while `db-apply` aborts retain its SQL/domain inputs and +do not alter the external database target. diff --git a/packages/reprint-importer/src/import.php b/packages/reprint-importer/src/import.php index 176d2675..ecd2589e 100755 --- a/packages/reprint-importer/src/import.php +++ b/packages/reprint-importer/src/import.php @@ -218,6 +218,22 @@ class ImportClient private const SAVE_STATE_EVERY_N_CHUNKS = 50; private const STATE_PATH_ENCODING_PREFIX = "base64:"; private const SQLITE_PREPARED_INSERT_CACHE_MAX = 128; + private const RESUMABLE_COMMAND_SCOPES = [ + "files-index" => ["files-index"], + "files-pull" => ["files-index", "files-pull"], + "db-index" => ["db-index"], + "db-pull" => ["db-index", "db-pull"], + "db-apply" => ["db-apply"], + "pull-files" => ["files-index", "files-pull"], + "pull-db" => ["db-index", "db-pull", "db-apply"], + "pull" => [ + "files-index", + "files-pull", + "db-index", + "db-pull", + "db-apply", + ], + ]; /** * Maximum number of consecutive interrupted responses with no cursor @@ -632,6 +648,10 @@ public function mark_pull_stage_complete(string $stage, string $pipeline = 'pull { $this->get_state()->pull_pipeline->started_by_command = $pipeline; $this->get_state()->pull_pipeline->last_completed_stage = $stage; + if ($this->get_state()->active_resumable_command->command_name !== null) { + $this->get_state()->active_resumable_command->started_by_command = + $pipeline; + } if ($stage_sequence !== []) { $this->get_state()->pull_pipeline->stage_sequence = $stage_sequence; } @@ -643,6 +663,10 @@ public function mark_pull_complete(string $pipeline = 'pull'): void { $this->get_state()->pull_pipeline->started_by_command = $pipeline; $this->get_state()->pull_pipeline->has_completed_once = true; + if ($this->get_state()->active_resumable_command->command_name !== null) { + $this->get_state()->active_resumable_command->started_by_command = + $pipeline; + } $this->get_state()->active_resumable_command->completion_state = 'complete'; $this->save_state(); } @@ -839,7 +863,8 @@ private function emit_skip_progress(string $path): void * * @param array $options Options: * - command: Required. One of the entries in $valid_commands below. - * - abort: Optional. Clear state for the command and exit immediately + * - abort: Optional. Clear state only when this command owns the + * unfinished import lifecycle, then exit immediately. * - verbose: Optional. Enable verbose output * @param ReprintProcessLock|null $process_lock Optional lock already held * for this state directory. @@ -857,18 +882,6 @@ public function run( } $this->verbose_mode = $options["verbose"] ?? false; $this->progress->set_verbose_mode($this->verbose_mode); - $this->follow_symlinks = $options["follow_symlinks"] ?? true; - $this->include_caches = $options["include_caches"] ?? false; - $this->extra_directory = $options["extra_directory"] ?? null; - if (isset($options["fs_root_nonempty_behavior"])) { - $this->fs_root_nonempty_behavior = $options["fs_root_nonempty_behavior"]; - if (!in_array($this->fs_root_nonempty_behavior, ['error', 'preserve-local'])) { - throw new InvalidArgumentException( - "Invalid --on-fs-root-nonempty value: {$this->fs_root_nonempty_behavior}. " . - "Valid values: error, preserve-local", - ); - } - } $command = $options["command"] ?? null; // Map accepted command aliases to the canonical command names. @@ -884,8 +897,6 @@ public function run( } $abort = $options["abort"] ?? false; - $this->pipeline_step = $options["pipeline_step"] ?? null; - $this->pipeline_steps = $options["pipeline_steps"] ?? null; $valid_commands = [ "pull", @@ -930,13 +941,6 @@ public function run( return; } - // High-level pulls persist resume state before they enter the stage - // runner. Reject invalid options first so a typo does not leave behind - // state that looks like an interrupted pull. - if (in_array($command, ["pull", "pull-db"], true)) { - $this->pull->assert_options_valid_before_state_write($command, $options); - } - $this->state = $this->load_state(); if ($command === "pull-metadata") { @@ -944,6 +948,43 @@ public function run( return; } + $unfinished_import_owner = $this->unfinished_import_owner(); + if (isset(self::RESUMABLE_COMMAND_SCOPES[$command])) { + $this->assert_import_lifecycle_admission( + $command, + $abort, + $unfinished_import_owner, + ); + if ($abort) { + $this->handle_abort( + $command, + $unfinished_import_owner, + ); + return; + } + } + + // High-level pulls persist resume state before they enter the stage + // runner. Reject invalid options before that first state write. + if (in_array($command, ["pull", "pull-db"], true)) { + $this->pull->assert_options_valid_before_state_write($command, $options); + } + + $this->follow_symlinks = $options["follow_symlinks"] ?? true; + $this->include_caches = $options["include_caches"] ?? false; + $this->extra_directory = $options["extra_directory"] ?? null; + $this->pipeline_step = $options["pipeline_step"] ?? null; + $this->pipeline_steps = $options["pipeline_steps"] ?? null; + if (isset($options["fs_root_nonempty_behavior"])) { + $this->fs_root_nonempty_behavior = $options["fs_root_nonempty_behavior"]; + if (!in_array($this->fs_root_nonempty_behavior, ['error', 'preserve-local'])) { + throw new InvalidArgumentException( + "Invalid --on-fs-root-nonempty value: {$this->fs_root_nonempty_behavior}. " . + "Valid values: error, preserve-local", + ); + } + } + // Persist follow_symlinks in state so it survives across invocations. // If explicitly set on CLI, store it. Otherwise, restore from persisted state. if (isset($options["follow_symlinks"])) { @@ -1118,20 +1159,25 @@ public function run( // Pull-like commands orchestrate preflight and lower-level stages // internally, so they run before the normal command dispatch. if (in_array($command, ["pull", "pull-files", "pull-db"], true)) { - if ($abort) { - $this->pull->abort($command); - return; - } try { switch ($command) { case "pull": - $this->pull->run($options); + $this->pull->run( + $options, + $unfinished_import_owner === $command, + ); break; case "pull-files": - $this->pull->run_pull_files($options); + $this->pull->run_pull_files( + $options, + $unfinished_import_owner === $command, + ); break; case "pull-db": - $this->pull->run_pull_db($options); + $this->pull->run_pull_db( + $options, + $unfinished_import_owner === $command, + ); break; } } catch (\Exception $e) { @@ -1179,10 +1225,6 @@ public function run( return; } if ($command === "db-apply") { - if ($abort) { - $this->handle_abort($command); - return; - } try { $this->run_db_apply($options); $final_status = $this->get_state()->active_resumable_command->completion_state ?? "complete"; @@ -1207,17 +1249,7 @@ public function run( $this->require_preflight(); if (in_array($command, ["files-pull", "files-index"], true)) { - $this->prepare_files_pull_options($options, $command === "files-pull" && !$abort); - } - - // Handle --abort: clear state for the command and exit immediately. - // To abort a sync, run ` --abort` (clears state), then - // run `` again (starts fresh). - if ($abort) { - // @TODO: Co-locate abort for each command with the run_*() method - // for that command. - $this->handle_abort($command); - return; + $this->prepare_files_pull_options($options, $command === "files-pull"); } // Dispatch to appropriate command handler @@ -1916,138 +1948,462 @@ private static function mask_url_credentials(string $url): string } /** - * Handle --abort for any command: clear relevant state and exit. + * Clear only the state and artifacts owned by one resumable command. * - * Each command has its own set of files and state fields that need clearing. - * After clearing, we save state and return — the caller exits without - * running the actual sync. The user then runs the command again to start fresh. + * Downloaded files, pull/remote-index.jsonl, external databases, and state + * owned by other commands remain in place. */ - private function handle_abort(string $command): void - { - switch ($command) { - case "files-pull": - $this->clear_files_pull_progress(); - break; - - case "files-index": - $this->audit_log( - "RESTART | Clearing files-index state", - true, - ); - $this->get_state()->active_resumable_command->command_name = "files-index"; - $this->get_state()->active_resumable_command->completion_state = null; - $this->get_state()->active_resumable_command->current_stage = null; - $this->get_state()->index = new RemoteFileIndexCursorState(); - if (file_exists($this->next_remote_index_file)) { - @unlink($this->next_remote_index_file); - $this->audit_log("FILE DELETE | {$this->next_remote_index_file}"); - } - $this->save_state(); - break; + private function handle_abort( + string $command, + ?string $unfinished_import_owner + ): void { + $scopes = self::RESUMABLE_COMMAND_SCOPES[$command]; + $active_checkpoint = $this->get_state()->active_resumable_command; + $pipeline_checkpoint = $this->get_state()->pull_pipeline; + $owns_active_checkpoint = + $active_checkpoint->command_name === $command || + $active_checkpoint->started_by_command === $command || + ( + $unfinished_import_owner === $command && + $pipeline_checkpoint->started_by_command === $command + ); - case "db-pull": - $this->audit_log( - "RESTART | Clearing db-pull state", - true, - ); - $this->reset_state(); - $this->save_state(); + // A legacy checkpoint cannot identify a broader starter whose scopes + // overlap this abort. Clear its exact lower-level command first. + $legacy_checkpoint_scopes = + $active_checkpoint->started_by_command === null && + isset(self::RESUMABLE_COMMAND_SCOPES[$active_checkpoint->command_name]) + ? self::RESUMABLE_COMMAND_SCOPES[$active_checkpoint->command_name] + : []; + if ( + !$owns_active_checkpoint && + $active_checkpoint->completion_state === "complete" && + array_intersect($scopes, $legacy_checkpoint_scopes) !== [] + ) { + // phpcs:disable WordPress.Security.EscapeOutput.ExceptionNotEscaped -- This exception is CLI text, not HTML. + throw new RuntimeException( + "Cannot safely abort {$command}: the completed {$active_checkpoint->command_name} checkpoint does not record which command started it. " . + "Run `reprint {$active_checkpoint->command_name} --abort` first, then retry `reprint {$command} --abort`.", + ); + // phpcs:enable WordPress.Security.EscapeOutput.ExceptionNotEscaped + } - if ($this->sql_output_mode === "file") { - $sql_file = $this->state_dir . "/db.sql"; - if (file_exists($sql_file)) { - unlink($sql_file); - $this->audit_log( - "FILE DELETE | {$sql_file} | abort db-pull", - ); - } - } - $tables_file = $this->state_dir . "/db-tables.jsonl"; - if (file_exists($tables_file)) { - unlink($tables_file); - $this->audit_log( - "FILE DELETE | {$tables_file} | abort db-pull", - ); - } - $domains_file = $this->state_dir . "/pull/domains.json"; - if (file_exists($domains_file)) { - unlink($domains_file); - $this->audit_log( - "FILE DELETE | {$domains_file} | abort db-pull", - ); - } - break; + // A different completed checkpoint still validates its owned state + // and artifacts until another command actually starts fresh. + $completed_checkpoint_starter = + $active_checkpoint->started_by_command ?? + $active_checkpoint->command_name; + if ( + !$owns_active_checkpoint && + $active_checkpoint->completion_state === "complete" && + isset(self::RESUMABLE_COMMAND_SCOPES[$completed_checkpoint_starter]) + ) { + $scopes = array_values(array_diff( + $scopes, + self::RESUMABLE_COMMAND_SCOPES[$completed_checkpoint_starter], + )); + } - case "db-index": - $this->audit_log( - "RESTART | Clearing db-index state", - true, - ); - $this->reset_state(); - $this->save_state(); + $this->prepare_owned_state_reset($scopes, "abort {$command}"); + $this->reset_owned_pull_state($scopes); - $tables_file = $this->state_dir . "/db-tables.jsonl"; - if (file_exists($tables_file)) { - unlink($tables_file); - $this->audit_log( - "FILE DELETE | {$tables_file} | abort db-index", - ); - } - break; + if ($owns_active_checkpoint) { + $this->get_state()->active_resumable_command = + new ResumableCommandCheckpointState(); + $this->get_state()->consecutive_interrupted_responses = 0; + } - case "db-apply": - $this->audit_log( - "RESTART | Clearing db-apply state", - true, - ); - $this->reset_state(); - $this->save_state(); - break; + if ($pipeline_checkpoint->started_by_command === $command) { + $has_completed_once = $pipeline_checkpoint->has_completed_once; + $this->get_state()->pull_pipeline = + new PullPipelineCheckpointState(); + $this->get_state()->pull_pipeline->has_completed_once = + $has_completed_once; } - $this->progress->show_lifecycle_line("State cleared for {$command}.\n"); + $this->save_state(); + $this->remove_owned_artifacts($scopes, "abort {$command}"); - $this->output_progress(["status" => "aborted", "message" => "State cleared for {$command}."]); + $message = "State cleared for {$command}."; + if (in_array("files-pull", $scopes, true)) { + $message .= " Downloaded files and pull/remote-index.jsonl were preserved."; + } + $this->audit_log("ABORT {$command} | complete", true); + $this->progress->show_lifecycle_line("{$message}\n"); + if (in_array($command, ["pull", "pull-files", "pull-db"], true)) { + $this->output_progress([ + "type" => "lifecycle", + "event" => "aborted", + "command" => $command, + "message" => $message, + ], true); + } else { + $this->output_progress([ + "status" => "aborted", + "message" => $message, + ], true); + } } /** - * Clear sync progress and transient files while keeping the remote index - * and downloaded files, so the next files-pull computes a delta. + * Durably take ownership of a new high-level pull before its first stage. + * + * @param string[] $stages Ordered pipeline stages. */ - public function clear_files_pull_progress(): void - { + public function prepare_fresh_pull_pipeline( + string $command, + array $stages + ): void { + $scopes = self::RESUMABLE_COMMAND_SCOPES[$command]; + $this->prepare_owned_state_reset( + $scopes, + "start fresh {$command}", + ); + $this->reset_owned_pull_state($scopes); + + $has_completed_once = + $this->get_state()->pull_pipeline->has_completed_once; + $this->get_state()->active_resumable_command = + new ResumableCommandCheckpointState(); + $this->get_state()->consecutive_interrupted_responses = 0; + $this->get_state()->pull_pipeline = + new PullPipelineCheckpointState(); + $this->get_state()->pull_pipeline->started_by_command = $command; + $this->get_state()->pull_pipeline->stage_sequence = $stages; + $this->get_state()->pull_pipeline->has_completed_once = + $has_completed_once; + if (in_array("files-pull", $scopes, true)) { + $this->get_state()->filter = $this->filter; + } + if (in_array("db-pull", $scopes, true)) { + // High-level pulls always download db.sql. MySQL streaming inputs + // from a completed standalone db-pull do not belong to this new + // pipeline. + $this->mysql_host = null; + $this->mysql_port = null; + $this->mysql_user = null; + $this->mysql_database = null; + $this->get_state()->sql_output = $this->sql_output_mode; + } + + $this->save_state(); + $this->remove_owned_artifacts( + $scopes, + "start fresh {$command}", + ); $this->audit_log( - "RESTART | Clearing files-pull progress (keeping remote index and files)", + strtoupper($command) . " | prepared fresh pipeline", true, ); - // Replay the remote index WAL before clearing the cursor which made its records durable. + } + + /** + * Durably reset and claim one direct resumable command before its first + * request. + */ + private function prepare_fresh_resumable_command( + string $command + ): void { + $started_by_command = $this->unfinished_import_owner() ?? $command; + $scopes = self::RESUMABLE_COMMAND_SCOPES[$command]; + $this->prepare_owned_state_reset( + $scopes, + "start fresh {$command}", + ); + $this->reset_owned_pull_state($scopes); + + $this->get_state()->active_resumable_command = + new ResumableCommandCheckpointState(); + $this->get_state()->active_resumable_command->command_name = + $command; + $this->get_state()->active_resumable_command->started_by_command = + $started_by_command; + $this->get_state()->active_resumable_command->completion_state = + "in_progress"; + $this->get_state()->active_resumable_command->current_stage = + "artifact-cleanup"; + $this->get_state()->consecutive_interrupted_responses = 0; + + if ($command === "files-pull") { + $this->get_state()->filter = $this->filter; + } elseif ($command === "db-pull") { + $this->get_state()->sql_output = $this->sql_output_mode; + $this->get_state()->mysql_host = $this->mysql_host; + $this->get_state()->mysql_port = $this->mysql_port; + $this->get_state()->mysql_user = $this->mysql_user; + $this->get_state()->mysql_database = $this->mysql_database; + } + + $this->save_state(); + $this->remove_owned_artifacts( + $scopes, + "start fresh {$command}", + ); + $this->get_state()->active_resumable_command->current_stage = + "fresh-initialization"; + $this->save_state(); + } + + /** + * Repeat artifact cleanup and report whether fresh initialization remains. + */ + private function finish_pending_artifact_cleanup( + string $command + ): bool { + $checkpoint = $this->get_state()->active_resumable_command; + if ($checkpoint->command_name !== $command) { + return false; + } + + if ($checkpoint->current_stage === "artifact-cleanup") { + $this->remove_owned_artifacts( + self::RESUMABLE_COMMAND_SCOPES[$command], + "start fresh {$command}", + ); + $checkpoint->current_stage = "fresh-initialization"; + $this->save_state(); + } + + return $checkpoint->current_stage === "fresh-initialization"; + } + + /** + * Save command-specific fresh state and expose its first work phase. + */ + private function finish_fresh_resumable_command_initialization( + ?string $current_stage + ): void { + $this->get_state()->active_resumable_command->current_stage = + $current_stage; + $this->save_state(); + } + + /** + * Return the user-facing command which owns unfinished import state. + */ + private function unfinished_import_owner(): ?string + { + $pipeline = $this->get_state()->pull_pipeline; + if ( + in_array( + $pipeline->started_by_command, + ["pull", "pull-files", "pull-db"], + true, + ) && + $pipeline->stage_sequence !== [] + ) { + $final_stage = + $pipeline->stage_sequence[ + count($pipeline->stage_sequence) - 1 + ]; + if ($pipeline->last_completed_stage !== $final_stage) { + return $pipeline->started_by_command; + } + } + + $active_checkpoint = + $this->get_state()->active_resumable_command; + if ( + in_array( + $active_checkpoint->completion_state, + ["in_progress", "partial"], + true, + ) && + in_array( + $active_checkpoint->command_name, + [ + "files-pull", + "files-index", + "db-pull", + "db-index", + "db-apply", + ], + true, + ) + ) { + return $active_checkpoint->command_name; + } + + return null; + } + + /** + * Reject a command which does not own the unfinished import lifecycle. + * + * @throws RuntimeException When another command owns unfinished import state. + */ + private function assert_import_lifecycle_admission( + string $command, + bool $abort, + ?string $unfinished_import_owner + ): void { + if ( + $unfinished_import_owner === null || + $unfinished_import_owner === $command + ) { + return; + } + + $action = $abort ? "abort" : "run"; + // phpcs:disable WordPress.Security.EscapeOutput.ExceptionNotEscaped -- This exception is CLI text, not HTML. + throw new RuntimeException( + "Cannot {$action} {$command} while {$unfinished_import_owner} owns unfinished import state. " . + "Run `reprint {$unfinished_import_owner}` to resume it or " . + "`reprint {$unfinished_import_owner} --abort` to discard it.", + ); + // phpcs:enable WordPress.Security.EscapeOutput.ExceptionNotEscaped + } + + /** + * @param string[] $scopes State groups whose persisted paths will be reset. + */ + private function prepare_owned_state_reset( + array $scopes, + string $reason + ): void { + if (!in_array("files-pull", $scopes, true)) { + return; + } + + // Apply remote index changes before clearing the cursor which made the + // WAL records durable. $this->replay_remote_index_wal(); - $this->remove_remote_index_wal(); - $this->reset_state(); - $this->remote_index_wal_handle = null; - $this->remote_index_wal_record_count = 0; - if (file_exists($this->next_remote_index_file)) { - @unlink($this->next_remote_index_file); - $this->audit_log("FILE DELETE | {$this->next_remote_index_file}"); + // Batch request bodies may live outside --state-dir. Remove them + // before clearing the only persisted paths which can find them. + $batch_files = [ + $this->get_state()->fetch->batch_file, + $this->get_state()->fetch_skipped->batch_file, + ]; + foreach (array_unique(array_filter($batch_files, "is_string")) as $batch_file) { + $this->remove_artifact($batch_file, $reason); + } + } + + /** + * @param string[] $scopes State groups to return to their defaults. + */ + private function reset_owned_pull_state(array $scopes): void + { + if (in_array("files-pull", $scopes, true)) { + $this->get_state()->local_followed_symlinks_root_fingerprint = + null; + $this->get_state()->filter = "none"; + $this->get_state()->files_pull_only_fingerprint = null; + $this->get_state()->files_pull_summary = + new FilesPullSummaryState(); + $this->get_state()->diff = new FileDiffProgressState(); + $this->get_state()->fetch = new FetchListProgressState(); + $this->get_state()->fetch_skipped = + new FetchListProgressState(); + $this->get_state()->current_file = null; + $this->get_state()->current_file_bytes = null; + $this->get_state()->pull_pipeline->skipped_pending = false; + $this->files_pulled = 0; + $this->fetch_list_total = null; + $this->fetch_list_done = null; } - if (file_exists($this->fetch_list_file)) { - @unlink($this->fetch_list_file); - $this->audit_log("FILE DELETE | {$this->fetch_list_file}"); + + if (in_array("files-index", $scopes, true)) { + $this->get_state()->index = + new RemoteFileIndexCursorState(); + $this->next_remote_index_entries_counted = 0; } - if (file_exists($this->skipped_fetch_list_file)) { - @unlink($this->skipped_fetch_list_file); - $this->audit_log("FILE DELETE | {$this->skipped_fetch_list_file}"); + + if (in_array("db-pull", $scopes, true)) { + $this->get_state()->sql_bytes = null; + $this->get_state()->sql_statements_counted = 0; + $this->get_state()->sql_output = null; + $this->get_state()->mysql_host = null; + $this->get_state()->mysql_port = null; + $this->get_state()->mysql_user = null; + $this->get_state()->mysql_database = null; } - if (file_exists($this->volatile_files_file)) { - @unlink($this->volatile_files_file); - $this->audit_log("FILE DELETE | {$this->volatile_files_file}"); + + if (in_array("db-index", $scopes, true)) { + $this->get_state()->db_index = + new DatabaseTableIndexState(); } - $this->get_state()->index = new RemoteFileIndexCursorState(); - $this->get_state()->fetch = new FetchListProgressState(); - $this->get_state()->fetch_skipped = new FetchListProgressState(); - $this->save_state(); + if (in_array("db-apply", $scopes, true)) { + $this->get_state()->apply = + new DatabaseApplyCommandState(); + } + } + + /** + * @param string[] $scopes State groups whose transient artifacts can be removed. + */ + private function remove_owned_artifacts( + array $scopes, + string $reason + ): void { + if (in_array("files-pull", $scopes, true)) { + foreach ([ + $this->fetch_list_file, + $this->skipped_fetch_list_file, + $this->volatile_files_file, + $this->remote_index_file . ".new", + $this->remote_index_file . ".swap", + ] as $path) { + $this->remove_artifact($path, $reason); + } + $this->remove_remote_index_wal(); + } + + if (in_array("files-index", $scopes, true)) { + foreach ([ + $this->next_remote_index_file, + $this->next_remote_index_file . ".sorted", + $this->next_remote_index_file . ".keyed", + $this->next_remote_index_file . ".keyed.sorted", + $this->next_remote_index_file . ".merge-sorted", + ] as $path) { + $this->remove_artifact($path, $reason); + } + foreach ( + glob(dirname($this->next_remote_index_file) . "/merge-chunk-*") ?: [] + as $path + ) { + $this->remove_artifact($path, $reason); + } + } + + if (in_array("db-pull", $scopes, true)) { + foreach ([ + $this->state_dir . "/db.sql", + $this->state_dir . "/pull/domains.json", + $this->state_dir . "/pull/sql-buffer", + $this->state_dir . "/pull/sql-stats.json", + ] as $path) { + $this->remove_artifact($path, $reason); + } + } + + if (in_array("db-index", $scopes, true)) { + $this->remove_artifact( + $this->state_dir . "/db-tables.jsonl", + $reason, + ); + } + } + + /** + * Remove one owned transient artifact or report its exact path. + */ + private function remove_artifact(string $path, string $reason): void + { + if (!file_exists($path) && !is_link($path)) { + return; + } + if (!@unlink($path)) { + // phpcs:disable WordPress.Security.EscapeOutput.ExceptionNotEscaped -- This exception is CLI text, not HTML. + throw new RuntimeException( + "Failed to remove {$path} while attempting to {$reason}.", + ); + // phpcs:enable WordPress.Security.EscapeOutput.ExceptionNotEscaped + } + $this->audit_log("FILE DELETE | {$path} | {$reason}"); } /** @@ -2653,6 +3009,8 @@ private function finalize_tuned_request( */ public function run_files_pull(): void { + $fresh_initialization_pending = + $this->finish_pending_artifact_cleanup("files-pull"); $state_command = $this->get_state()->active_resumable_command->command_name ?? null; // A full `pull` leaves active_resumable_command on its last stage @@ -2668,6 +3026,8 @@ public function run_files_pull(): void $this->get_state()->pull_pipeline->skipped_pending ) { $this->get_state()->active_resumable_command->command_name = "files-pull"; + $this->get_state()->active_resumable_command->started_by_command = + "files-pull"; $this->get_state()->active_resumable_command->completion_state = "complete"; $this->get_state()->active_resumable_command->current_stage = null; $this->save_state(); @@ -2681,11 +3041,16 @@ public function run_files_pull(): void $has_progress = $state_command === "files-pull" && $current_status !== null && - $current_status !== "complete"; + $current_status !== "complete" && + !$fresh_initialization_pending; - $this->replay_remote_index_wal(); - $this->assert_files_pull_only_unchanged_while_resuming($has_progress); - $this->assert_local_followed_symlinks_root_unchanged(); + if ($has_progress || $current_status === "complete") { + $this->replay_remote_index_wal(); + $this->assert_files_pull_only_unchanged_while_resuming( + $has_progress, + ); + $this->assert_local_followed_symlinks_root_unchanged(); + } // Already completed. if ($current_status === "complete") { @@ -2835,16 +3200,12 @@ public function run_files_pull(): void ); } - $this->get_state()->active_resumable_command->command_name = "files-pull"; - $this->get_state()->active_resumable_command->completion_state = "in_progress"; - $this->get_state()->active_resumable_command->current_stage = "index"; + if (!$fresh_initialization_pending) { + $this->prepare_fresh_resumable_command("files-pull"); + } + $this->assert_local_followed_symlinks_root_unchanged(); $this->get_state()->files_pull_only_fingerprint = $this->files_pull_only_fingerprint(); - $this->get_state()->diff = new FileDiffProgressState(); - $this->get_state()->index = new RemoteFileIndexCursorState(); - $this->get_state()->fetch = new FetchListProgressState(); - $this->get_state()->fetch_skipped = new FetchListProgressState(); - $this->get_state()->files_pull_summary = new FilesPullSummaryState(); - $this->save_state(); + $this->finish_fresh_resumable_command_initialization("index"); if ($is_delta) { $this->files_pulled = 0; @@ -3118,6 +3479,8 @@ private function run_files_pull_pipeline(): void */ private function run_files_index(): void { + $fresh_initialization_pending = + $this->finish_pending_artifact_cleanup("files-index"); $state_command = $this->get_state()->active_resumable_command->command_name ?? null; $current_status = $state_command === "files-index" @@ -3130,11 +3493,11 @@ private function run_files_index(): void ); } - if ($current_status === null) { - $this->get_state()->active_resumable_command->command_name = "files-index"; - $this->get_state()->active_resumable_command->completion_state = "in_progress"; - $this->get_state()->active_resumable_command->current_stage = "index"; - $this->save_state(); + if ($current_status === null || $fresh_initialization_pending) { + if (!$fresh_initialization_pending) { + $this->prepare_fresh_resumable_command("files-index"); + } + $this->finish_fresh_resumable_command_initialization("index"); $this->audit_log("START files-index", true); $this->progress->show_lifecycle_line("Starting files-index\n"); $this->output_progress([ @@ -3608,12 +3971,19 @@ private function recreate_intermediate_symlinks(): void */ public function run_db_sync(): void { + $fresh_initialization_pending = + $this->finish_pending_artifact_cleanup("db-pull"); $state_command = $this->get_state()->active_resumable_command->command_name ?? null; $sql_file = $this->state_dir . "/db.sql"; $has_progress = $state_command === "db-pull" && - ($this->get_state()->active_resumable_command->completion_state ?? null) === "in_progress"; + in_array( + $this->get_state()->active_resumable_command->completion_state ?? null, + ["in_progress", "partial"], + true, + ) && + !$fresh_initialization_pending; $current_status = $state_command === "db-pull" ? $this->get_state()->active_resumable_command->completion_state ?? null @@ -3662,13 +4032,10 @@ public function run_db_sync(): void ], true); } else { // Starting fresh - $this->get_state()->active_resumable_command->command_name = "db-pull"; - $this->get_state()->active_resumable_command->completion_state = "in_progress"; - $this->get_state()->active_resumable_command->remote_cursor = null; - $this->get_state()->active_resumable_command->current_stage = "db-index"; - $this->get_state()->diff = new FileDiffProgressState(); - $this->get_state()->db_index = new DatabaseTableIndexState(); - $this->save_state(); + if (!$fresh_initialization_pending) { + $this->prepare_fresh_resumable_command("db-pull"); + } + $this->finish_fresh_resumable_command_initialization("db-index"); $this->audit_log("START db-pull", true); @@ -3682,6 +4049,8 @@ public function run_db_sync(): void } $this->get_state()->active_resumable_command->command_name = "db-pull"; + $this->get_state()->active_resumable_command->completion_state = + "in_progress"; $this->save_state(); // Stage 1: db-index (table metadata for progress estimation) @@ -5222,6 +5591,8 @@ public function run_db_apply(array $options): void "db.sql not found in {$this->state_dir}. Run db-pull first.", ); } + $fresh_initialization_pending = + $this->finish_pending_artifact_cleanup("db-apply"); // If --new-site-url is provided, derive the source origin from the // export URL and add an implicit --rewrite-url mapping. @@ -5275,7 +5646,11 @@ public function run_db_apply(array $options): void $apply_state = $this->get_state()->apply; $statements_executed = $apply_state->statements_executed; $bytes_read = $apply_state->bytes_read; - $is_resume = $current_status === "in_progress" && $statements_executed > 0; + $is_resume = in_array( + $current_status, + ["in_progress", "partial"], + true, + ) && !$fresh_initialization_pending; if ($is_resume) { $this->audit_log( @@ -5296,13 +5671,14 @@ public function run_db_apply(array $options): void "message" => "Resuming db-apply (executed: {$statements_executed} statements)", ], true); } else { - $this->get_state()->active_resumable_command->command_name = "db-apply"; - $this->get_state()->active_resumable_command->completion_state = "in_progress"; - $this->get_state()->apply = new DatabaseApplyCommandState(); + if (!$fresh_initialization_pending) { + $this->prepare_fresh_resumable_command("db-apply"); + } if (!empty($url_mapping)) { $this->get_state()->apply->rewrite_url = $url_mapping; } - $this->save_state(); + $this->finish_fresh_resumable_command_initialization(null); + $apply_state = $this->get_state()->apply; $statements_executed = 0; $bytes_read = 0; @@ -5866,16 +6242,20 @@ private function deactivate_plugins_by_dir(PDO $pdo, array $plugin_dirs, string */ private function run_db_index(): void { + $fresh_initialization_pending = + $this->finish_pending_artifact_cleanup("db-index"); $state_command = $this->get_state()->active_resumable_command->command_name ?? null; $tables_file = $this->state_dir . "/db-tables.jsonl"; - $has_cursor = - $state_command === "db-index" && - !empty($this->get_state()->active_resumable_command->remote_cursor ?? null); $current_status = $state_command === "db-index" ? $this->get_state()->active_resumable_command->completion_state ?? null : null; + $has_progress = in_array( + $current_status, + ["in_progress", "partial"], + true, + ) && !$fresh_initialization_pending; $tables_exists = file_exists($tables_file); if ($current_status === "complete") { @@ -5890,14 +6270,11 @@ private function run_db_index(): void } } - if (!$has_cursor) { - $this->get_state()->active_resumable_command->command_name = "db-index"; - $this->get_state()->active_resumable_command->completion_state = "in_progress"; - $this->get_state()->active_resumable_command->remote_cursor = null; - $this->get_state()->active_resumable_command->current_stage = null; - $this->get_state()->diff = new FileDiffProgressState(); - $this->get_state()->db_index = new DatabaseTableIndexState(); - $this->save_state(); + if (!$has_progress) { + if (!$fresh_initialization_pending) { + $this->prepare_fresh_resumable_command("db-index"); + } + $this->finish_fresh_resumable_command_initialization(null); $this->audit_log("START db-index", true); $this->progress->show_lifecycle_line("Starting db-index\n"); @@ -5908,10 +6285,14 @@ private function run_db_index(): void "message" => "Starting db-index", ], true); } else { + $cursor = + $this->get_state()->active_resumable_command->remote_cursor; $this->audit_log( sprintf( "RESUME db-index | cursor=%s", - substr($this->get_state()->active_resumable_command->remote_cursor, 0, 20) . "...", + $cursor !== null + ? substr($cursor, 0, 20) . "..." + : "none", ), true, ); @@ -5925,6 +6306,8 @@ private function run_db_index(): void } $this->get_state()->active_resumable_command->command_name = "db-index"; + $this->get_state()->active_resumable_command->completion_state = + "in_progress"; $this->save_state(); $this->fetch_database_index(); @@ -6260,7 +6643,16 @@ private function fetch_next_remote_index(?string $list_dir_override = null): boo ); } - $next_remote_index_file_mode = file_exists($this->next_remote_index_file) ? "a" : "w"; + // A base traversal with no cursor is a fresh producer run and replaces + // any stale next index left after an interrupted cleanup. Traversals + // of discovered symlink targets append to that base index. + $next_remote_index_file_mode = + $cursor === null && $list_dir_override === null + ? "w" + : (file_exists($this->next_remote_index_file) ? "a" : "w"); + if ($next_remote_index_file_mode === "w") { + $this->next_remote_index_entries_counted = 0; + } // Initialize the index counter from the existing file so resume // shows a monotonically increasing count. if ($next_remote_index_file_mode === "a" && $this->next_remote_index_entries_counted === 0) { @@ -7454,13 +7846,21 @@ private function remove_remote_index_wal(): void is_file($this->remote_index_wal_path) && filesize($this->remote_index_wal_path) > 0 ) { - throw new RuntimeException("Cannot remove an unapplied remote index WAL."); + // phpcs:disable WordPress.Security.EscapeOutput.ExceptionNotEscaped -- This exception is CLI text, not HTML. + throw new RuntimeException( + "Cannot remove unapplied remote index WAL {$this->remote_index_wal_path}.", + ); + // phpcs:enable WordPress.Security.EscapeOutput.ExceptionNotEscaped } if ( is_file($this->remote_index_wal_path) && !unlink($this->remote_index_wal_path) ) { - throw new RuntimeException("Failed to remove the remote index WAL."); + // phpcs:disable WordPress.Security.EscapeOutput.ExceptionNotEscaped -- This exception is CLI text, not HTML. + throw new RuntimeException( + "Failed to remove remote index WAL {$this->remote_index_wal_path}.", + ); + // phpcs:enable WordPress.Security.EscapeOutput.ExceptionNotEscaped } $this->remote_index_wal_record_count = 0; } @@ -7569,6 +7969,7 @@ private function read_remote_index_wal_record($remote_index_wal_file_handle, ?ar private function fetch_sql(): void { $cursor = $this->get_state()->active_resumable_command->remote_cursor ?? null; + $is_resume = $cursor !== null; $complete = false; $mode = $this->sql_output_mode; @@ -7579,13 +7980,22 @@ private function fetch_sql(): void $sql_buffer_handle = null; $sql_bytes_written = 0; $sql_buffer = ""; + $sql_buffer_file = $this->state_dir . "/pull/sql-buffer"; + if (!$is_resume) { + $this->remove_artifact( + $sql_buffer_file, + "start fresh db-pull SQL download", + ); + } if ($mode === "file") { $sql_file = $this->state_dir . "/db.sql"; // Crash recovery: if SQL file is larger than expected, truncate it. // This happens if we crashed after writing but before saving the new cursor. - $tracked_bytes = $this->get_state()->sql_bytes ?? null; + $tracked_bytes = $is_resume + ? $this->get_state()->sql_bytes ?? null + : null; if ($tracked_bytes !== null && file_exists($sql_file)) { $actual_size = filesize($sql_file); if ($actual_size > $tracked_bytes) { @@ -7605,19 +8015,26 @@ private function fetch_sql(): void } } - $sql_bytes_written = file_exists($sql_file) ? filesize($sql_file) : 0; + $sql_bytes_written = + $is_resume && file_exists($sql_file) + ? filesize($sql_file) + : 0; // Open in write mode if no cursor (starting fresh), append mode if resuming - $sql_handle = fopen($sql_file, $cursor ? "a" : "w"); + $sql_handle = fopen($sql_file, $is_resume ? "a" : "w"); if (!$sql_handle) { throw new RuntimeException("Cannot open SQL file: {$sql_file}"); } } elseif ($mode === "stdout") { - $sql_bytes_written = $this->get_state()->sql_bytes ?? 0; + $sql_bytes_written = $is_resume + ? $this->get_state()->sql_bytes ?? 0 + : 0; } elseif ($mode === "mysql") { - $sql_bytes_written = $this->get_state()->sql_bytes ?? 0; + $sql_bytes_written = $is_resume + ? $this->get_state()->sql_bytes ?? 0 + : 0; $host = $this->mysql_host ?? "127.0.0.1"; $user = $this->mysql_user ?? "root"; @@ -7651,10 +8068,9 @@ private function fetch_sql(): void // Open a persistent buffer file so partial queries survive crashes. // Each SQL chunk is appended to this file as it arrives; when the - // query completes and executes, the file is truncated. If the process - // dies at any point, the next run reloads whatever was accumulated. - $sql_buffer_file = $this->state_dir . "/pull/sql-buffer"; - if (file_exists($sql_buffer_file)) { + // query completes and executes, the file is truncated. A resumed + // run reloads the saved partial query; a fresh run removes it. + if ($is_resume && file_exists($sql_buffer_file)) { $sql_buffer = file_get_contents($sql_buffer_file); $this->audit_log( sprintf("CRASH RECOVERY | Restored %d bytes from pull/sql-buffer", strlen($sql_buffer)), @@ -7678,7 +8094,19 @@ private function fetch_sql(): void : null; $domains_file = $this->state_dir . "/pull/domains.json"; $sql_stats_file = $this->state_dir . "/pull/sql-stats.json"; - $sql_statements_counted = (int) ($this->get_state()->sql_statements_counted ?? 0); + if (!$is_resume) { + $this->remove_artifact( + $domains_file, + "start fresh db-pull SQL download", + ); + $this->remove_artifact( + $sql_stats_file, + "start fresh db-pull SQL download", + ); + } + $sql_statements_counted = $is_resume + ? (int) ($this->get_state()->sql_statements_counted ?? 0) + : 0; // Auto-detect the source site domain from the export URL so it // always appears in pull/domains.json even if the SQL dump @@ -7695,7 +8123,7 @@ private function fetch_sql(): void } // Load previously discovered domains (from earlier partial downloads) - if ($domain_collector && file_exists($domains_file)) { + if ($is_resume && $domain_collector && file_exists($domains_file)) { $prev = json_decode(file_get_contents($domains_file), true); if (is_array($prev)) { $domain_collector->merge($prev); @@ -8015,7 +8443,6 @@ private function fetch_sql(): void $mysql_conn = null; // Clean up buffer file — if we got here with an empty buffer, // all queries were executed successfully. - $sql_buffer_file = $this->state_dir . "/pull/sql-buffer"; if ($pending === "" && file_exists($sql_buffer_file)) { unlink($sql_buffer_file); } @@ -8274,15 +8701,18 @@ private static function sql_starts_with_token(string $sql, int $expected_token_i private function fetch_database_index(): void { $cursor = $this->get_state()->active_resumable_command->remote_cursor ?? null; + $is_resume = $cursor !== null; $complete = false; $tables_file = $this->state_dir . "/db-tables.jsonl"; - $stats = $this->get_state()->db_index; + $stats = $is_resume + ? $this->get_state()->db_index + : new DatabaseTableIndexState(); $tables_written = $stats->tables; $rows_estimated = $stats->rows_estimated; $bytes_written = $stats->bytes; - if ($bytes_written > 0 && file_exists($tables_file)) { + if ($is_resume && $bytes_written > 0 && file_exists($tables_file)) { $actual_size = filesize($tables_file); if ($actual_size > $bytes_written) { $this->audit_log( @@ -8301,7 +8731,7 @@ private function fetch_database_index(): void } } - $handle = fopen($tables_file, $cursor ? "a" : "w"); + $handle = fopen($tables_file, $is_resume ? "a" : "w"); if (!$handle) { throw new RuntimeException("Cannot open table stats file: {$tables_file}"); } @@ -11051,23 +11481,6 @@ function ($ch, $dl_total, $dl_now, $ul_total, $ul_now) { } } - /** - * Reset command state while preserving data shared across commands. - */ - private function reset_state(): void - { - $previous_state = $this->state; - $this->state = new PullState(); - $this->state->preflight = $previous_state->preflight; - $this->state->version = $previous_state->version; - $this->state->webhost = $previous_state->webhost; - $this->state->follow_symlinks = $previous_state->follow_symlinks; - $this->state->fs_root_nonempty_behavior = $previous_state->fs_root_nonempty_behavior; - $this->state->max_allowed_packet = $previous_state->max_allowed_packet; - $this->state->resolved_path_mappings_fingerprint = $previous_state->resolved_path_mappings_fingerprint; - $this->state->pull_pipeline = $previous_state->pull_pipeline; - } - /** Return the in-process pull state. */ public function get_state(): PullState { diff --git a/packages/reprint-importer/src/lib/pull/class-pull.php b/packages/reprint-importer/src/lib/pull/class-pull.php index a428c254..fc89c656 100644 --- a/packages/reprint-importer/src/lib/pull/class-pull.php +++ b/packages/reprint-importer/src/lib/pull/class-pull.php @@ -98,7 +98,10 @@ public function stage_label(string $stage): string /** * Run the pull pipeline. */ - public function run(array $options): void + public function run( + array $options, + bool $resume_pipeline = false + ): void { $this->normalize_url(); $this->progress->set_mode('pipeline'); @@ -109,48 +112,18 @@ public function run(array $options): void 'pull', $this->stages($options), $options, - 'Pulling' + 'Pulling', + $resume_pipeline ); } - /** - * Handle --abort for high-level pull commands. - * - * File pipelines keep downloaded site files in place. The database - * pipeline removes stale database artifacts so the next pull-db fetches - * and applies a fresh dump. - */ - public function abort(string $command = 'pull'): void - { - $state = $this->client->get_state(); - if ( - $command === 'pull-files' - || ( - $command === 'pull' - && $state->active_resumable_command->command_name === 'files-pull' - ) - ) { - $this->client->clear_files_pull_progress(); - } - $this->prepare_repull($command); - $label = $command === 'pull' ? 'Pull' : $command; - $message = "{$label} state cleared."; - $message .= $command === 'pull-db' - ? " Database artifacts will be downloaded again." - : " Downloaded files are preserved."; - $this->progress->show_lifecycle_line("{$message}\n"); - $this->client->output_progress([ - "type" => "lifecycle", - "event" => "aborted", - "command" => $command, - "message" => $message, - ], true); - } - /** * Run only the file stages from the pull pipeline. */ - public function run_pull_files(array $options): void + public function run_pull_files( + array $options, + bool $resume_pipeline = false + ): void { $this->normalize_url(); $this->progress->set_mode('pipeline'); @@ -169,14 +142,18 @@ public function run_pull_files(array $options): void 'pull-files', ['preflight', 'files-pull'], $options, - 'Pulling files from' + 'Pulling files from', + $resume_pipeline ); } /** * Run only the database stages from the pull pipeline. */ - public function run_pull_db(array $options): void + public function run_pull_db( + array $options, + bool $resume_pipeline = false + ): void { $this->normalize_url(); $this->progress->set_mode('pipeline'); @@ -189,7 +166,8 @@ public function run_pull_db(array $options): void 'pull-db', ['preflight', 'db-pull', 'db-apply'], $options, - 'Pulling database from' + 'Pulling database from', + $resume_pipeline ); } @@ -211,122 +189,19 @@ private function run_pipeline( string $command, array $stages, array $options, - string $title + string $title, + bool $resume_pipeline ): void { $state = $this->client->get_state(); - $pull_pipeline = $state->pull_pipeline->started_by_command; - $pull_stage = $state->pull_pipeline->last_completed_stage; - $stage_sequence = $state->pull_pipeline->stage_sequence; - if (!is_array($stage_sequence) || $stage_sequence === []) { - $stage_sequence = $stages; - } - $pipeline_final_stage = $stage_sequence[count($stage_sequence) - 1] ?? null; - $state_command = $state->active_resumable_command->command_name; - $state_status = $state->active_resumable_command->completion_state; - $completed_stage = $pull_pipeline === $command ? $pull_stage : null; - $completed_pipeline = - $pull_pipeline !== null && - $pull_stage !== null && - $pipeline_final_stage !== null && - $pull_stage === $pipeline_final_stage; - - if ($completed_pipeline) { - $this->prepare_repull($command); + if ($resume_pipeline) { + $stages = $state->pull_pipeline->stage_sequence; + $completed_stage = $state->pull_pipeline->last_completed_stage; + } else { + // A completed standalone command is not this pipeline's + // checkpoint. Start every selected stage fresh rather than + // allowing that historical checkpoint to skip its matching stage. + $this->client->prepare_fresh_pull_pipeline($command, $stages); $completed_stage = null; - $pull_pipeline = null; - $pull_stage = null; - $state_command = null; - $state_status = null; - } - - $pipeline_has_resume_state = - $pull_pipeline !== null && - ( - $pull_stage !== null || - $state_command !== null || - $state_status !== null - ); - - if ($pipeline_has_resume_state && $pull_pipeline !== $command) { - throw new RuntimeException( - "Another command is already in progress: {$pull_pipeline}. " . - "Rerun {$pull_pipeline} to resume it. Only use --abort if you want to discard " . - "that pipeline's resume state before running {$command}." - ); - } - - $has_direct_command_state = !$pipeline_has_resume_state && $state_status !== null; - if ( - $has_direct_command_state && - $state_status !== 'complete' && - in_array($state_command, ['files-pull', 'db-pull', 'db-apply'], true) && - !in_array($state_command, $stages, true) - ) { - throw new RuntimeException( - "Another command is already in progress: {$state_command}. " . - "Rerun {$state_command} to resume it. Only use --abort if you want to discard " . - "that command's resume state before running {$command}." - ); - } - - if ($has_direct_command_state && $state_status === 'complete') { - // Users can run lower-level commands directly, e.g. - // `reprint files-pull` or `reprint db-pull`, without going through - // this pull pipeline. Those commands save their own completion - // state in active_resumable_command. That state must not make a - // high-level command skip its matching stage: the pipeline has not - // recorded that stage as complete. Clear the direct command - // checkpoint first so the stage computes a fresh delta. - $state_dir = $this->client->state_dir; - if ($state_command === 'files-pull' && in_array('files-pull', $stages, true)) { - // Keep the remote index, but clear transient files-pull state - // so this pipeline downloads a next remote index and compares - // it with the remote index. - $state = $this->client->get_state(); - $state->active_resumable_command->command_name = null; - $state->active_resumable_command->completion_state = null; - $state->active_resumable_command->remote_cursor = null; - $state->active_resumable_command->current_stage = null; - $state->consecutive_interrupted_responses = 0; - $state->current_file = null; - $state->current_file_bytes = null; - $state->diff = new FileDiffProgressState(); - $state->index = new RemoteFileIndexCursorState(); - $state->fetch = new FetchListProgressState(); - $state->fetch_skipped = new FetchListProgressState(); - $state->files_pull_summary = new FilesPullSummaryState(); - $state->files_pull_only_fingerprint = null; - $this->client->save_state(); - foreach ([ - "{$state_dir}/pull/remote-index.next.jsonl", - "{$state_dir}/pull/fetch-list.jsonl", - "{$state_dir}/pull/skipped-fetch-list.jsonl", - ] as $path) { - if (file_exists($path)) { - @unlink($path); - } - } - } elseif ($state_command === 'db-pull' && in_array('db-pull', $stages, true)) { - // Discard database dump artifacts from any previous runs. - $state = $this->client->get_state(); - $state->active_resumable_command->command_name = null; - $state->active_resumable_command->completion_state = null; - $state->active_resumable_command->remote_cursor = null; - $state->active_resumable_command->current_stage = null; - $state->consecutive_interrupted_responses = 0; - $state->sql_bytes = null; - $state->db_index = new DatabaseTableIndexState(); - $this->client->save_state(); - foreach ([ - "{$state_dir}/db.sql", - "{$state_dir}/db-tables.jsonl", - "{$state_dir}/pull/domains.json", - ] as $path) { - if (file_exists($path)) { - @unlink($path); - } - } - } } $total = count($stages); @@ -757,94 +632,6 @@ private function normalize_url(): void } } - /** - * Reset sub-command state for a delta re-pull. - * - * Keeps preflight data in place, then clears only the checkpoint groups - * owned by the high-level command being restarted. Keeping that ownership - * map here prevents callers from having to know which file/database state - * belongs to which pipeline. - */ - private function prepare_repull(string $command): void - { - $state_dir = $this->client->state_dir; - switch ($command) { - case 'pull': - $reset_file_transfer_state = true; - $reset_file_selection_state = false; - $reset_db_state = true; - break; - - case 'pull-files': - $reset_file_transfer_state = true; - $reset_file_selection_state = true; - $reset_db_state = false; - break; - - case 'pull-db': - $reset_file_transfer_state = false; - $reset_file_selection_state = false; - $reset_db_state = true; - break; - - default: - throw new InvalidArgumentException("Unknown pull command: {$command}"); - } - - - $state = $this->client->get_state(); - $state->pull_pipeline->started_by_command = $command; - $state->pull_pipeline->stage_sequence = []; - $state->pull_pipeline->last_completed_stage = null; - $state->pull_pipeline->files_filter = null; - $state->pull_pipeline->skipped_pending = false; - $state->pull_pipeline->has_completed_once = true; - $state->active_resumable_command->command_name = null; - $state->active_resumable_command->completion_state = null; - $state->active_resumable_command->remote_cursor = null; - $state->active_resumable_command->current_stage = null; - $state->consecutive_interrupted_responses = 0; - if ($reset_file_transfer_state) { - $state->current_file = null; - $state->current_file_bytes = null; - $state->diff = new FileDiffProgressState(); - $state->fetch = new FetchListProgressState(); - $state->fetch_skipped = new FetchListProgressState(); - $state->files_pull_summary = new FilesPullSummaryState(); - } - if ($reset_file_selection_state) { - $state->index = new RemoteFileIndexCursorState(); - $state->files_pull_only_fingerprint = null; - } - if ($reset_db_state) { - $state->sql_bytes = null; - $state->db_index = new DatabaseTableIndexState(); - $state->apply = new DatabaseApplyCommandState(); - $state->sql_output = null; - } - $this->client->save_state(); - - $paths = []; - if ($reset_file_transfer_state) { - $paths[] = $state_dir . "/pull/remote-index.next.jsonl"; - $paths[] = $state_dir . "/pull/fetch-list.jsonl"; - $paths[] = $state_dir . "/pull/skipped-fetch-list.jsonl"; - } - if ($reset_db_state) { - $paths[] = $state_dir . "/db.sql"; - $paths[] = $state_dir . "/db-tables.jsonl"; - $paths[] = $state_dir . "/pull/domains.json"; - } - - foreach ($paths as $path) { - if (file_exists($path)) { - @unlink($path); - } - } - - $this->client->audit_log(strtoupper($command) . " | prepared for delta re-pull", true); - } - /** * Lower-level commands return with completion_state="partial" when a * server timeout drops the connection. This loop retries automatically, diff --git a/packages/reprint-importer/src/lib/state/class-pull-state.php b/packages/reprint-importer/src/lib/state/class-pull-state.php index 2b9dcc8d..633ec476 100644 --- a/packages/reprint-importer/src/lib/state/class-pull-state.php +++ b/packages/reprint-importer/src/lib/state/class-pull-state.php @@ -14,6 +14,9 @@ class ResumableCommandCheckpointState /** @var string|null Lower-level command name, e.g. files-pull/db-pull/db-apply. */ public ?string $command_name = null; + /** @var string|null User-facing command which started this checkpoint. */ + public ?string $started_by_command = null; + /** @var string|null Completion state: in_progress, partial, complete, or null before start. */ public ?string $completion_state = null; @@ -26,8 +29,14 @@ class ResumableCommandCheckpointState public static function from_array(array $data): self { $state = new self(); + // State written before checkpoint provenance was persisted has no + // reliable way to distinguish pipeline work from a later direct run. + if (!array_key_exists('started_by_command', $data)) { + $data['started_by_command'] = null; + } reprint_assert_state_keys($data, array_keys($state->to_array()), self::class); $state->command_name = $data['command_name']; + $state->started_by_command = $data['started_by_command']; $state->completion_state = $data['completion_state']; $state->current_stage = $data['current_stage']; $state->remote_cursor = $data['remote_cursor']; @@ -38,6 +47,7 @@ public function to_array(): array { return [ 'command_name' => $this->command_name, + 'started_by_command' => $this->started_by_command, 'completion_state' => $this->completion_state, 'current_stage' => $this->current_stage, 'remote_cursor' => $this->remote_cursor, @@ -298,7 +308,7 @@ public function to_array(): array class PullPipelineCheckpointState { - /** @var string|null User-facing pipeline command that owns the checkpoint. */ + /** @var string|null User-facing pipeline command that started the checkpoint. */ public ?string $started_by_command = null; /** @var string[] Ordered stage names for the pipeline currently being resumed. */ @@ -346,7 +356,8 @@ public function to_array(): array * In-process pull state with typed properties for each persisted field. * * This object mirrors pull/state.json. Add new persistent state here first; - * from_array() requires the complete current schema. + * from_array() validates the on-disk schema, with explicit nested defaults + * for fields added after state files entered use. */ class PullState { diff --git a/tests/Import/AbortStateTest.php b/tests/Import/AbortStateTest.php new file mode 100644 index 00000000..36ef0646 --- /dev/null +++ b/tests/Import/AbortStateTest.php @@ -0,0 +1,1379 @@ +root = sys_get_temp_dir() + . '/abort-state-' + . bin2hex(random_bytes(6)); + $this->stateDirectory = $this->root . '/state'; + $this->fileRoot = $this->root . '/files'; + mkdir($this->stateDirectory . '/pull', 0700, true); + mkdir($this->fileRoot . '/wp-content', 0700, true); + mkdir($this->root . '/batches', 0700, true); + mkdir($this->root . '/runtime', 0700, true); + } + + protected function tearDown(): void + { + $this->removeTree($this->root); + } + + public function testExplicitPullStateFixtureRoundTrips(): void + { + $fixture = $this->populatedState(); + + $this->assertSame( + $fixture, + \PullState::from_array($fixture)->to_array(), + ); + } + + public function testOwnershipMapCoversEveryResumableCommand(): void + { + $constant = ( + new \ReflectionClass(\ImportClient::class) + )->getReflectionConstant('RESUMABLE_COMMAND_SCOPES'); + $this->assertNotFalse($constant); + $this->assertSame($this->ownershipMap(), $constant->getValue()); + } + + /** + * @dataProvider abortCommandProvider + * + * @param string[] $pipelineStages + */ + public function testPublicAbortClearsOnlyOwnedStateAndArtifacts( + string $command, + string $activeCommand, + array $pipelineStages, + ?string $lastCompletedStage + ): void { + $client = $this->client(); + $before = $this->populatedState(); + $before['active_resumable_command'] = [ + 'command_name' => $activeCommand, + 'started_by_command' => $command, + 'completion_state' => 'partial', + 'current_stage' => 'interrupted-stage', + 'remote_cursor' => 'interrupted-cursor', + ]; + if ($pipelineStages === []) { + $before['pull_pipeline']['started_by_command'] = 'pull'; + $before['pull_pipeline']['stage_sequence'] = [ + 'preflight', + 'files-pull', + 'db-pull', + ]; + $before['pull_pipeline']['last_completed_stage'] = 'db-pull'; + } else { + $before['pull_pipeline']['started_by_command'] = $command; + $before['pull_pipeline']['stage_sequence'] = $pipelineStages; + $before['pull_pipeline']['last_completed_stage'] = + $lastCompletedStage; + } + \write_current_pull_state($client, $before); + $this->createArtifacts(); + + $this->runCommand($client, [ + 'command' => $command, + 'abort' => true, + // Abort ignores normal work options. + 'filter' => 'not-a-filter', + 'fs_root_nonempty_behavior' => 'not-a-behavior', + 'sql_output' => 'not-an-output', + 'target_engine' => 'not-an-engine', + ]); + + $scopes = $this->ownershipMap()[$command]; + $this->assertSame( + $this->expectedStateAfterReset( + $before, + $scopes, + true, + $pipelineStages !== [] ? $command : null, + ), + $this->loadPersistedState($client), + ); + $this->assertArtifactOwnership($scopes); + $this->assertFileExists( + $this->stateDirectory . '/pull/remote-index.jsonl', + ); + $this->assertFileExists( + $this->fileRoot . '/wp-content/downloaded.php', + ); + $this->assertFileExists($this->root . '/database.sqlite'); + $this->assertFileExists($this->root . '/runtime/runtime.php'); + } + + /** @return array */ + public static function abortCommandProvider(): array + { + return [ + 'files-pull' => ['files-pull', 'files-pull', [], null], + 'files-index' => ['files-index', 'files-index', [], null], + 'db-pull' => ['db-pull', 'db-pull', [], null], + 'db-index' => ['db-index', 'db-index', [], null], + 'db-apply' => ['db-apply', 'db-apply', [], null], + 'pull-files' => [ + 'pull-files', + 'files-pull', + ['preflight', 'files-pull'], + 'preflight', + ], + 'pull-db' => [ + 'pull-db', + 'db-pull', + ['preflight', 'db-pull', 'db-apply'], + 'preflight', + ], + 'pull' => [ + 'pull', + 'files-pull', + ['preflight', 'files-pull', 'db-pull'], + 'preflight', + ], + ]; + } + + /** + * @dataProvider unfinishedOwnerProvider + * + * @param array $changes + */ + public function testUnfinishedOwnerMatrix( + array $changes, + ?string $expectedOwner + ): void { + $client = $this->client(); + \write_current_pull_state($client, $changes); + + $method = ( + new \ReflectionClass(\ImportClient::class) + )->getMethod('unfinished_import_owner'); + $this->assertSame($expectedOwner, $method->invoke($client)); + } + + /** @return array,string|null}> */ + public static function unfinishedOwnerProvider(): array + { + $unfinishedPull = [ + 'pull_pipeline' => [ + 'started_by_command' => 'pull', + 'stage_sequence' => [ + 'preflight', + 'files-pull', + 'db-pull', + ], + 'last_completed_stage' => 'preflight', + ], + ]; + + return [ + 'pipeline before lower-level completion' => [ + array_replace_recursive($unfinishedPull, [ + 'active_resumable_command' => [ + 'command_name' => 'files-pull', + 'completion_state' => 'partial', + ], + ]), + 'pull', + ], + 'pipeline after lower-level completion' => [ + array_replace_recursive($unfinishedPull, [ + 'active_resumable_command' => [ + 'command_name' => 'files-pull', + 'completion_state' => 'complete', + ], + ]), + 'pull', + ], + 'partial standalone after completed pipeline' => [ + [ + 'active_resumable_command' => [ + 'command_name' => 'db-index', + 'completion_state' => 'partial', + ], + 'pull_pipeline' => [ + 'started_by_command' => 'pull', + 'stage_sequence' => [ + 'preflight', + 'files-pull', + 'db-pull', + ], + 'last_completed_stage' => 'db-pull', + ], + ], + 'db-index', + ], + 'completed standalone after completed pipeline' => [ + [ + 'active_resumable_command' => [ + 'command_name' => 'db-index', + 'completion_state' => 'complete', + ], + 'pull_pipeline' => [ + 'started_by_command' => 'pull', + 'stage_sequence' => [ + 'preflight', + 'files-pull', + 'db-pull', + ], + 'last_completed_stage' => 'db-pull', + ], + ], + null, + ], + ]; + } + + /** + * @dataProvider rejectedCommandProvider + * + * @param array $stateChanges + */ + public function testRejectedCommandLeavesRawStateAndArtifactsUnchanged( + string $requestedCommand, + bool $abort, + array $stateChanges, + string $expectedOwner + ): void { + $client = $this->client(); + $before = array_replace_recursive( + $this->populatedState(), + $stateChanges, + ); + \write_current_pull_state($client, $before); + $this->createArtifacts(); + $statePath = $this->stateDirectory . '/pull/state.json'; + $rawState = file_get_contents($statePath); + $artifactSnapshot = $this->artifactSnapshot(); + + $caught = null; + try { + $this->runCommand($client, [ + 'command' => $requestedCommand, + 'abort' => $abort, + 'filter' => 'not-a-filter', + 'fs_root_nonempty_behavior' => 'not-a-behavior', + 'sql_output' => 'not-an-output', + 'target_engine' => 'not-an-engine', + ]); + } catch (\RuntimeException $error) { + $caught = $error; + } + + $this->assertInstanceOf(\RuntimeException::class, $caught); + $action = $abort ? 'abort' : 'run'; + $this->assertStringContainsString( + "Cannot {$action} {$requestedCommand} while {$expectedOwner} owns unfinished import state.", + $caught->getMessage(), + ); + $this->assertStringContainsString( + "`reprint {$expectedOwner}`", + $caught->getMessage(), + ); + $this->assertStringContainsString( + "`reprint {$expectedOwner} --abort`", + $caught->getMessage(), + ); + $this->assertSame($rawState, file_get_contents($statePath)); + $this->assertSame($artifactSnapshot, $this->artifactSnapshot()); + } + + /** @return array,string}> */ + public static function rejectedCommandProvider(): array + { + return [ + 'normal command blocked by partial standalone' => [ + 'db-index', + false, + [ + 'active_resumable_command' => [ + 'command_name' => 'db-apply', + 'completion_state' => 'partial', + ], + 'pull_pipeline' => [ + 'last_completed_stage' => 'db-pull', + ], + ], + 'db-apply', + ], + 'abort blocked by in-progress standalone' => [ + 'db-index', + true, + [ + 'active_resumable_command' => [ + 'command_name' => 'db-apply', + 'completion_state' => 'in_progress', + ], + 'pull_pipeline' => [ + 'last_completed_stage' => 'db-pull', + ], + ], + 'db-apply', + ], + 'pipeline takes precedence before stage completion' => [ + 'db-index', + true, + [ + 'active_resumable_command' => [ + 'command_name' => 'files-pull', + 'completion_state' => 'partial', + ], + 'pull_pipeline' => [ + 'started_by_command' => 'pull', + 'stage_sequence' => [ + 'preflight', + 'files-pull', + 'db-pull', + ], + 'last_completed_stage' => 'preflight', + ], + ], + 'pull', + ], + 'pipeline takes precedence after stage completion' => [ + 'files-pull', + true, + [ + 'active_resumable_command' => [ + 'command_name' => 'files-pull', + 'completion_state' => 'complete', + ], + 'pull_pipeline' => [ + 'started_by_command' => 'pull', + 'stage_sequence' => [ + 'preflight', + 'files-pull', + 'db-pull', + ], + 'last_completed_stage' => 'preflight', + ], + ], + 'pull', + ], + 'conflicting pipeline' => [ + 'pull-db', + false, + [ + 'active_resumable_command' => [ + 'command_name' => 'files-pull', + 'completion_state' => 'partial', + ], + 'pull_pipeline' => [ + 'started_by_command' => 'pull-files', + 'stage_sequence' => [ + 'preflight', + 'files-pull', + ], + 'last_completed_stage' => 'preflight', + ], + ], + 'pull-files', + ], + ]; + } + + /** + * @dataProvider completedCheckpointCollisionProvider + * + * @param string[] $effectiveScopes + */ + public function testAbortPreservesDifferentCompletedCheckpoint( + string $requestedCommand, + string $completedCommand, + array $effectiveScopes + ): void { + $client = $this->client(); + $before = $this->populatedState(); + $before['active_resumable_command'] = [ + 'command_name' => $completedCommand, + 'started_by_command' => $completedCommand, + 'completion_state' => 'complete', + 'current_stage' => null, + 'remote_cursor' => null, + ]; + $before['pull_pipeline']['last_completed_stage'] = 'db-pull'; + \write_current_pull_state($client, $before); + $this->createArtifacts(); + + $this->runCommand($client, [ + 'command' => $requestedCommand, + 'abort' => true, + ]); + + $this->assertSame( + $this->expectedStateAfterReset( + $before, + $effectiveScopes, + false, + null, + ), + $this->loadPersistedState($client), + ); + $this->assertArtifactOwnership($effectiveScopes); + } + + public function testCompletedPipelineAbortClearsItsCompletedCheckpoint(): void + { + $client = $this->client(); + $before = $this->populatedState(); + $before['active_resumable_command'] = [ + 'command_name' => 'files-pull', + 'started_by_command' => 'pull-files', + 'completion_state' => 'complete', + 'current_stage' => null, + 'remote_cursor' => null, + ]; + $before['pull_pipeline'] = [ + 'started_by_command' => 'pull-files', + 'stage_sequence' => ['preflight', 'files-pull'], + 'last_completed_stage' => 'files-pull', + 'files_filter' => 'none', + 'skipped_pending' => false, + 'has_completed_once' => true, + ]; + \write_current_pull_state($client, $before); + $this->createArtifacts(); + + $this->runCommand($client, [ + 'command' => 'pull-files', + 'abort' => true, + ]); + + $this->assertSame( + $this->expectedStateAfterReset( + $before, + ['files-index', 'files-pull'], + true, + 'pull-files', + ), + $this->loadPersistedState($client), + ); + $this->assertArtifactOwnership([ + 'files-index', + 'files-pull', + ]); + } + + /** @return array */ + public static function completedCheckpointCollisionProvider(): array + { + return [ + 'files-pull preserves files-index producer' => [ + 'files-pull', + 'files-index', + ['files-pull'], + ], + 'files-index preserves files-pull producer' => [ + 'files-index', + 'files-pull', + [], + ], + 'db-pull preserves db-index producer' => [ + 'db-pull', + 'db-index', + ['db-pull'], + ], + 'db-index preserves db-pull producer' => [ + 'db-index', + 'db-pull', + [], + ], + 'pull-db preserves completed db-apply state' => [ + 'pull-db', + 'db-apply', + ['db-index', 'db-pull'], + ], + ]; + } + + public function testCompletedPipelineAbortPreservesNewerCompletedCheckpoint(): void + { + $client = $this->client(); + $before = $this->populatedState(); + $before['active_resumable_command'] = [ + 'command_name' => 'db-index', + 'started_by_command' => 'db-index', + 'completion_state' => 'complete', + 'current_stage' => null, + 'remote_cursor' => null, + ]; + $before['pull_pipeline'] = [ + 'started_by_command' => 'pull-db', + 'stage_sequence' => [ + 'preflight', + 'db-pull', + 'db-apply', + ], + 'last_completed_stage' => 'db-apply', + 'files_filter' => null, + 'skipped_pending' => false, + 'has_completed_once' => true, + ]; + \write_current_pull_state($client, $before); + $this->createArtifacts(); + + $this->runCommand($client, [ + 'command' => 'pull-db', + 'abort' => true, + ]); + + $this->assertSame( + $this->expectedStateAfterReset( + $before, + ['db-pull', 'db-apply'], + false, + 'pull-db', + ), + $this->loadPersistedState($client), + ); + $this->assertArtifactOwnership(['db-pull', 'db-apply']); + } + + public function testAbortProtectsCompletedCheckpointStarterScopes(): void + { + $client = $this->client(); + $before = $this->populatedState(); + $before['active_resumable_command'] = [ + 'command_name' => 'db-apply', + 'started_by_command' => 'pull-db', + 'completion_state' => 'complete', + 'current_stage' => null, + 'remote_cursor' => null, + ]; + $before['pull_pipeline'] = [ + 'started_by_command' => 'pull-db', + 'stage_sequence' => [ + 'preflight', + 'db-pull', + 'db-apply', + ], + 'last_completed_stage' => 'db-apply', + 'files_filter' => null, + 'skipped_pending' => false, + 'has_completed_once' => true, + ]; + \write_current_pull_state($client, $before); + $this->createArtifacts(); + + $this->runCommand($client, [ + 'command' => 'pull', + 'abort' => true, + ]); + + $this->assertSame( + $this->expectedStateAfterReset( + $before, + ['files-index', 'files-pull'], + false, + null, + ), + $this->loadPersistedState($client), + ); + $this->assertArtifactOwnership(['files-index', 'files-pull']); + } + + public function testLegacyCompletedCheckpointRequiresExactAbort(): void + { + $client = $this->client(); + $before = $this->populatedState(); + $before['active_resumable_command'] = [ + 'command_name' => 'files-pull', + 'started_by_command' => null, + 'completion_state' => 'complete', + 'current_stage' => null, + 'remote_cursor' => null, + ]; + $before['pull_pipeline'] = [ + 'started_by_command' => 'pull-files', + 'stage_sequence' => ['preflight', 'files-pull'], + 'last_completed_stage' => 'files-pull', + 'files_filter' => 'none', + 'skipped_pending' => false, + 'has_completed_once' => true, + ]; + $statePath = $this->stateDirectory . '/pull/state.json'; + \write_current_pull_state($client, $before); + $legacyState = json_decode( + file_get_contents($statePath), + true, + 512, + JSON_THROW_ON_ERROR, + ); + unset( + $legacyState['active_resumable_command'][ + 'started_by_command' + ], + ); + file_put_contents( + $statePath, + json_encode( + $legacyState, + JSON_PRETTY_PRINT | + JSON_UNESCAPED_SLASHES | + JSON_THROW_ON_ERROR, + ), + ); + $this->createArtifacts(); + $rawState = file_get_contents($statePath); + $artifactSnapshot = $this->artifactSnapshot(); + + $caught = null; + try { + $this->runCommand($client, [ + 'command' => 'pull-files', + 'abort' => true, + ]); + } catch (\RuntimeException $error) { + $caught = $error; + } + + $this->assertInstanceOf(\RuntimeException::class, $caught); + $this->assertStringContainsString( + 'the completed files-pull checkpoint does not record which command started it', + $caught->getMessage(), + ); + $this->assertStringContainsString( + '`reprint files-pull --abort`', + $caught->getMessage(), + ); + $this->assertStringContainsString( + '`reprint pull-files --abort`', + $caught->getMessage(), + ); + $this->assertSame($rawState, file_get_contents($statePath)); + $this->assertSame($artifactSnapshot, $this->artifactSnapshot()); + + $this->runCommand($client, [ + 'command' => 'files-pull', + 'abort' => true, + ]); + $this->runCommand($client, [ + 'command' => 'pull-files', + 'abort' => true, + ]); + $state = $this->loadPersistedState($client); + $this->assertNull( + $state['active_resumable_command']['command_name'], + ); + $this->assertNull($state['pull_pipeline']['started_by_command']); + } + + /** + * @dataProvider completedPipelineCollisionProvider + */ + public function testCompletedPipelineCannotStealPartialStandaloneWork( + bool $abort + ): void { + $client = $this->client(); + $before = $this->populatedState(); + $before['active_resumable_command'] = [ + 'command_name' => 'files-pull', + 'started_by_command' => 'files-pull', + 'completion_state' => 'partial', + 'current_stage' => 'fetch-skipped', + 'remote_cursor' => 'fetch-cursor', + ]; + $before['pull_pipeline'] = [ + 'started_by_command' => 'pull', + 'stage_sequence' => [ + 'preflight', + 'files-pull', + 'db-pull', + ], + 'last_completed_stage' => 'db-pull', + 'files_filter' => 'essential-files', + 'skipped_pending' => true, + 'has_completed_once' => true, + ]; + \write_current_pull_state($client, $before); + $rawState = file_get_contents( + $this->stateDirectory . '/pull/state.json', + ); + + $caught = null; + try { + $this->runCommand($client, [ + 'command' => 'pull', + 'abort' => $abort, + 'runtime' => 'none', + ]); + } catch (\RuntimeException $error) { + $caught = $error; + } + + $this->assertInstanceOf(\RuntimeException::class, $caught); + $this->assertStringContainsString( + 'files-pull owns unfinished import state', + $caught->getMessage(), + ); + $this->assertSame( + $rawState, + file_get_contents($this->stateDirectory . '/pull/state.json'), + ); + } + + /** @return array */ + public static function completedPipelineCollisionProvider(): array + { + return [ + 'normal rerun' => [false], + 'pipeline abort' => [true], + ]; + } + + /** + * @dataProvider batchStateProvider + */ + public function testFileAbortRemovesReachableExternalBatch( + string $stateKey, + string $stage + ): void { + $client = $this->client(); + $batchPath = $this->root . "/batches/{$stateKey}.jsonl"; + file_put_contents($batchPath, "batch\n"); + $before = $this->populatedState(); + $before['active_resumable_command'] = [ + 'command_name' => 'files-pull', + 'started_by_command' => 'files-pull', + 'completion_state' => 'partial', + 'current_stage' => $stage, + 'remote_cursor' => 'cursor', + ]; + $before['pull_pipeline']['last_completed_stage'] = 'db-pull'; + $before['fetch']['batch_file'] = null; + $before['fetch_skipped']['batch_file'] = null; + $before[$stateKey]['batch_file'] = $batchPath; + \write_current_pull_state($client, $before); + + $this->runCommand($client, [ + 'command' => 'files-pull', + 'abort' => true, + ]); + + $this->assertFileDoesNotExist($batchPath); + } + + /** @return array */ + public static function batchStateProvider(): array + { + return [ + 'fetch batch' => ['fetch', 'fetch'], + 'fetch-skipped batch' => ['fetch_skipped', 'fetch-skipped'], + ]; + } + + public function testFileAbortReplaysRemoteIndexWalAndRemovesScratchWithoutDeletingDownloadedFiles(): void + { + $client = $this->client(); + $before = $this->populatedState(); + $before['active_resumable_command'] = [ + 'command_name' => 'files-pull', + 'started_by_command' => 'files-pull', + 'completion_state' => 'partial', + 'current_stage' => 'fetch', + 'remote_cursor' => 'cursor', + ]; + $before['pull_pipeline']['last_completed_stage'] = 'db-pull'; + \write_current_pull_state($client, $before); + file_put_contents( + $this->stateDirectory . '/pull/remote-index.jsonl', + $this->indexRecord('/site/existing.txt'), + ); + file_put_contents( + $this->stateDirectory . '/pull/remote-index.wal', + json_encode([ + 'op' => 'F', + 'path' => base64_encode('/site/downloaded.txt'), + 'ctime' => 42, + 'size' => 5, + 'type' => 'file', + ], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n", + ); + foreach ([ + 'pull/remote-index.next.jsonl.sorted', + 'pull/remote-index.next.jsonl.keyed', + 'pull/remote-index.next.jsonl.keyed.sorted', + 'pull/remote-index.next.jsonl.merge-sorted', + 'pull/merge-chunk-stale', + 'pull/remote-index.jsonl.new', + 'pull/remote-index.jsonl.swap', + ] as $artifact) { + file_put_contents( + $this->stateDirectory . '/' . $artifact, + "stale\n", + ); + } + file_put_contents( + $this->fileRoot . '/wp-content/downloaded.php', + "runCommand($client, [ + 'command' => 'files-pull', + 'abort' => true, + ]); + $firstRemoteIndex = file_get_contents( + $this->stateDirectory . '/pull/remote-index.jsonl', + ); + $this->runCommand($client, [ + 'command' => 'files-pull', + 'abort' => true, + ]); + + $this->assertSame( + $firstRemoteIndex, + file_get_contents( + $this->stateDirectory . '/pull/remote-index.jsonl', + ), + ); + $this->assertStringContainsString( + base64_encode('/site/existing.txt'), + $firstRemoteIndex, + ); + $this->assertStringContainsString( + base64_encode('/site/downloaded.txt'), + $firstRemoteIndex, + ); + $this->assertFileDoesNotExist( + $this->stateDirectory . '/pull/remote-index.wal', + ); + $this->assertFileExists( + $this->fileRoot . '/wp-content/downloaded.php', + ); + $this->assertFalse( + $this->loadPersistedState($client)['pull_pipeline'][ + 'skipped_pending' + ], + ); + } + + /** + * @dataProvider completionHistoryProvider + */ + public function testPipelineAbortPreservesCompletionHistory( + bool $hasCompletedOnce + ): void { + $client = $this->client(); + $before = $this->populatedState(); + $before['active_resumable_command'] = [ + 'command_name' => 'db-pull', + 'started_by_command' => 'pull-db', + 'completion_state' => 'partial', + 'current_stage' => 'sql', + 'remote_cursor' => 'cursor', + ]; + $before['pull_pipeline'] = [ + 'started_by_command' => 'pull-db', + 'stage_sequence' => [ + 'preflight', + 'db-pull', + 'db-apply', + ], + 'last_completed_stage' => 'preflight', + 'files_filter' => null, + 'skipped_pending' => false, + 'has_completed_once' => $hasCompletedOnce, + ]; + \write_current_pull_state($client, $before); + + $this->runCommand($client, [ + 'command' => 'pull-db', + 'abort' => true, + ]); + $this->runCommand($client, [ + 'command' => 'pull-db', + 'abort' => true, + ]); + + $pipeline = $this->loadPersistedState($client)['pull_pipeline']; + $this->assertNull($pipeline['started_by_command']); + $this->assertSame([], $pipeline['stage_sequence']); + $this->assertNull($pipeline['last_completed_stage']); + $this->assertSame( + $hasCompletedOnce, + $pipeline['has_completed_once'], + ); + } + + /** @return array */ + public static function completionHistoryProvider(): array + { + return [ + 'never completed' => [false], + 'completed before' => [true], + ]; + } + + public function testAbortReportsCorruptArtifactPathAfterSavingResetState(): void + { + $client = $this->client(); + $before = $this->populatedState(); + $before['active_resumable_command'] = [ + 'command_name' => 'files-index', + 'started_by_command' => 'files-index', + 'completion_state' => 'partial', + 'current_stage' => 'index', + 'remote_cursor' => 'cursor', + ]; + $before['pull_pipeline']['last_completed_stage'] = 'db-pull'; + \write_current_pull_state($client, $before); + $nextRemoteIndex = + $this->stateDirectory . '/pull/remote-index.next.jsonl'; + mkdir($nextRemoteIndex); + + $caught = null; + try { + $this->runCommand($client, [ + 'command' => 'files-index', + 'abort' => true, + ]); + } catch (\RuntimeException $error) { + $caught = $error; + } + + $this->assertInstanceOf(\RuntimeException::class, $caught); + $this->assertStringContainsString( + $nextRemoteIndex, + $caught->getMessage(), + ); + $this->assertSame( + ( new \ResumableCommandCheckpointState() )->to_array(), + $this->loadPersistedState($client)[ + 'active_resumable_command' + ], + ); + + rmdir($nextRemoteIndex); + $this->runCommand($client, [ + 'command' => 'files-index', + 'abort' => true, + ]); + } + + /** @return array */ + private function ownershipMap(): array + { + return [ + 'files-index' => ['files-index'], + 'files-pull' => ['files-index', 'files-pull'], + 'db-index' => ['db-index'], + 'db-pull' => ['db-index', 'db-pull'], + 'db-apply' => ['db-apply'], + 'pull-files' => ['files-index', 'files-pull'], + 'pull-db' => ['db-index', 'db-pull', 'db-apply'], + 'pull' => [ + 'files-index', + 'files-pull', + 'db-index', + 'db-pull', + 'db-apply', + ], + ]; + } + + /** + * Populate every state group so schema additions require an ownership decision. + * + * @return array + */ + private function populatedState(): array + { + return [ + 'active_resumable_command' => [ + 'command_name' => 'previous-command', + 'started_by_command' => 'previous-command', + 'completion_state' => 'complete', + 'current_stage' => 'previous-stage', + 'remote_cursor' => 'remote-cursor', + ], + 'preflight' => [ + 'data' => ['ok' => true], + 'http_code' => 200, + ], + 'remote_protocol_version' => 1, + 'version' => '0.9.3-dev', + 'webhost' => 'wpcloud', + 'follow_symlinks' => false, + 'local_followed_symlinks_root_fingerprint' => 'followed-root', + 'fs_root_nonempty_behavior' => 'preserve-local', + 'filter' => 'essential-files', + 'user_agent' => 'Reprint test', + 'max_allowed_packet' => 1048576, + 'resolved_path_mappings_fingerprint' => 'path-mappings', + 'files_pull_only_fingerprint' => 'only-files', + 'files_pull_summary' => [ + 'files_pulled' => 42, + ], + 'db_index' => [ + 'file' => $this->stateDirectory . '/db-tables.jsonl', + 'tables' => 3, + 'rows_estimated' => 120, + 'bytes' => 2048, + 'updated_at' => '1234567890', + ], + 'diff' => [ + 'next_remote_index_byte_offset' => 64, + 'last_consumed_remote_index_entry_path' => + '/remote/wp-content', + ], + 'index' => [ + 'cursor' => 'file-index-cursor', + ], + 'fetch' => [ + 'offset' => 128, + 'next_offset' => 256, + 'batch_file' => $this->root . '/batches/fetch.jsonl', + 'cursor' => 'fetch-cursor', + 'batch_entries' => 5, + ], + 'fetch_skipped' => [ + 'offset' => 512, + 'next_offset' => 1024, + 'batch_file' => $this->root . '/batches/fetch-skipped.jsonl', + 'cursor' => 'skipped-cursor', + 'batch_entries' => 7, + ], + 'current_file' => + $this->fileRoot . '/wp-content/current.php', + 'current_file_bytes' => 4096, + 'sql_bytes' => 8192, + 'sql_statements_counted' => 99, + 'apply' => [ + 'statements_executed' => 17, + 'bytes_read' => 16384, + 'rewrite_url' => [ + 'https://source.example' => + 'https://local.example', + ], + 'target_engine' => 'sqlite', + 'target_db' => 'local_db', + 'target_host' => '127.0.0.1', + 'target_port' => 3307, + 'target_user' => 'local_user', + 'target_pass' => 'local_pass', + 'target_sqlite_path' => $this->root . '/database.sqlite', + 'remote_paths_removed_from_local_site' => [ + 'wp-content/object-cache.php', + ], + ], + 'sql_output' => 'mysql', + 'mysql_host' => 'database.example', + 'mysql_port' => 3308, + 'mysql_user' => 'stream_user', + 'mysql_database' => 'stream_db', + 'consecutive_interrupted_responses' => 4, + 'tuning' => [ + 'config' => ['enabled' => true], + 'state' => ['file_chunk_bytes' => 2097152], + ], + 'pull_pipeline' => [ + 'started_by_command' => 'pull', + 'stage_sequence' => [ + 'preflight', + 'files-pull', + 'db-pull', + ], + 'last_completed_stage' => 'files-pull', + 'files_filter' => 'essential-files', + 'skipped_pending' => true, + 'has_completed_once' => true, + ], + ]; + } + + /** + * @param array $before + * @param string[] $scopes + * @return array + */ + private function expectedStateAfterReset( + array $before, + array $scopes, + bool $clearCheckpoint, + ?string $resetPipeline + ): array { + $expected = $before; + if ($clearCheckpoint) { + $expected['active_resumable_command'] = ( + new \ResumableCommandCheckpointState() + )->to_array(); + $expected['consecutive_interrupted_responses'] = 0; + } + + if (in_array('files-pull', $scopes, true)) { + $expected['local_followed_symlinks_root_fingerprint'] = null; + $expected['filter'] = 'none'; + $expected['files_pull_only_fingerprint'] = null; + $expected['files_pull_summary'] = ( + new \FilesPullSummaryState() + )->to_array(); + $expected['diff'] = ( + new \FileDiffProgressState() + )->to_array(); + $expected['fetch'] = ( + new \FetchListProgressState() + )->to_array(); + $expected['fetch_skipped'] = ( + new \FetchListProgressState() + )->to_array(); + $expected['current_file'] = null; + $expected['current_file_bytes'] = null; + $expected['pull_pipeline']['skipped_pending'] = false; + } + if (in_array('files-index', $scopes, true)) { + $expected['index'] = ( + new \RemoteFileIndexCursorState() + )->to_array(); + } + if (in_array('db-pull', $scopes, true)) { + $expected['sql_bytes'] = null; + $expected['sql_statements_counted'] = 0; + $expected['sql_output'] = null; + $expected['mysql_host'] = null; + $expected['mysql_port'] = null; + $expected['mysql_user'] = null; + $expected['mysql_database'] = null; + } + if (in_array('db-index', $scopes, true)) { + $expected['db_index'] = ( + new \DatabaseTableIndexState() + )->to_array(); + } + if (in_array('db-apply', $scopes, true)) { + $expected['apply'] = ( + new \DatabaseApplyCommandState() + )->to_array(); + } + if ( + $resetPipeline !== null && + $before['pull_pipeline']['started_by_command'] === + $resetPipeline + ) { + $hasCompletedOnce = + $before['pull_pipeline']['has_completed_once']; + $expected['pull_pipeline'] = ( + new \PullPipelineCheckpointState() + )->to_array(); + $expected['pull_pipeline']['has_completed_once'] = + $hasCompletedOnce; + } + + return $expected; + } + + private function createArtifacts(): void + { + foreach ($this->artifactPaths() as $name => $path) { + if ($name === 'runtime' || $name === 'database') { + file_put_contents($path, "preserve\n"); + continue; + } + if ($name === 'remote-index') { + file_put_contents( + $path, + $this->indexRecord('/site/existing.txt'), + ); + continue; + } + if ($name === 'remote-index-wal') { + file_put_contents($path, ''); + continue; + } + file_put_contents($path, "{$name}\n"); + } + file_put_contents( + $this->fileRoot . '/wp-content/downloaded.php', + " */ + private function artifactPaths(): array + { + return [ + 'remote-index' => + $this->stateDirectory . '/pull/remote-index.jsonl', + 'remote-index-wal' => + $this->stateDirectory . '/pull/remote-index.wal', + 'remote-index-new' => + $this->stateDirectory . '/pull/remote-index.jsonl.new', + 'remote-index-swap' => + $this->stateDirectory . '/pull/remote-index.jsonl.swap', + 'next-remote-index' => + $this->stateDirectory . '/pull/remote-index.next.jsonl', + 'next-remote-index-sorted' => + $this->stateDirectory . '/pull/remote-index.next.jsonl.sorted', + 'next-remote-index-keyed' => + $this->stateDirectory . '/pull/remote-index.next.jsonl.keyed', + 'next-remote-index-keyed-sorted' => + $this->stateDirectory . + '/pull/remote-index.next.jsonl.keyed.sorted', + 'next-remote-index-merge-sorted' => + $this->stateDirectory . + '/pull/remote-index.next.jsonl.merge-sorted', + 'merge-chunk' => + $this->stateDirectory . '/pull/merge-chunk-stale', + 'fetch-list' => + $this->stateDirectory . '/pull/fetch-list.jsonl', + 'skipped-fetch-list' => + $this->stateDirectory . + '/pull/skipped-fetch-list.jsonl', + 'volatile-files' => + $this->stateDirectory . '/pull/volatile-files.json', + 'fetch-batch' => $this->root . '/batches/fetch.jsonl', + 'fetch-skipped-batch' => + $this->root . '/batches/fetch-skipped.jsonl', + 'sql-buffer' => + $this->stateDirectory . '/pull/sql-buffer', + 'sql-stats' => + $this->stateDirectory . '/pull/sql-stats.json', + 'sql' => $this->stateDirectory . '/db.sql', + 'db-index' => + $this->stateDirectory . '/db-tables.jsonl', + 'domains' => + $this->stateDirectory . '/pull/domains.json', + 'database' => $this->root . '/database.sqlite', + 'runtime' => $this->root . '/runtime/runtime.php', + ]; + } + + /** + * @param string[] $scopes + */ + private function assertArtifactOwnership(array $scopes): void + { + $removed = []; + if (in_array('files-pull', $scopes, true)) { + $removed = array_merge($removed, [ + 'remote-index-wal', + 'remote-index-new', + 'remote-index-swap', + 'fetch-list', + 'skipped-fetch-list', + 'volatile-files', + 'fetch-batch', + 'fetch-skipped-batch', + ]); + } + if (in_array('files-index', $scopes, true)) { + $removed = array_merge($removed, [ + 'next-remote-index', + 'next-remote-index-sorted', + 'next-remote-index-keyed', + 'next-remote-index-keyed-sorted', + 'next-remote-index-merge-sorted', + 'merge-chunk', + ]); + } + if (in_array('db-pull', $scopes, true)) { + $removed = array_merge($removed, [ + 'sql-buffer', + 'sql-stats', + 'sql', + 'domains', + ]); + } + if (in_array('db-index', $scopes, true)) { + $removed[] = 'db-index'; + } + + foreach ($this->artifactPaths() as $name => $path) { + if (in_array($name, $removed, true)) { + $this->assertFileDoesNotExist($path, $name); + } else { + $this->assertFileExists($path, $name); + } + } + } + + /** @return array */ + private function artifactSnapshot(): array + { + $snapshot = []; + foreach ($this->artifactPaths() as $name => $path) { + $contents = file_get_contents($path); + $this->assertIsString($contents); + $snapshot[$name] = $contents; + } + return $snapshot; + } + + private function indexRecord(string $path): string + { + return json_encode([ + 'path' => base64_encode($path), + 'ctime' => 42, + 'size' => 5, + 'type' => 'file', + ], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n"; + } + + /** + * @param array $options + */ + private function runCommand( + \ImportClient $client, + array $options + ): void { + $processLock = new \ReprintProcessLock($this->stateDirectory); + ob_start(); + try { + $client->run($options, $processLock); + } finally { + ob_end_clean(); + $processLock->close(); + } + } + + private function client(): \ImportClient + { + return new \ImportClient( + 'https://example.com/?site-export-api', + $this->stateDirectory, + $this->fileRoot, + ); + } + + /** @return array */ + private function loadPersistedState(\ImportClient $client): array + { + $method = ( + new \ReflectionClass(\ImportClient::class) + )->getMethod('load_state'); + return $method->invoke($client)->to_array(); + } + + private function removeTree(string $path): void + { + if (!file_exists($path) && !is_link($path)) { + return; + } + if (is_dir($path) && !is_link($path)) { + foreach (scandir($path) ?: [] as $entry) { + if ($entry !== '.' && $entry !== '..') { + $this->removeTree($path . '/' . $entry); + } + } + rmdir($path); + return; + } + unlink($path); + } +} diff --git a/tests/Import/FilesPullStateTest.php b/tests/Import/FilesPullStateTest.php index 7b43c583..9973201c 100644 --- a/tests/Import/FilesPullStateTest.php +++ b/tests/Import/FilesPullStateTest.php @@ -219,10 +219,7 @@ public function testAbortClearsCompletedStatus() ], ]); - [$client, $reflection] = $this->prepareClient(); - - $abortMethod = $reflection->getMethod('handle_abort'); - $abortMethod->invoke($client, 'files-pull'); + $this->abortFilesPull(); $state = $this->readState(); $this->assertNotEquals( @@ -250,8 +247,7 @@ public function testAbortThenRerunStartsFresh() ]); // Step 1: abort - [$client, $reflection] = $this->prepareClient(); - $reflection->getMethod('handle_abort')->invoke($client, 'files-pull'); + $this->abortFilesPull(); // Step 2: new client, try run_files_pull [$client2, $reflection2] = $this->prepareClient(); @@ -458,6 +454,28 @@ public function testFetchStageOverwritesPreviouslySyncedFile() "Fetch stage must overwrite existing files that were placed in the fetch list", ); } + + /** + * Abort files-pull through the public command lifecycle. + */ + private function abortFilesPull(): void + { + $client = $this->makeClient(); + $processLock = new \ReprintProcessLock($this->stateDir); + ob_start(); + try { + $client->run( + [ + 'command' => 'files-pull', + 'abort' => true, + ], + $processLock, + ); + } finally { + ob_end_clean(); + $processLock->close(); + } + } } /** diff --git a/tests/Import/NewSiteUrlSqliteTest.php b/tests/Import/NewSiteUrlSqliteTest.php index 26cb53d7..02b8d325 100644 --- a/tests/Import/NewSiteUrlSqliteTest.php +++ b/tests/Import/NewSiteUrlSqliteTest.php @@ -409,4 +409,72 @@ public function testSqlitePragmasDoNotChangeProgressCounters(): void $this->assertSame(3, $state['apply']['statements_executed']); $this->assertSame(strlen($sql), $state['apply']['bytes_read']); } + + public function testInterruptedFreshInitializationPersistsRewriteMapForResume(): void + { + $oldUrl = 'https://old-site.example.com'; + $newUrl = 'https://new-site.example.com'; + $sqlitePath = $this->tempDir . '/database/wordpress.sqlite'; + file_put_contents( + $this->tempDir . '/db.sql', + "INSERT INTO missing_table VALUES (1);\n", + ); + $this->writeState([ + 'active_resumable_command' => [ + 'command_name' => 'db-apply', + 'started_by_command' => 'db-apply', + 'completion_state' => 'in_progress', + 'current_stage' => 'fresh-initialization', + 'remote_cursor' => null, + ], + ]); + $client = new \ImportClient( + 'https://old-site.example.com/?reprint-api', + $this->tempDir, + $this->tempDir . '/fs-root', + ); + + $caught = null; + try { + $client->run([ + 'command' => 'db-apply', + 'target_engine' => 'sqlite', + 'target_sqlite_path' => $sqlitePath, + 'target_db' => 'wp_test', + 'rewrite_url' => [[$oldUrl, $newUrl]], + ]); + } catch (\RuntimeException $error) { + $caught = $error; + } + + $this->assertInstanceOf(\RuntimeException::class, $caught); + $state = json_decode( + file_get_contents($this->tempDir . '/pull/state.json'), + true, + 512, + JSON_THROW_ON_ERROR, + ); + $this->assertSame( + [$oldUrl => $newUrl], + $state['apply']['rewrite_url'], + ); + + file_put_contents( + $this->tempDir . '/db.sql', + $this->buildSqlDump($oldUrl), + ); + $client->run([ + 'command' => 'db-apply', + 'target_engine' => 'sqlite', + 'target_sqlite_path' => $sqlitePath, + 'target_db' => 'wp_test', + ]); + + $rows = $this->querySqlite( + $sqlitePath, + "SELECT option_value FROM wp_options WHERE option_name = 'siteurl'", + 'wp_test', + ); + $this->assertSame($newUrl, $rows[0]['option_value']); + } } diff --git a/tests/Import/OnlyCliParseTest.php b/tests/Import/OnlyCliParseTest.php index 3b5c96e2..cc4090ca 100644 --- a/tests/Import/OnlyCliParseTest.php +++ b/tests/Import/OnlyCliParseTest.php @@ -4,6 +4,8 @@ use PHPUnit\Framework\TestCase; +require_once __DIR__ . '/../../importer/import.php'; + /** * --only reuses the existing `value-or-next` option type (like --new-site-url), * but is repeatable because commas are valid path bytes. The parser lives @@ -226,7 +228,6 @@ public function testRepeatedOnlyOptionsAreAllPreserved(): void ':abspath:/wp-admin', '--only', ':wp-content:', - '--abort', '--state-dir=' . $this->tempDir . '/state', '--fs-root=' . $this->tempDir . '/fs', )); @@ -340,23 +341,208 @@ public function testRepeatedOnlyOptionsAreUsedByFilesPull(): void ), $fileIndexRequest['directory'] ?? null); } - public function testOnlyOptionKeepsCommaInsideSourcePath(): void + public function testFreshFilesIndexReplacesStaleIndexAfterStateReset(): void + { + $this->writePreflightState(true); + $this->writeStateAtCleanupInterruptionBoundary('files-index'); + $nextRemoteIndex = $this->tempDir + . '/state/pull/remote-index.next.jsonl'; + file_put_contents($nextRemoteIndex, "stale next remote index\n"); + + $requestsLog = $this->tempDir . '/fresh-file-requests.jsonl'; + $remoteUrl = $this->startDirectoryCaptureServer($requestsLog); + $output = $this->runCli(array( + 'files-index', + $remoteUrl, + '--state-dir=' . $this->tempDir . '/state', + '--fs-root=' . $this->tempDir . '/fs', + )); + + $this->assertStringNotContainsString( + '"status":"error"', + $output, + ); + $this->assertSame('', file_get_contents($nextRemoteIndex)); + $this->assertContains( + 'file_index', + array_column( + $this->capturedRequests($requestsLog), + 'endpoint', + ), + ); + } + + public function testInterruptedFreshFilesPullRepeatsArtifactCleanup(): void + { + $this->writePreflightState(true); + $this->writeStateAtCleanupInterruptionBoundary('files-pull'); + $volatileFiles = $this->tempDir + . '/state/pull/volatile-files.json'; + $mergeChunk = $this->tempDir + . '/state/pull/merge-chunk-stale'; + file_put_contents( + $volatileFiles, + json_encode( + array('/wp-content/volatile.php' => 1), + JSON_THROW_ON_ERROR, + ), + ); + file_put_contents($mergeChunk, "stale merge chunk\n"); + + $requestsLog = $this->tempDir + . '/fresh-files-cleanup-requests.jsonl'; + $remoteUrl = $this->startDirectoryCaptureServer($requestsLog); + $output = $this->runCli(array( + 'files-pull', + $remoteUrl, + '--state-dir=' . $this->tempDir . '/state', + '--fs-root=' . $this->tempDir . '/fs', + )); + + $this->assertStringNotContainsString('"status":"error"', $output); + $this->assertStringNotContainsString( + '"type":"volatile_files"', + $output, + ); + $this->assertFileDoesNotExist($volatileFiles); + $this->assertFileDoesNotExist($mergeChunk); + } + + public function testFreshDatabasePullReplacesStaleDownloadArtifacts(): void { - // --abort runs after --only resolution, avoiding a network request while - // still proving the CLI did not split the SOURCE at the comma. $this->writePreflightState(); + $this->writeStateAtCleanupInterruptionBoundary('db-pull'); + $artifacts = array( + $this->tempDir . '/state/db.sql', + $this->tempDir . '/state/db-tables.jsonl', + $this->tempDir . '/state/pull/domains.json', + $this->tempDir . '/state/pull/sql-buffer', + $this->tempDir . '/state/pull/sql-stats.json', + ); + foreach ($artifacts as $artifact) { + file_put_contents($artifact, "stale download\n"); + } + + $requestsLog = $this->tempDir . '/fresh-db-requests.jsonl'; + $remoteUrl = $this->startDirectoryCaptureServer($requestsLog); + $output = $this->runCli(array( + 'db-pull', + $remoteUrl, + '--state-dir=' . $this->tempDir . '/state', + '--fs-root=' . $this->tempDir . '/fs', + )); + + $this->assertStringNotContainsString( + '"status":"error"', + $output, + ); + $this->assertSame('', file_get_contents($artifacts[0])); + $this->assertSame('', file_get_contents($artifacts[1])); + $this->assertFileDoesNotExist($artifacts[3]); + $this->assertFileDoesNotExist($artifacts[4]); + if (file_exists($artifacts[2])) { + $this->assertStringNotContainsString( + 'stale download', + file_get_contents($artifacts[2]), + ); + } + $this->assertSame( + array('db_index', 'sql_chunk'), + array_column( + $this->capturedRequests($requestsLog), + 'endpoint', + ), + ); + } + + public function testFreshMysqlPullDoesNotRestoreStaleSqlBuffer(): void + { + if (!class_exists(\mysqli::class)) { + $this->markTestSkipped('The mysqli extension is unavailable.'); + } + + $this->writePreflightState(); + $this->writeStateAtCleanupInterruptionBoundary('db-pull'); + $sqlBuffer = $this->tempDir . '/state/pull/sql-buffer'; + file_put_contents($sqlBuffer, "stale partial query\n"); + + $requestsLog = $this->tempDir . '/fresh-mysql-requests.jsonl'; + $remoteUrl = $this->startDirectoryCaptureServer($requestsLog); + $mysqlPort = $this->findUnusedPort(); + $output = $this->runCli(array( + 'db-pull', + $remoteUrl, + '--sql-output=mysql', + '--mysql-host=127.0.0.1', + '--mysql-port=' . $mysqlPort, + '--mysql-user=root', + '--mysql-database=reprint', + '--state-dir=' . $this->tempDir . '/state', + '--fs-root=' . $this->tempDir . '/fs', + )); + + $this->assertStringContainsString('mysqli', strtolower($output)); + $this->assertFileDoesNotExist($sqlBuffer); + $this->assertContains( + 'db_index', + array_column( + $this->capturedRequests($requestsLog), + 'endpoint', + ), + ); + } + + public function testAbortIgnoresOnlyResolutionWithoutNetworkRequest(): void + { + $this->writePreflightState(); + $requestsLog = $this->tempDir . '/abort-requests.jsonl'; + $remoteUrl = $this->startDirectoryCaptureServer($requestsLog); $output = $this->runCli(array( 'files-pull', - 'http://fake.invalid/?site-export-api', + $remoteUrl, '--only', - ':wp-content:/plugins,custom', + ':missing-token:/plugins', '--abort', '--state-dir=' . $this->tempDir . '/state', '--fs-root=' . $this->tempDir . '/fs', )); $this->assertStringContainsString('"status":"aborted"', $output); - $this->assertStringNotContainsString('path "custom"', $output); + $this->assertStringNotContainsString('Cannot resolve token', $output); + $this->assertSame(array(), $this->capturedRequests($requestsLog)); + } + + /** + * Write the durable checkpoint saved before fixed artifact cleanup. + */ + private function writeStateAtCleanupInterruptionBoundary( + string $command + ): void { + // Fresh-start preparation saves this checkpoint before deleting fixed + // transient artifacts. A process may stop between those actions. + $statePath = $this->tempDir . '/state/pull/state.json'; + $state = json_decode( + file_get_contents($statePath), + true, + 512, + JSON_THROW_ON_ERROR, + ); + $state['active_resumable_command'] = array( + 'command_name' => $command, + 'started_by_command' => $command, + 'completion_state' => 'in_progress', + 'current_stage' => 'artifact-cleanup', + 'remote_cursor' => null, + ); + file_put_contents( + $statePath, + json_encode( + $state, + JSON_PRETTY_PRINT | + JSON_UNESCAPED_SLASHES | + JSON_THROW_ON_ERROR, + ), + ); } } diff --git a/tests/Import/PullFilterOptionTest.php b/tests/Import/PullFilterOptionTest.php index 4cfab737..86b3a5d7 100644 --- a/tests/Import/PullFilterOptionTest.php +++ b/tests/Import/PullFilterOptionTest.php @@ -15,6 +15,7 @@ class PullFilterFakeClient extends \ImportClient public int $db_sync_runs = 0; public int $db_apply_runs = 0; public array $progress_events = []; + public array $progress_event_force_flags = []; public array $progress_file_errors = []; /** @var resource|null */ @@ -33,6 +34,7 @@ public function audit_log(string $message, bool $to_console = true): void public function output_progress(array $data, bool $force = false): void { $this->progress_events[] = $data; + $this->progress_event_force_flags[] = $force; } public function write_progress_file(?string $error = null): void @@ -241,6 +243,94 @@ private function writeState(array $state): void \write_current_pull_state($this->makeClient(false), $state); } + public function testHighLevelAbortForcesLifecycleEvent(): void + { + $client = $this->makeClient(false); + + ob_start(); + $client->run([ + 'command' => 'pull-files', + ]); + $completedState = $this->readState(); + $client->run([ + 'command' => 'pull-files', + 'abort' => true, + ]); + ob_end_clean(); + + $this->assertSame( + 'pull-files', + $completedState['active_resumable_command'][ + 'started_by_command' + ], + ); + $abortedState = $this->readState(); + $this->assertNull( + $abortedState['active_resumable_command']['command_name'], + ); + $this->assertNull( + $abortedState['pull_pipeline']['started_by_command'], + ); + + $eventIndex = null; + foreach ($client->progress_events as $index => $event) { + $eventName = $event['event'] ?? null; + if ($eventName === 'aborted') { + $eventIndex = $index; + break; + } + } + $this->assertNotNull($eventIndex); + $this->assertSame( + [ + 'type' => 'lifecycle', + 'event' => 'aborted', + 'command' => 'pull-files', + 'message' => 'State cleared for pull-files. Downloaded files and pull/remote-index.jsonl were preserved.', + ], + $client->progress_events[$eventIndex], + ); + $this->assertTrue($client->progress_event_force_flags[$eventIndex]); + } + + public function testDirectAbortForcesStatusRecord(): void + { + $this->writeState([ + 'active_resumable_command' => [ + 'command_name' => 'files-index', + 'started_by_command' => 'files-index', + 'completion_state' => 'partial', + 'current_stage' => 'index', + ], + ]); + $client = $this->makeClient(false); + + ob_start(); + $client->run([ + 'command' => 'files-index', + 'abort' => true, + ]); + ob_end_clean(); + + $eventIndex = null; + foreach ($client->progress_events as $index => $event) { + $status = $event['status'] ?? null; + if ($status === 'aborted') { + $eventIndex = $index; + break; + } + } + $this->assertNotNull($eventIndex); + $this->assertSame( + [ + 'status' => 'aborted', + 'message' => 'State cleared for files-index.', + ], + $client->progress_events[$eventIndex], + ); + $this->assertTrue($client->progress_event_force_flags[$eventIndex]); + } + public function testPullRejectsSkippedEarlierFilterBeforePersistingIt(): void { $client = $this->makeClient(false); @@ -311,6 +401,7 @@ public function testPullResumesAfterFilesPullCompletedBeforePipelineStageWasMark ], "pull_pipeline" => [ "started_by_command" => "pull", + "stage_sequence" => ["preflight", "files-pull", "db-pull", "db-apply"], "last_completed_stage" => "preflight", ], "preflight" => ["http_code" => 200, "data" => ["ok" => true]], @@ -342,6 +433,7 @@ public function testPullResumesSameUnfinishedPipelineWithoutConflict(): void ], "pull_pipeline" => [ "started_by_command" => "pull", + "stage_sequence" => ["preflight", "files-pull", "db-pull", "db-apply"], "last_completed_stage" => "preflight", ], "preflight" => ["http_code" => 200, "data" => ["ok" => true]], @@ -372,6 +464,7 @@ public function testPullRefusesToClearCompletedCommandOwnedByDifferentUnfinished ], "pull_pipeline" => [ "started_by_command" => "pull-db", + "stage_sequence" => ["preflight", "db-pull", "db-apply"], "last_completed_stage" => null, ], "preflight" => ["http_code" => 200, "data" => ["ok" => true]], @@ -388,9 +481,9 @@ public function testPullRefusesToClearCompletedCommandOwnedByDifferentUnfinished ]); $this->fail('Expected pull to refuse a different unfinished pipeline'); } catch (\RuntimeException $e) { - $this->assertStringContainsString('Another command is already in progress: pull-db', $e->getMessage()); - $this->assertStringContainsString('Rerun pull-db to resume it', $e->getMessage()); - $this->assertStringContainsString('Only use --abort if you want to discard', $e->getMessage()); + $this->assertStringContainsString('pull-db owns unfinished import state', $e->getMessage()); + $this->assertStringContainsString('`reprint pull-db`', $e->getMessage()); + $this->assertStringContainsString('`reprint pull-db --abort`', $e->getMessage()); } finally { ob_end_clean(); } @@ -596,7 +689,7 @@ public function testPullAfterFilesPullCompletedBeforePipelineStageWasMarkedDoesN ]); $this->fail('Expected pull to reject an in-progress pull-files pipeline'); } catch (\RuntimeException $e) { - $this->assertStringContainsString('Another command is already in progress: pull-files', $e->getMessage()); + $this->assertStringContainsString('pull-files owns unfinished import state', $e->getMessage()); } finally { ob_end_clean(); } @@ -687,7 +780,7 @@ public function testPullDbRejectsConflictingInProgressPullFiles(): void ]); $this->fail('Expected pull-db to reject an in-progress pull-files command'); } catch (\RuntimeException $e) { - $this->assertStringContainsString('Another command is already in progress: pull-files', $e->getMessage()); + $this->assertStringContainsString('pull-files owns unfinished import state', $e->getMessage()); } finally { ob_end_clean(); } @@ -732,6 +825,14 @@ public function testPullDbResumesAfterDbPullCompletedBeforePipelineStageWasMarke public function testPullDbAfterStandaloneDbPullDownloadsFreshDump(): void { file_put_contents($this->stateDir . '/db.sql', "SELECT stale;\n"); + foreach ([ + 'db-tables.jsonl', + 'pull/domains.json', + 'pull/sql-buffer', + 'pull/sql-stats.json', + ] as $artifact) { + file_put_contents($this->stateDir . '/' . $artifact, "stale\n"); + } $this->writeState([ "active_resumable_command" => [ "command_name" => "db-pull", @@ -739,6 +840,20 @@ public function testPullDbAfterStandaloneDbPullDownloadsFreshDump(): void "current_stage" => null, ], "preflight" => ["http_code" => 200, "data" => ["ok" => true]], + "db_index" => [ + "file" => $this->stateDir . '/db-tables.jsonl', + "tables" => 1, + "rows_estimated" => 10, + "bytes" => 100, + "updated_at" => "1234567890", + ], + "sql_bytes" => 100, + "sql_statements_counted" => 5, + "sql_output" => "mysql", + "mysql_host" => "database.example", + "mysql_port" => 3307, + "mysql_user" => "database_user", + "mysql_database" => "database_name", ]); $client = $this->makeClient(false); @@ -756,6 +871,18 @@ public function testPullDbAfterStandaloneDbPullDownloadsFreshDump(): void $this->assertSame("SELECT 1;\n", file_get_contents($this->stateDir . '/db.sql')); $this->assertSame('pull-db', $state["pull_pipeline"]["started_by_command"]); $this->assertSame('db-apply', $state["pull_pipeline"]["last_completed_stage"]); + $this->assertSame(( new \DatabaseTableIndexState() )->to_array(), $state["db_index"]); + $this->assertNull($state["sql_bytes"]); + $this->assertSame(0, $state["sql_statements_counted"]); + $this->assertSame('file', $state["sql_output"]); + $this->assertNull($state["mysql_host"]); + $this->assertNull($state["mysql_port"]); + $this->assertNull($state["mysql_user"]); + $this->assertNull($state["mysql_database"]); + $this->assertFileDoesNotExist($this->stateDir . '/db-tables.jsonl'); + $this->assertFileDoesNotExist($this->stateDir . '/pull/domains.json'); + $this->assertFileDoesNotExist($this->stateDir . '/pull/sql-buffer'); + $this->assertFileDoesNotExist($this->stateDir . '/pull/sql-stats.json'); } public function testInvalidPullDbOptionsFailBeforeStateIsPersisted(): void @@ -878,12 +1005,10 @@ public function testRepullAfterSkippedEarlierTailUsesCompletedFilesPullState(): $this->assertSame('essential-files', $state["filter"]); } - public function testRepullAfterInterruptedSkippedEarlierTailIsNotBlocked(): void + public function testCompletedPullCannotStealInterruptedSkippedEarlierTail(): void { - // An interrupted skipped-earlier tail leaves completion_state="partial" - // with filter=skipped-earlier. A new essential-files pull must still be - // allowed: the filter-change guard must not treat the terminal tail as - // a mid-flight sync. + // The completed pipeline is historical. Once a standalone + // skipped-earlier tail starts, files-pull owns that unfinished work. $this->writeState([ "active_resumable_command" => [ "command_name" => "files-pull", @@ -902,16 +1027,30 @@ public function testRepullAfterInterruptedSkippedEarlierTailIsNotBlocked(): void ]); $client = $this->makeClient(false); + $rawState = file_get_contents($this->stateDir . '/pull/state.json'); - ob_start(); - $client->run([ - "command" => "pull", - "filter" => "essential-files", - "runtime" => "none", - ]); - ob_end_clean(); + $caught = null; + try { + ob_start(); + $client->run([ + "command" => "pull", + "filter" => "essential-files", + "runtime" => "none", + ]); + } catch (\RuntimeException $error) { + $caught = $error; + } finally { + ob_end_clean(); + } - $state = $this->readState(); - $this->assertSame('essential-files', $state["filter"]); + $this->assertInstanceOf(\RuntimeException::class, $caught); + $this->assertStringContainsString( + 'files-pull owns unfinished import state', + $caught->getMessage(), + ); + $this->assertSame( + $rawState, + file_get_contents($this->stateDir . '/pull/state.json'), + ); } } diff --git a/tests/Import/PullStateTest.php b/tests/Import/PullStateTest.php index 3dbbfefa..3384abca 100644 --- a/tests/Import/PullStateTest.php +++ b/tests/Import/PullStateTest.php @@ -13,6 +13,7 @@ public function testStateHydratesDocumentedNestedObjects(): void $data = (new \PullState())->to_array(); $data['active_resumable_command'] = [ 'command_name' => 'files-pull', + 'started_by_command' => 'pull-files', 'completion_state' => 'partial', 'current_stage' => 'fetch', 'remote_cursor' => 'cursor-1', @@ -31,6 +32,7 @@ public function testStateHydratesDocumentedNestedObjects(): void $state = \PullState::from_array($data); $this->assertSame('files-pull', $state->active_resumable_command->command_name); + $this->assertSame('pull-files', $state->active_resumable_command->started_by_command); $this->assertSame('partial', $state->active_resumable_command->completion_state); $this->assertSame('preflight', $state->pull_pipeline->last_completed_stage); $this->assertSame(['preflight', 'files-pull'], $state->pull_pipeline->stage_sequence); @@ -41,6 +43,7 @@ public function testStateRoundTripsToPersistedArraySchema(): void { $state = new \PullState(); $state->active_resumable_command->command_name = 'db-pull'; + $state->active_resumable_command->started_by_command = 'pull-db'; $state->active_resumable_command->completion_state = 'complete'; $state->pull_pipeline->started_by_command = 'pull'; $state->diff->next_remote_index_byte_offset = 123; @@ -50,6 +53,7 @@ public function testStateRoundTripsToPersistedArraySchema(): void $array = $state->to_array(); $this->assertSame('db-pull', $array['active_resumable_command']['command_name']); + $this->assertSame('pull-db', $array['active_resumable_command']['started_by_command']); $this->assertSame('complete', $array['active_resumable_command']['completion_state']); $this->assertSame('pull', $array['pull_pipeline']['started_by_command']); $this->assertSame(123, $array['diff']['next_remote_index_byte_offset']); @@ -65,6 +69,16 @@ public function testStateObjectsDoNotExposeArrayOffsetMutation(): void $this->assertNotInstanceOf(\ArrayAccess::class, $state->active_resumable_command); } + public function testStateWithoutCheckpointProvenanceLoadsAsUnknown(): void + { + $data = ( new \PullState() )->to_array(); + unset($data['active_resumable_command']['started_by_command']); + + $state = \PullState::from_array($data); + + $this->assertNull($state->active_resumable_command->started_by_command); + } + public function testStateRejectsAnIncompleteSchema(): void { $data = (new \PullState())->to_array(); diff --git a/tests/e2e/tests/import-36-mysql-mode-crash-recovery.test.js b/tests/e2e/tests/import-36-mysql-mode-crash-recovery.test.js index 029580d8..39edf866 100644 --- a/tests/e2e/tests/import-36-mysql-mode-crash-recovery.test.js +++ b/tests/e2e/tests/import-36-mysql-mode-crash-recovery.test.js @@ -4,8 +4,8 @@ * When using --sql-output=mysql with short execution times, the server * may pause mid-query (x-query-complete: 0). The importer buffers the * partial SQL in memory and persists it to pull/sql-buffer on disk as each - * chunk arrives. If the process dies at any point, the next run reloads - * whatever was accumulated. + * chunk arrives. A resumed SQL request reloads its persisted buffer. A fresh + * download removes a stale buffer. * * This test forces many resume cycles with --max-exec=1, verifies the * database is correct after completion, and confirms pull/sql-buffer is @@ -90,7 +90,7 @@ describe('Import: MySQL Mode Crash Recovery', { timeout: 120000 }, () => { }); }); - describe('pre-seeded pull/sql-buffer is loaded on resume', () => { + describe('pre-seeded pull/sql-buffer is removed on a fresh run', () => { let tempDir; const importDb = 'e2e_basic_import_36_seeded'; @@ -109,19 +109,17 @@ describe('Import: MySQL Mode Crash Recovery', { timeout: 120000 }, () => { await conn.end(); }); - it('loads pull/sql-buffer from disk and logs recovery', { timeout: 300000 }, () => { + it('does not execute SQL from a stale pull/sql-buffer', { timeout: 300000 }, () => { // Run preflight so db-pull can proceed runImporter(importUrl(), tempDir, 'preflight', { secret: getSiteSecret(site), }); - // Seed a pull/sql-buffer file before running db-pull. - // The content is a harmless SQL comment that won't affect execution - // — the point is to verify the importer reads it and logs recovery. + // Seed invalid SQL before a fresh db-pull. Restoring this buffer + // would make the first query fail. const bufferFile = join(tempDir, 'pull/sql-buffer'); - writeFileSync(bufferFile, '-- pre-seeded buffer\n'); + writeFileSync(bufferFile, 'THIS IS NOT SQL;\n'); - // Run a fresh db-pull — the importer should detect the buffer file const result = runImporter(importUrl(), tempDir, 'db-pull', { secret: getSiteSecret(site), extraArgs: mysqlArgs(importDb), @@ -130,10 +128,9 @@ describe('Import: MySQL Mode Crash Recovery', { timeout: 120000 }, () => { assert.equal(result.exitCode, 0, `Expected exit 0, got ${result.exitCode}\nstderr: ${result.stderr}`); - // Verify recovery was logged const audit = readAuditLog(tempDir); - assert.ok(audit.includes('CRASH RECOVERY') && audit.includes('pull/sql-buffer'), - 'Expected audit log to mention pull/sql-buffer crash recovery'); + assert.ok(!audit.includes('from pull/sql-buffer'), + 'Expected a fresh db-pull not to restore pull/sql-buffer'); // Buffer should be cleaned up assert.ok(!existsSync(bufferFile),