diff --git a/packages/reprint-importer/src/import.php b/packages/reprint-importer/src/import.php index f0935e90..aab093d1 100755 --- a/packages/reprint-importer/src/import.php +++ b/packages/reprint-importer/src/import.php @@ -11,7 +11,9 @@ */ use function WordPress\Filesystem\wp_join_unix_paths; +use Reprint\Importer\Filesystem\PulledFilesystem; use Reprint\Importer\Remote\RemoteExportApiClient; +use Reprint\Importer\Remote\StreamingContext; use Reprint\Importer\Remote\TransientInterruptionException; use Reprint\Importer\Tuning\AdaptiveTuner; use function WordPress\Reprint\Exporter\assert_valid_path; @@ -69,6 +71,14 @@ // Typed state objects for the persisted pull state. require_once __DIR__ . '/lib/state/class-pull-state.php'; +// Remote-to-local filesystem projection and safe local mutations. +require_once __DIR__ . '/lib/filesystem/class-pulled-file-context.php'; +require_once __DIR__ . '/lib/filesystem/class-preserve-local-skip-exception.php'; +require_once __DIR__ . '/lib/filesystem/class-pulled-filesystem.php'; + +// Context retained while a remote streaming response is processed. +require_once __DIR__ . '/lib/remote/class-streaming-context.php'; + // Adaptive sizing for push request bodies require_once __DIR__ . '/lib/upload/class-push-request-sizer.php'; require_once __DIR__ . '/lib/upload/class-multipart-push-stream-client.php'; @@ -406,6 +416,9 @@ class ImportClient /** @var Pull Orchestrates high-level pull pipelines. */ private Pull $pull; + /** @var PulledFilesystem Projects remote paths and entries onto the local filesystem. */ + private PulledFilesystem $pulled_filesystem; + /** @var int Cumulative count of index entries written (survives retries). */ private $next_remote_index_entries_counted = 0; @@ -642,6 +655,14 @@ public function prepare_files_pull_options(array $options, bool $assert_remap = if ($assert_remap) { $this->assert_resolved_path_mappings_consistent(); } + + $this->pulled_filesystem = new PulledFilesystem( + $this->filesystem_root, + $this->resolved_path_mappings, + $this->local_followed_symlinks_root, + $this->fs_root_nonempty_behavior, + $this->get_export_directories(), + ); } /** True when the skipped-fetch list exists and still has entries. */ @@ -795,6 +816,14 @@ private function emit_skip_progress(string $path): void ], true); } + /** Write filesystem-operation messages produced by PulledFilesystem. */ + private function audit_pulled_filesystem_operation_messages(): void + { + foreach ($this->pulled_filesystem->drain_operation_messages() as $message) { + $this->audit_log($message, true); + } + } + /** * Runs one Reprint command while holding the state directory's process lock. * @@ -2630,6 +2659,12 @@ private function finalize_tuned_request( */ public function run_files_pull(): void { + if (!isset($this->pulled_filesystem)) { + throw new LogicException( + "File pull options must be prepared before running files-pull.", + ); + } + $state_command = $this->get_state()->active_resumable_command->command_name ?? null; // A full `pull` leaves active_resumable_command on its last stage @@ -3479,7 +3514,7 @@ private function recreate_intermediate_symlinks(): void } try { - $local_absolute_path = $this->map_remote_absolute_path_to_local_absolute_path( + $local_absolute_path = $this->pulled_filesystem->map_remote_absolute_path( $remote_absolute_path ); } catch (RuntimeException $e) { @@ -3508,8 +3543,10 @@ private function recreate_intermediate_symlinks(): void $parent = dirname($local_absolute_path); if (!is_dir($parent)) { try { - $this->create_directory_if_missing($parent); + $this->pulled_filesystem->create_directory($parent); + $this->audit_pulled_filesystem_operation_messages(); } catch (RuntimeException $e) { + $this->audit_pulled_filesystem_operation_messages(); $this->audit_log( "INTERMEDIATE SYMLINK SKIP: failed to prepare parent for {$remote_absolute_path}: " . $e->getMessage(), @@ -3536,12 +3573,10 @@ private function recreate_intermediate_symlinks(): void } // Validate that the symlink target doesn't escape the filesystem root. - $root = $this->get_filesystem_root_path(); try { - $this->assert_symlink_target_within_root( + $this->pulled_filesystem->assert_symlink_target_within_root( dirname($local_absolute_path), $symlink_target, - $root ); } catch (RuntimeException $e) { $this->audit_log( @@ -6635,7 +6670,7 @@ private function compare_remote_indexes_and_build_fetch_list(): bool strcmp($remote_index_entry["path"], $next_remote_index_entry["path"]) > 0 ) { $preserve_local_skip_reason = - $this->should_skip_for_preserve_local( + $this->pulled_filesystem->preserve_local_skip_reason( $next_remote_index_entry["path"], ); if ($preserve_local_skip_reason) { @@ -7076,7 +7111,7 @@ private function apply_remote_deletion_locally(string $remote_absolute_path): vo return; } try { - $local_absolute_path = $this->map_remote_absolute_path_to_local_absolute_path( + $local_absolute_path = $this->pulled_filesystem->map_remote_absolute_path( $remote_absolute_path ); } catch (RuntimeException $e) { @@ -7090,7 +7125,7 @@ private function apply_remote_deletion_locally(string $remote_absolute_path): vo return; } - if ($this->remove_local_absolute_path_without_following_symlinks($local_absolute_path)) { + if ($this->pulled_filesystem->remove_path($local_absolute_path)) { $this->audit_log("Deleted: {$remote_absolute_path}", false); return; } @@ -7098,46 +7133,6 @@ private function apply_remote_deletion_locally(string $remote_absolute_path): vo $this->audit_log("Failed to delete: {$remote_absolute_path}", true); } - /** - * Remove a local absolute path recursively without traversing symlink targets. - * - * Symlinks are always unlinked as links. Directories are traversed - * depth-first. - */ - private function remove_local_absolute_path_without_following_symlinks( - string $local_absolute_path - ): bool { - if (!file_exists($local_absolute_path) && !is_link($local_absolute_path)) { - return true; - } - - if (is_link($local_absolute_path) || is_file($local_absolute_path)) { - return true === @unlink($local_absolute_path); - } - - if (is_dir($local_absolute_path)) { - $entries = @scandir($local_absolute_path); - if ($entries === false) { - return false; - } - foreach ($entries as $entry) { - if ($entry === "." || $entry === "..") { - continue; - } - if ( - !$this->remove_local_absolute_path_without_following_symlinks( - $local_absolute_path . "/" . $entry - ) - ) { - return false; - } - } - return true === @rmdir($local_absolute_path); - } - - return true === @unlink($local_absolute_path); - } - /** * Parse one JSON index line into an array. */ @@ -8432,37 +8427,6 @@ private function fetch_database_index(): void } - /** - * Assert that a symlink target resolves to a path within $root. - * - * For absolute targets, the target itself must be under $root. - * For relative targets, the resolved path (parent dir + target) must be - * under $root. We normalize ".." segments without touching the filesystem, - * since the target may not exist yet. - * - * @throws RuntimeException if the target escapes the root. - */ - private function assert_symlink_target_within_root( - string $symlink_parent_dir, - string $target, - string $root - ): void { - if (str_starts_with($target, "/")) { - // Absolute target: must be under root - $resolved = normalize_path($target); - } else { - // Relative target: resolve against the symlink's parent directory - $resolved = normalize_path($symlink_parent_dir . "/" . $target); - } - - if (!path_is_within_root($resolved, $root)) { - throw new RuntimeException( - "Security: symlink target escapes filesystem root: {$target} " . - "(resolves to {$resolved}, root is {$root})" - ); - } - } - /** * Rewrite a remote symlink target for the local filesystem when possible. * @@ -8519,7 +8483,7 @@ private function rewrite_symlink_target_for_local_filesystem( // Repoint to where the target's content is placed by the same pull // mapping used for file chunks, so the symlink does not dangle. - $local_absolute_target = $this->map_remote_absolute_path_to_local_absolute_path( + $local_absolute_target = $this->pulled_filesystem->map_remote_absolute_path( $remote_absolute_target ); $local_relative_target = self::compute_relative_path( @@ -9013,63 +8977,6 @@ private function resolve_token_path(string $raw, array $tokens): string return $resolved; } - /** - * Map a remote absolute path to a local absolute path under the filesystem - * root. Symlink traversal checks prevent writes outside the filesystem root. - * - * With --remap active, a matched remote absolute path is routed to its - * mapped local absolute path. An unmatched path remains nested beneath - * --fs-root, as in an identity pull mapping. - */ - private function map_remote_absolute_path_to_local_absolute_path( - string $remote_absolute_path - ): string { - assert_valid_path($remote_absolute_path, "remote absolute path"); - $local_absolute_path = null; - $longest_remote_prefix_length = -1; - foreach ($this->resolved_path_mappings as $remote_prefix => $local_prefix) { - $remainder = self::path_remainder_under($remote_absolute_path, $remote_prefix); - if ($remainder !== null && strlen($remote_prefix) > $longest_remote_prefix_length) { - $local_absolute_path = wp_join_unix_paths($local_prefix, $remainder); - $longest_remote_prefix_length = strlen($remote_prefix); - } - } - if ($local_absolute_path !== null) { - return $local_absolute_path; - } - - // Following symlinks is currently the only way paths outside the original export scope reach this mapper. - // Use the same local followed symlinks root for copied content and rewritten symlink targets so the links do not dangle. - if ($this->local_followed_symlinks_root !== null - && !$this->path_is_within_original_export_scope($remote_absolute_path)) { - return $this->local_followed_symlinks_root . $remote_absolute_path; - } - - return $this->get_filesystem_root_path() . $remote_absolute_path; - } - - - /** - * Returns the remainder of $path underneath $prefix, - * empty string if $path === $prefix, - * or null if $path is not under $prefix. - */ - private static function path_remainder_under(string $path, string $prefix): ?string - { - $path = rtrim($path, "/"); - $prefix = rtrim($prefix, "/"); - - if ($path === $prefix) { - return ""; - } - - if (str_starts_with($path, $prefix . "/")) { - return substr($path, strlen($prefix)); - } - - return null; - } - /** * Handle a metadata chunk from multipart response. */ @@ -9083,65 +8990,33 @@ private function handle_metadata_chunk(array $chunk): void { } /** - * Handle a file chunk from multipart response. + * Handle a file chunk from a multipart response. */ private function handle_file_chunk( array $chunk, StreamingContext $context ): void { - $headers = $chunk["headers"]; - $raw_header = $headers["x-file-path"] ?? ""; - $path = base64_decode($raw_header, true); - $is_first = ($headers["x-first-chunk"] ?? "0") === "1"; - $is_last = ($headers["x-last-chunk"] ?? "0") === "1"; - - if ($path === false || $path === "") { - if ($raw_header !== "") { - $this->audit_log( - "Warning: base64_decode failed for x-file-path header: " . - substr($raw_header, 0, 100), - true, - ); - } + $result = $this->pulled_filesystem->write_file_chunk($chunk, $context); + if ($result["warning"] !== null) { + $this->audit_log($result["warning"], true); return; } - $local_absolute_path = $this->map_remote_absolute_path_to_local_absolute_path($path); - - // Open file on first chunk - if ($is_first) { - // Reset skip flag for each new file - $context->skip_current_file = false; - - if ( - (file_exists($local_absolute_path) || is_link($local_absolute_path)) && - (!is_file($local_absolute_path) || is_link($local_absolute_path)) - ) { - if ( - !$this->remove_local_absolute_path_without_following_symlinks( - $local_absolute_path - ) - ) { - throw new RuntimeException( - "Failed to replace path with file: {$path}", - ); - } - } - - // Check if file exists locally - $exists_locally = file_exists($local_absolute_path); - $local_size = $exists_locally ? filesize($local_absolute_path) : 0; - $file_size = (int) ($headers["x-file-size"] ?? 0); + $path = $result["remote_absolute_path"]; + if ($path === null) { + return; + } - // Log file pull with useful context + if ($result["started"] !== null) { + $started = $result["started"]; $this->audit_log( sprintf( "File: %s (remote_size=%d, ctime=%d, local_exists=%s, local_size=%d)", $path, - $file_size, - (int) ($headers["x-file-ctime"] ?? 0), - $exists_locally ? "yes" : "no", - $local_size, + $started["remote_size"], + $started["ctime"], + $started["local_exists"] ? "yes" : "no", + $started["local_size"], ), false, ); @@ -9159,7 +9034,7 @@ private function handle_file_chunk( "type" => "file_progress", "files_done" => $files_done, "path" => $path, - "size" => $file_size, + "size" => $started["remote_size"], "message" => $file_progress_message, ]; if ($this->fetch_list_total !== null) { @@ -9168,117 +9043,44 @@ private function handle_file_chunk( $this->output_progress($progress_record); } - // Skip body/close for files being preserved - if ($context->skip_current_file) { + $this->audit_pulled_filesystem_operation_messages(); + + if ($result["skip_message"] !== null) { + $this->audit_log($result["skip_message"], true); + $this->emit_skip_progress($path); return; } - // Open file handle on first chunk - if ($is_first) { - // Close previous file if any - if ($context->file_handle) { - fclose($context->file_handle); - if ($context->file_ctime && $context->file_path) { - touch($context->file_path, $context->file_ctime); - } - } - - // Create parent directory if needed - $dir = dirname($local_absolute_path); - if (!is_dir($dir)) { - // Check if any component of the path exists as a file and remove it - try { - $this->create_directory_if_missing($dir); - } catch (PreserveLocalSkipException $e) { - $context->skip_current_file = true; - $this->audit_log($e->getMessage(), true); - $this->emit_skip_progress($path); - return; - } - } - - // Open new file - $context->file_handle = fopen($local_absolute_path, "wb"); - if (!$context->file_handle) { - $error = error_get_last(); - throw new RuntimeException( - "Failed to open file for writing: {$local_absolute_path}\n" . - "Parent directory: {$dir}\n" . - "Directory exists: " . - (is_dir($dir) ? "yes" : "no") . - "\n" . - "Error: " . - ($error["message"] ?? "unknown"), - ); - } - $context->file_path = $local_absolute_path; - $context->file_ctime = (int) ($headers["x-file-ctime"] ?? 0); - $context->file_bytes_written = 0; // Reset byte counter for new file + if (!$result["completed"]) { + return; } - // Write body data if present - if (isset($chunk["body"]) && $chunk["body"] !== "") { - if ($context->file_handle) { - $data = $chunk["body"]; - $bytes = fwrite($context->file_handle, $data); - if ($bytes === false || $bytes !== strlen($data)) { - throw new RuntimeException( - "Write failed for {$context->file_path}: wrote " . - ($bytes === false ? "0" : $bytes) . "/" . strlen($data) . - " bytes (disk full?)" - ); - } - $context->file_bytes_written += $bytes; - } + if ($result["index_entry"] !== null) { + $index_entry = $result["index_entry"]; + $this->upsert_remote_index_entry( + $index_entry["path"], + $index_entry["ctime"], + $index_entry["size"], + $index_entry["type"], + ); + $this->files_pulled++; + $this->clear_volatile_file($path); + $this->audit_log( + sprintf(" Indexed (wrote %d bytes)", $result["final_size"]), + false, + ); + } elseif ($result["file_changed"]) { + $this->audit_log( + " File changed during stream; index not updated", + true, + ); } - // Close on last chunk - if ($is_last && $context->file_handle) { - fclose($context->file_handle); - - // Set file modification time - if ($context->file_ctime && $context->file_path) { - touch($context->file_path, $context->file_ctime); - } - - // Index update (JSON lines) - $file_size = (int) ($headers["x-file-size"] ?? 0); - $final_size = file_exists($context->file_path) - ? filesize($context->file_path) - : 0; - - $file_changed = ($headers["x-file-changed"] ?? "0") === "1"; - - if ($context->file_ctime && !$file_changed) { - $this->upsert_remote_index_entry( - $path, - $context->file_ctime, - $file_size, - "file", - ); - $this->files_pulled++; // Count completed files only - $this->clear_volatile_file($path); - $this->audit_log( - sprintf(" Indexed (wrote %d bytes)", $final_size), - false, - ); - } elseif ($file_changed) { - $this->audit_log( - " File changed during stream; index not updated", - true, - ); - } - - $context->file_handle = null; - $context->file_path = null; - $context->file_ctime = null; - $context->file_bytes_written = 0; - // Clear crash recovery tracking - file is complete - $this->get_state()->current_file = null; - $this->get_state()->current_file_bytes = null; - } + $this->get_state()->current_file = null; + $this->get_state()->current_file_bytes = null; } + /** * Build a short display path for progress messages: strip leading slash, * truncate from the left when too long. @@ -9293,434 +9095,102 @@ private function display_path(string $path): string return $rel; } - /** - * Check whether any component of the path (between the filesystem root - * and the remote absolute path) is a symlink. In preserve-local mode this is used - * to prevent creating new content through symlinked directories — their - * contents belong to shared hosting infrastructure and must not be - * modified. - */ - private function should_skip_for_preserve_local(string $remote_absolute_path): ?string - { - if ($this->fs_root_nonempty_behavior !== 'preserve-local') { - return null; - } - - $local_absolute_path = $this->map_remote_absolute_path_to_local_absolute_path( - $remote_absolute_path - ); - - // Skip if anything already exists at this path — regular file, symlink - // (even to a file), or directory. This preserves hosting symlinks like - // wp-load.php -> __wp__/wp-load.php and drop-in symlinks like - // object-cache.php -> ../../wordpress/drop-ins/... - if (file_exists($local_absolute_path) || is_link($local_absolute_path)) { - return "PRESERVE-LOCAL skip file (exists): {$remote_absolute_path}"; - } - - // Skip if parent directory is not writable or if any directory component - // in the path is a symlink. We never create new files through symlinks — - // the symlink and its target contents are shared hosting infrastructure. - $dir = dirname($local_absolute_path); - if (is_dir($dir) && !is_writable($dir)) { - return "PRESERVE-LOCAL skip file (dir not writable): {$remote_absolute_path}"; - } - if ($this->path_traverses_symlink($dir)) { - return "PRESERVE-LOCAL skip file (symlink in path): {$remote_absolute_path}"; - } - - return null; - } - - private function path_traverses_symlink(string $path): bool - { - $root = $this->get_filesystem_root_path(); - $relative = ltrim(substr($path, strlen($root)), "/"); - if ($relative === "") { - return false; - } - - $current = $root; - foreach (explode("/", $relative) as $part) { - if ($part === "") { - continue; - } - $current .= "/" . $part; - if (is_link($current)) { - return true; - } - if (!file_exists($current)) { - break; - } - } - return false; - } - - /** - * Create a directory path when missing, removing blockers. - * - * @param string $dir Directory path to create - * @throws RuntimeException if directory cannot be created or is outside allowed path - */ - private function create_directory_if_missing(string $dir): void - { - // Security: Ensure path is under the filesystem root - $real_filesystem_root = $this->get_filesystem_root_path(); - - // Resolve the target path (or what it would be) - // For non-existent paths, resolve the parent and append the final component - $check_path = $dir; - while ( - !file_exists($check_path) && - $check_path !== dirname($check_path) - ) { - $check_path = dirname($check_path); - } - - if (file_exists($check_path)) { - $real_check = realpath($check_path); - if ( - $real_check === false || - !path_is_within_root($real_check, $real_filesystem_root) - ) { - // In preserve-local mode, a path that resolves outside the - // filesystem root is expected when a directory like wp-content/plugins - // is symlinked to a shared hosting location. Skip gracefully - // instead of treating it as a security violation. - if ($this->fs_root_nonempty_behavior === 'preserve-local') { - throw new PreserveLocalSkipException( - "PRESERVE-LOCAL: path resolves outside filesystem root via symlink: {$dir}", - ); - } - throw new RuntimeException( - "Security: Refusing to create directory outside filesystem root: {$dir}", - ); - } - } - - if (is_dir($dir) && !is_link($dir)) { - if ($this->fs_root_nonempty_behavior === 'preserve-local' && !is_writable($dir)) { - throw new PreserveLocalSkipException( - "PRESERVE-LOCAL: directory not writable: {$dir}", - ); - } - return; - } - - if ( - $dir !== $real_filesystem_root && - !str_starts_with($dir, $real_filesystem_root . "/") - ) { - throw new RuntimeException( - "Security: Refusing to create directory outside filesystem root: {$dir}", - ); - } - - $relative = ltrim(substr($dir, strlen($real_filesystem_root)), "/"); - if ($relative === "") { - return; - } - - $current = $real_filesystem_root; - foreach (explode("/", $relative) as $part) { - if ($part === "") { - continue; - } - $current .= "/" . $part; - - if (is_link($current)) { - if ($this->fs_root_nonempty_behavior === 'preserve-local') { - // Never create directories through symlinks — the symlink - // and its target contents are shared hosting infrastructure - // that must not be modified. - throw new PreserveLocalSkipException( - "PRESERVE-LOCAL: symlink in directory path: {$current}", - ); - } - $this->audit_log( - "Removing symlink blocking directory: {$current}", - true, - ); - if (!unlink($current)) { - throw new RuntimeException( - "Failed to remove symlink blocking directory: {$current}", - ); - } - // Clear cached realpath so the subsequent realpath() check - // sees the new directory instead of the removed symlink. - clearstatcache(true, $current); - } - - // Remove file if blocking directory creation - if (is_file($current)) { - if ($this->fs_root_nonempty_behavior === 'preserve-local') { - throw new PreserveLocalSkipException( - "PRESERVE-LOCAL: file blocks directory creation: {$current}", - ); - } - $this->audit_log( - "Removing file blocking directory: {$current}", - true, - ); - if (!unlink($current)) { - throw new RuntimeException( - "Failed to remove file blocking directory: {$current}", - ); - } - } - - // Create directory if it doesn't exist - if (is_dir($current)) { - if ($this->fs_root_nonempty_behavior === 'preserve-local' && !is_writable($current)) { - throw new PreserveLocalSkipException( - "PRESERVE-LOCAL: directory not writable: {$current}", - ); - } - } elseif (!mkdir($current, 0755) && !is_dir($current)) { - throw new RuntimeException( - "Failed to create directory: {$current}\n" . - "Error: " . - (error_get_last()["message"] ?? "unknown"), - ); - } - - $resolved = realpath($current); - if ($resolved === false || !path_is_within_root($resolved, $real_filesystem_root)) { - throw new RuntimeException( - "Security: Refusing to create directory outside filesystem root: {$current}", - ); - } - } - } - - /** - * Handle a directory chunk (create empty directory). - */ + /** Handle a directory chunk from a multipart response. */ private function handle_directory_chunk(array $chunk): void { - $headers = $chunk["headers"]; - $raw_header = $headers["x-directory-path"] ?? ""; - $remote_absolute_path = base64_decode($raw_header, true); - $ctime = (int) ($headers["x-directory-ctime"] ?? 0); - - if ($remote_absolute_path === false || $remote_absolute_path === "") { - if ($raw_header !== "") { - $this->audit_log( - "Warning: base64_decode failed for x-directory-path header: " . - substr($raw_header, 0, 100), - true, - ); - } + $result = $this->pulled_filesystem->create_remote_directory($chunk); + $this->audit_pulled_filesystem_operation_messages(); + if ($result["warning"] !== null) { + $this->audit_log($result["warning"], true); return; } - $local_absolute_path = $this->map_remote_absolute_path_to_local_absolute_path( - $remote_absolute_path - ); - - // In preserve-local mode, if the directory already exists (as a real - // directory or via a symlink to a directory), keep it as-is. - // Also skip if any parent component is a symlink — we never create - // new directories through symlinked paths. - if ($this->fs_root_nonempty_behavior === 'preserve-local') { - if (is_dir($local_absolute_path)) { - $this->audit_log("PRESERVE-LOCAL skip directory (exists): {$remote_absolute_path}", true); - $this->emit_skip_progress($remote_absolute_path); - if ($ctime > 0) { - $this->upsert_remote_index_entry($remote_absolute_path, $ctime, 0, "dir"); - } - return; - } - if ($this->path_traverses_symlink($local_absolute_path)) { - $this->audit_log("PRESERVE-LOCAL skip directory (symlink in path): {$remote_absolute_path}", true); - $this->emit_skip_progress($remote_absolute_path); - if ($ctime > 0) { - $this->upsert_remote_index_entry($remote_absolute_path, $ctime, 0, "dir"); - } - return; - } - } - - if ( - (file_exists($local_absolute_path) || is_link($local_absolute_path)) && - (!is_dir($local_absolute_path) || is_link($local_absolute_path)) - ) { - if ( - !$this->remove_local_absolute_path_without_following_symlinks($local_absolute_path) - ) { - throw new RuntimeException( - "Failed to replace path with directory: {$remote_absolute_path}", - ); - } - } - - // Create directory, removing any files that block the path - try { - $this->create_directory_if_missing($local_absolute_path); - } catch (PreserveLocalSkipException $e) { - $this->audit_log($e->getMessage(), true); - $this->emit_skip_progress($remote_absolute_path); + $path = $result["remote_absolute_path"]; + if ($path === null) { return; } - $this->audit_log("Directory: {$remote_absolute_path}", false); + if ($result["skip_message"] !== null) { + $this->audit_log($result["skip_message"], true); + $this->emit_skip_progress($path); + } else { + $this->audit_log("Directory: {$path}", false); + } - if ($ctime > 0) { - $this->upsert_remote_index_entry($remote_absolute_path, $ctime, 0, "dir"); + if ($result["ctime"] > 0 && ( + $result["skip_message"] === null + || str_starts_with($result["skip_message"], "PRESERVE-LOCAL skip directory (") + )) { + $this->upsert_remote_index_entry($path, $result["ctime"], 0, "dir"); } } - /** - * Recreates a symlink from the export stream in the local filesystem. - * - * Decodes the base64-encoded path and target from the chunk headers, - * validates that the target stays within the filesystem root (preventing - * directory traversal), then creates the symlink. Failures are logged - * to the audit log and reported as symlink_error progress events — they - * do not halt the pull. - * - * @param array $chunk Multipart chunk with x-symlink-path, x-symlink-target, - * and x-symlink-ctime headers (all base64-encoded). - */ + + /** Handle a symlink chunk from a multipart response. */ private function handle_symlink_chunk(array $chunk): void { $headers = $chunk["headers"]; - $raw_path = $headers["x-symlink-path"] ?? ""; - $path = base64_decode($raw_path, true); + $path = base64_decode($headers["x-symlink-path"] ?? "", true); $target = base64_decode($headers["x-symlink-target"] ?? "", true); - $ctime = (int) ($headers["x-symlink-ctime"] ?? 0); - - // Skip if path or target is missing/empty if ($path === false || $path === "" || $target === false || $target === "") { - if ($raw_path !== "" && ($path === false || $path === "")) { - $this->audit_log( - "Warning: base64_decode failed for x-symlink-path header: " . - substr($raw_path, 0, 100), - true, - ); + $result = $this->pulled_filesystem->create_remote_symlink($chunk, ""); + if ($result["warning"] !== null) { + $this->audit_log($result["warning"], true); } return; } - $local_absolute_path = $this->map_remote_absolute_path_to_local_absolute_path($path); + $local_absolute_path = $this->pulled_filesystem->map_remote_absolute_path($path); $target_for_local = $this->rewrite_symlink_target_for_local_filesystem( $path, $local_absolute_path, $target, ); + $result = $this->pulled_filesystem->create_remote_symlink( + $chunk, + $target_for_local, + ); + $this->audit_pulled_filesystem_operation_messages(); - // In preserve-local mode, if something already exists at the symlink - // path, keep it — whether it's a file, directory, or another symlink. - // Also skip if any parent component is a symlink — we never create - // new content through symlinked directories. - if ($this->fs_root_nonempty_behavior === 'preserve-local') { - if (file_exists($local_absolute_path) || is_link($local_absolute_path)) { - $this->audit_log("PRESERVE-LOCAL skip symlink (path exists): {$path} -> {$target}", true); - $this->emit_skip_progress($path); - return; - } - if ($this->path_traverses_symlink(dirname($local_absolute_path))) { - $this->audit_log("PRESERVE-LOCAL skip symlink (symlink in path): {$path} -> {$target}", true); - $this->emit_skip_progress($path); - return; - } - } - - // Validate that the symlink target doesn't escape the filesystem root. - $root = $this->get_filesystem_root_path(); - try { - $this->assert_symlink_target_within_root( - dirname($local_absolute_path), - $target_for_local, - $root - ); - } catch (RuntimeException $e) { - $this->audit_log($e->getMessage(), true); - $this->output_progress([ - "type" => "symlink_error", - "path" => $path, - "target" => $target_for_local, - "error" => $e->getMessage(), - "message" => "Symlink error: {$path} -> {$target}", - ]); + if ($result["skip_message"] !== null) { + $this->audit_log($result["skip_message"], true); + $this->emit_skip_progress($path); return; } - // Remove existing file/symlink if present - if (file_exists($local_absolute_path) || is_link($local_absolute_path)) { - if ( - !$this->remove_local_absolute_path_without_following_symlinks($local_absolute_path) - ) { + if ($result["error"] !== null) { + if (str_starts_with($result["error"], "Security:")) { + $this->audit_log($result["error"], true); + } elseif ($result["error"] === "Failed to replace existing path") { $this->audit_log( "Failed to remove existing path for symlink: {$local_absolute_path}", true, ); - $this->output_progress([ - "type" => "symlink_error", - "path" => $path, - "target" => $target_for_local, - "error" => "Failed to replace existing path", - "message" => "Symlink error: {$path} -> {$target}", - ]); - return; - } - } - - // Create parent directory - $dir = dirname($local_absolute_path); - if (!is_dir($dir)) { - try { - $this->create_directory_if_missing($dir); - } catch (PreserveLocalSkipException $e) { - $this->audit_log($e->getMessage(), true); - $this->emit_skip_progress($path); - return; - } catch (RuntimeException $e) { - // Log error and skip this symlink + } elseif ($result["error"] === "Failed to create parent directory") { $this->audit_log( - "Failed to create directory for symlink: {$dir}", + "Failed to create directory for symlink: " . dirname($local_absolute_path), + true, + ); + } else { + $this->audit_log( + "Failed to create symlink: {$local_absolute_path} -> {$target_for_local}", true, ); - $this->output_progress([ - "type" => "symlink_error", - "path" => $path, - "target" => $target_for_local, - "error" => "Failed to create parent directory", - "message" => "Symlink error: {$path} -> {$target}", - ]); - return; } - } - - // Create symlink - $symlink_result = symlink($target_for_local, $local_absolute_path); - if (true !== $symlink_result || !is_link($local_absolute_path)) { - // Log error and skip this symlink - $this->audit_log( - "Failed to create symlink: {$local_absolute_path} -> {$target_for_local}", - true, - ); $this->output_progress([ "type" => "symlink_error", "path" => $path, "target" => $target_for_local, - "error" => "Failed to create symlink", + "error" => $result["error"], "message" => "Symlink error: {$path} -> {$target}", ]); return; } - // Try to set the ctime (may not work on all systems) - if ($ctime > 0) { - @touch($local_absolute_path, $ctime); - } - $this->audit_log("Symlink: {$path} -> {$target_for_local}", false); - - if ($ctime > 0) { - $this->upsert_remote_index_entry($path, $ctime, 0, "link"); + if ($result["ctime"] > 0) { + $this->upsert_remote_index_entry($path, $result["ctime"], 0, "link"); } - $this->output_progress([ "type" => "symlink", "path" => $path, @@ -9729,6 +9199,7 @@ private function handle_symlink_chunk(array $chunk): void ]); } + /** * Handle an error chunk from the server. */ @@ -9853,22 +9324,6 @@ private function get_root_directories_from_preflight(): array return $dirs; } - /** - * Whether $path falls under one of the ORIGINAL export directories (the - * --only prefixes, or the base roots without --only) — i.e. it was going to - * be pulled anyway. Evaluated against the pre-follow scope; a followed - * target outside all of these is "escaping" and eligible for symlink bundling. - */ - private function path_is_within_original_export_scope(string $path): bool - { - foreach ($this->get_export_directories() as $root) { - if (path_is_within_root($path, $root)) { - return true; - } - } - return false; - } - /** * Build the list of directories the server should traverse. * @@ -10866,32 +10321,6 @@ public function output_progress(array $data, bool $force = false): void } } -/** - * Context object passed to streaming callbacks. - */ -class StreamingContext -{ - public $on_chunk = null; - public $file_handle = null; - public $file_path = null; - public $file_ctime = null; - // Crash recovery: track bytes written for current file - public $file_bytes_written = 0; - // Last response stats from completion chunk - public $response_stats = []; - // Stream integrity - public $saw_completion = false; - // When true, skip writing the current file (preserve-local mode) - public $skip_current_file = false; -} - -/** - * Thrown by create_directory_if_missing() in preserve-local mode when a directory - * component is not writable or a symlink blocks directory creation. - * Callers catch this to skip the current file/directory/symlink gracefully. - */ -class PreserveLocalSkipException extends RuntimeException {} - // Only run CLI logic when this file is an importer entry point. if ( PHP_SAPI === 'cli' diff --git a/packages/reprint-importer/src/lib/filesystem/class-preserve-local-skip-exception.php b/packages/reprint-importer/src/lib/filesystem/class-preserve-local-skip-exception.php new file mode 100644 index 00000000..fd5d7799 --- /dev/null +++ b/packages/reprint-importer/src/lib/filesystem/class-preserve-local-skip-exception.php @@ -0,0 +1,10 @@ + */ + private $resolved_path_mappings = array(); + + /** @var string|null */ + private $local_followed_symlinks_root; + + /** @var string */ + private $nonempty_behavior = 'error'; + + /** @var list */ + private $original_export_directories = array(); + + /** @var list */ + private $operation_messages = array(); + + /** + * Create the filesystem projection for one immutable pull configuration. + * + * @param string $filesystem_root Local filesystem root. + * @param array $resolved_path_mappings Remote-to-local absolute path mappings. + * @param string|null $local_followed_symlinks_root Local root for followed paths outside the original export scope. + * @param string $nonempty_behavior Either "error" or "preserve-local". + * @param list $original_export_directories Original remote roots selected for pulling. + */ + public function __construct( + string $filesystem_root, + array $resolved_path_mappings, + ?string $local_followed_symlinks_root, + string $nonempty_behavior, + array $original_export_directories + ) { + $this->filesystem_root = rtrim( $filesystem_root, '/' ); + $this->resolved_path_mappings = $resolved_path_mappings; + $this->local_followed_symlinks_root = $local_followed_symlinks_root; + $this->nonempty_behavior = $nonempty_behavior; + $this->original_export_directories = $original_export_directories; + } + + /** Return and clear filesystem-operation messages produced since the prior call. */ + public function drain_operation_messages(): array { + $operation_messages = $this->operation_messages; + $this->operation_messages = array(); + return $operation_messages; + } + + /** Return the resolved absolute filesystem root, creating it when needed. */ + public function get_root_path(): string { + if ( ! is_dir( $this->filesystem_root ) ) { + if ( ! mkdir( $this->filesystem_root, 0755, true ) && ! is_dir( $this->filesystem_root ) ) { + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + throw new RuntimeException( "Failed to create filesystem root directory: {$this->filesystem_root}" ); + } + } + + $real_filesystem_root = realpath( $this->filesystem_root ); + if ( false === $real_filesystem_root ) { + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + throw new RuntimeException( "Failed to resolve filesystem root path: {$this->filesystem_root}" ); + } + + return $real_filesystem_root; + } + + /** Map one remote absolute path to its local absolute path. */ + public function map_remote_absolute_path( string $remote_absolute_path ): string { + assert_valid_path( $remote_absolute_path, 'remote absolute path' ); + $local_absolute_path = null; + $longest_remote_prefix_length = -1; + foreach ( $this->resolved_path_mappings as $remote_prefix => $local_prefix ) { + $remainder = self::path_remainder_under( $remote_absolute_path, $remote_prefix ); + if ( null !== $remainder && strlen( $remote_prefix ) > $longest_remote_prefix_length ) { + $local_absolute_path = wp_join_unix_paths( $local_prefix, $remainder ); + $longest_remote_prefix_length = strlen( $remote_prefix ); + } + } + if ( null !== $local_absolute_path ) { + return $local_absolute_path; + } + + if ( + null !== $this->local_followed_symlinks_root + && ! $this->is_within_original_export_scope( $remote_absolute_path ) + ) { + return $this->local_followed_symlinks_root . $remote_absolute_path; + } + + return $this->get_root_path() . $remote_absolute_path; + } + + /** Return why a new remote path must be preserved, or null when it may be written. */ + public function preserve_local_skip_reason( string $remote_absolute_path ): ?string { + if ( 'preserve-local' !== $this->nonempty_behavior ) { + return null; + } + + $local_absolute_path = $this->map_remote_absolute_path( $remote_absolute_path ); + if ( file_exists( $local_absolute_path ) || is_link( $local_absolute_path ) ) { + return "PRESERVE-LOCAL skip file (exists): {$remote_absolute_path}"; + } + + $parent_directory = dirname( $local_absolute_path ); + if ( is_dir( $parent_directory ) && ! is_writable( $parent_directory ) ) { + return "PRESERVE-LOCAL skip file (dir not writable): {$remote_absolute_path}"; + } + if ( $this->path_traverses_symlink( $parent_directory ) ) { + return "PRESERVE-LOCAL skip file (symlink in path): {$remote_absolute_path}"; + } + + return null; + } + + /** + * Write one streamed file chunk. + * + * @return array { + * Result of the filesystem operation. + * + * @type string|null $remote_absolute_path Decoded remote path, or null for an invalid header. + * @type string|null $warning Invalid-header warning. + * @type array|null $started Remote and local size information for a first chunk. + * @type string|null $skip_message Preserve-local reason when the file was skipped. + * @type bool $completed Whether this chunk completed the file. + * @type bool $file_changed Whether the remote reported a mid-stream change. + * @type int $final_size Local size after a completed write. + * @type array|null $index_entry Remote index fields for a stable completed file. + * } + */ + public function write_file_chunk( array $chunk, PulledFileContext $context ): array { + $result = array( + 'remote_absolute_path' => null, + 'warning' => null, + 'started' => null, + 'skip_message' => null, + 'completed' => false, + 'file_changed' => false, + 'final_size' => 0, + 'index_entry' => null, + ); + $headers = $chunk['headers']; + $raw_path = $headers['x-file-path'] ?? ''; + $remote_absolute_path = base64_decode( $raw_path, true ); + $is_first = ( $headers['x-first-chunk'] ?? '0' ) === '1'; + $is_last = ( $headers['x-last-chunk'] ?? '0' ) === '1'; + + if ( false === $remote_absolute_path || '' === $remote_absolute_path ) { + if ( '' !== $raw_path ) { + $result['warning'] = 'Warning: base64_decode failed for x-file-path header: ' . substr( $raw_path, 0, 100 ); + } + return $result; + } + + $result['remote_absolute_path'] = $remote_absolute_path; + $local_absolute_path = $this->map_remote_absolute_path( $remote_absolute_path ); + + if ( $is_first ) { + $context->skip_current_file = false; + if ( + ( file_exists( $local_absolute_path ) || is_link( $local_absolute_path ) ) + && ( ! is_file( $local_absolute_path ) || is_link( $local_absolute_path ) ) + && ! $this->remove_path( $local_absolute_path ) + ) { + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + throw new RuntimeException( "Failed to replace path with file: {$remote_absolute_path}" ); + } + + $exists_locally = file_exists( $local_absolute_path ); + $result['started'] = array( + 'remote_size' => (int) ( $headers['x-file-size'] ?? 0 ), + 'ctime' => (int) ( $headers['x-file-ctime'] ?? 0 ), + 'local_exists' => $exists_locally, + 'local_size' => $exists_locally ? (int) filesize( $local_absolute_path ) : 0, + ); + } + + if ( $context->skip_current_file ) { + return $result; + } + + if ( $is_first ) { + if ( $context->file_handle ) { + fclose( $context->file_handle ); + if ( $context->file_ctime && $context->file_path ) { + touch( $context->file_path, $context->file_ctime ); + } + } + + $parent_directory = dirname( $local_absolute_path ); + if ( ! is_dir( $parent_directory ) ) { + try { + $this->create_directory( $parent_directory ); + } catch ( PreserveLocalSkipException $error ) { + $context->skip_current_file = true; + $result['skip_message'] = $error->getMessage(); + return $result; + } + } + + $context->file_handle = fopen( $local_absolute_path, 'wb' ); + if ( ! $context->file_handle ) { + $last_error = error_get_last(); + throw new RuntimeException( + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + "Failed to open file for writing: {$local_absolute_path}\n" + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + . "Parent directory: {$parent_directory}\n" + . 'Directory exists: ' . ( is_dir( $parent_directory ) ? 'yes' : 'no' ) . "\n" + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + . 'Error: ' . ( $last_error['message'] ?? 'unknown' ) + ); + } + $context->file_path = $local_absolute_path; + $context->file_ctime = (int) ( $headers['x-file-ctime'] ?? 0 ); + $context->file_bytes_written = 0; + } + + if ( isset( $chunk['body'] ) && '' !== $chunk['body'] && $context->file_handle ) { + $data = $chunk['body']; + $bytes_written = fwrite( $context->file_handle, $data ); + if ( false === $bytes_written || $bytes_written !== strlen( $data ) ) { + throw new RuntimeException( + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + "Write failed for {$context->file_path}: wrote " + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + . ( false === $bytes_written ? '0' : $bytes_written ) . '/' . strlen( $data ) + . ' bytes (disk full?)' + ); + } + $context->file_bytes_written += $bytes_written; + } + + if ( $is_last && $context->file_handle ) { + fclose( $context->file_handle ); + if ( $context->file_ctime && $context->file_path ) { + touch( $context->file_path, $context->file_ctime ); + } + + $result['completed'] = true; + $result['file_changed'] = ( $headers['x-file-changed'] ?? '0' ) === '1'; + $result['final_size'] = file_exists( $context->file_path ) ? (int) filesize( $context->file_path ) : 0; + if ( $context->file_ctime && ! $result['file_changed'] ) { + $result['index_entry'] = array( + 'path' => $remote_absolute_path, + 'ctime' => $context->file_ctime, + 'size' => (int) ( $headers['x-file-size'] ?? 0 ), + 'type' => 'file', + ); + } + + $context->file_handle = null; + $context->file_path = null; + $context->file_ctime = null; + $context->file_bytes_written = 0; + } + + return $result; + } + + /** + * Materialize one remote directory. + * + * @return array { + * Result of the directory operation. + * + * @type string|null $remote_absolute_path Decoded remote path. + * @type string|null $warning Invalid-header warning. + * @type string|null $skip_message Preserve-local reason. + * @type int $ctime Remote ctime. + * } + */ + public function create_remote_directory( array $chunk ): array { + $headers = $chunk['headers']; + $raw_path = $headers['x-directory-path'] ?? ''; + $remote_absolute_path = base64_decode( $raw_path, true ); + $ctime = (int) ( $headers['x-directory-ctime'] ?? 0 ); + $result = array( + 'remote_absolute_path' => null, + 'warning' => null, + 'skip_message' => null, + 'ctime' => $ctime, + ); + + if ( false === $remote_absolute_path || '' === $remote_absolute_path ) { + if ( '' !== $raw_path ) { + $result['warning'] = 'Warning: base64_decode failed for x-directory-path header: ' . substr( $raw_path, 0, 100 ); + } + return $result; + } + + $result['remote_absolute_path'] = $remote_absolute_path; + $local_absolute_path = $this->map_remote_absolute_path( $remote_absolute_path ); + if ( 'preserve-local' === $this->nonempty_behavior ) { + if ( is_dir( $local_absolute_path ) ) { + $result['skip_message'] = "PRESERVE-LOCAL skip directory (exists): {$remote_absolute_path}"; + return $result; + } + if ( $this->path_traverses_symlink( $local_absolute_path ) ) { + $result['skip_message'] = "PRESERVE-LOCAL skip directory (symlink in path): {$remote_absolute_path}"; + return $result; + } + } + + if ( + ( file_exists( $local_absolute_path ) || is_link( $local_absolute_path ) ) + && ( ! is_dir( $local_absolute_path ) || is_link( $local_absolute_path ) ) + && ! $this->remove_path( $local_absolute_path ) + ) { + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + throw new RuntimeException( "Failed to replace path with directory: {$remote_absolute_path}" ); + } + + try { + $this->create_directory( $local_absolute_path ); + } catch ( PreserveLocalSkipException $error ) { + $result['skip_message'] = $error->getMessage(); + } + + return $result; + } + + /** + * Materialize one remote symlink using an already-routed local target. + * + * @return array { + * Result of the symlink operation. + * + * @type string|null $remote_absolute_path Decoded remote path. + * @type string|null $remote_target Original decoded target. + * @type string|null $local_target Target used for the local symlink. + * @type string|null $warning Invalid-header warning. + * @type string|null $skip_message Preserve-local reason. + * @type string|null $error Non-fatal creation error. + * @type int $ctime Remote ctime. + * } + */ + public function create_remote_symlink( array $chunk, string $local_target ): array { + $headers = $chunk['headers']; + $raw_path = $headers['x-symlink-path'] ?? ''; + $remote_absolute_path = base64_decode( $raw_path, true ); + $remote_target = base64_decode( $headers['x-symlink-target'] ?? '', true ); + $result = array( + 'remote_absolute_path' => null, + 'remote_target' => false === $remote_target ? null : $remote_target, + 'local_target' => $local_target, + 'warning' => null, + 'skip_message' => null, + 'error' => null, + 'ctime' => (int) ( $headers['x-symlink-ctime'] ?? 0 ), + ); + + if ( false === $remote_absolute_path || '' === $remote_absolute_path || false === $remote_target || '' === $remote_target ) { + if ( '' !== $raw_path && ( false === $remote_absolute_path || '' === $remote_absolute_path ) ) { + $result['warning'] = 'Warning: base64_decode failed for x-symlink-path header: ' . substr( $raw_path, 0, 100 ); + } + return $result; + } + + $result['remote_absolute_path'] = $remote_absolute_path; + $local_absolute_path = $this->map_remote_absolute_path( $remote_absolute_path ); + if ( 'preserve-local' === $this->nonempty_behavior ) { + if ( file_exists( $local_absolute_path ) || is_link( $local_absolute_path ) ) { + $result['skip_message'] = "PRESERVE-LOCAL skip symlink (path exists): {$remote_absolute_path} -> {$remote_target}"; + return $result; + } + if ( $this->path_traverses_symlink( dirname( $local_absolute_path ) ) ) { + $result['skip_message'] = "PRESERVE-LOCAL skip symlink (symlink in path): {$remote_absolute_path} -> {$remote_target}"; + return $result; + } + } + + try { + $this->assert_symlink_target_within_root( dirname( $local_absolute_path ), $local_target ); + } catch ( RuntimeException $error ) { + $result['error'] = $error->getMessage(); + return $result; + } + + if ( file_exists( $local_absolute_path ) || is_link( $local_absolute_path ) ) { + if ( ! $this->remove_path( $local_absolute_path ) ) { + $result['error'] = 'Failed to replace existing path'; + return $result; + } + } + + $parent_directory = dirname( $local_absolute_path ); + if ( ! is_dir( $parent_directory ) ) { + try { + $this->create_directory( $parent_directory ); + } catch ( PreserveLocalSkipException $error ) { + $result['skip_message'] = $error->getMessage(); + return $result; + } catch ( RuntimeException $error ) { + $result['error'] = 'Failed to create parent directory'; + return $result; + } + } + + if ( true !== symlink( $local_target, $local_absolute_path ) || ! is_link( $local_absolute_path ) ) { + $result['error'] = 'Failed to create symlink'; + return $result; + } + if ( $result['ctime'] > 0 ) { + @touch( $local_absolute_path, $result['ctime'] ); + } + + return $result; + } + + /** Remove a local path recursively without following symlink targets. */ + public function remove_path( string $local_absolute_path ): bool { + if ( ! file_exists( $local_absolute_path ) && ! is_link( $local_absolute_path ) ) { + return true; + } + if ( is_link( $local_absolute_path ) || is_file( $local_absolute_path ) ) { + return true === @unlink( $local_absolute_path ); + } + if ( is_dir( $local_absolute_path ) ) { + $entries = @scandir( $local_absolute_path ); + if ( false === $entries ) { + return false; + } + foreach ( $entries as $entry ) { + if ( '.' === $entry || '..' === $entry ) { + continue; + } + if ( ! $this->remove_path( $local_absolute_path . '/' . $entry ) ) { + return false; + } + } + return true === @rmdir( $local_absolute_path ); + } + return true === @unlink( $local_absolute_path ); + } + + /** Create a directory path, removing blockers without traversing symlinks. */ + public function create_directory( string $directory ): void { + $real_filesystem_root = $this->get_root_path(); + $check_path = $directory; + while ( ! file_exists( $check_path ) && $check_path !== dirname( $check_path ) ) { + $check_path = dirname( $check_path ); + } + + if ( file_exists( $check_path ) ) { + $real_check = realpath( $check_path ); + if ( false === $real_check || ! path_is_within_root( $real_check, $real_filesystem_root ) ) { + if ( 'preserve-local' === $this->nonempty_behavior ) { + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + throw new PreserveLocalSkipException( "PRESERVE-LOCAL: path resolves outside filesystem root via symlink: {$directory}" ); + } + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + throw new RuntimeException( "Security: Refusing to create directory outside filesystem root: {$directory}" ); + } + } + + if ( is_dir( $directory ) && ! is_link( $directory ) ) { + if ( 'preserve-local' === $this->nonempty_behavior && ! is_writable( $directory ) ) { + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + throw new PreserveLocalSkipException( "PRESERVE-LOCAL: directory not writable: {$directory}" ); + } + return; + } + + if ( $directory !== $real_filesystem_root && ! str_starts_with( $directory, $real_filesystem_root . '/' ) ) { + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + throw new RuntimeException( "Security: Refusing to create directory outside filesystem root: {$directory}" ); + } + + $relative_path = ltrim( substr( $directory, strlen( $real_filesystem_root ) ), '/' ); + if ( '' === $relative_path ) { + return; + } + + $current_path = $real_filesystem_root; + foreach ( explode( '/', $relative_path ) as $path_component ) { + if ( '' === $path_component ) { + continue; + } + $current_path .= '/' . $path_component; + if ( is_link( $current_path ) ) { + if ( 'preserve-local' === $this->nonempty_behavior ) { + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + throw new PreserveLocalSkipException( "PRESERVE-LOCAL: symlink in directory path: {$current_path}" ); + } + if ( ! unlink( $current_path ) ) { + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + throw new RuntimeException( "Failed to remove symlink blocking directory: {$current_path}" ); + } + $this->operation_messages[] = "Removing symlink blocking directory: {$current_path}"; + clearstatcache( true, $current_path ); + } + if ( is_file( $current_path ) ) { + if ( 'preserve-local' === $this->nonempty_behavior ) { + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + throw new PreserveLocalSkipException( "PRESERVE-LOCAL: file blocks directory creation: {$current_path}" ); + } + if ( ! unlink( $current_path ) ) { + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + throw new RuntimeException( "Failed to remove file blocking directory: {$current_path}" ); + } + $this->operation_messages[] = "Removing file blocking directory: {$current_path}"; + } + if ( is_dir( $current_path ) ) { + if ( 'preserve-local' === $this->nonempty_behavior && ! is_writable( $current_path ) ) { + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + throw new PreserveLocalSkipException( "PRESERVE-LOCAL: directory not writable: {$current_path}" ); + } + } elseif ( ! mkdir( $current_path, 0755 ) && ! is_dir( $current_path ) ) { + $last_error = error_get_last(); + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + throw new RuntimeException( "Failed to create directory: {$current_path}\nError: " . ( $last_error['message'] ?? 'unknown' ) ); + } + + $resolved_path = realpath( $current_path ); + if ( false === $resolved_path || ! path_is_within_root( $resolved_path, $real_filesystem_root ) ) { + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + throw new RuntimeException( "Security: Refusing to create directory outside filesystem root: {$current_path}" ); + } + } + } + + /** Assert that a local symlink target remains inside the filesystem root. */ + public function assert_symlink_target_within_root( string $symlink_parent_directory, string $target ): void { + $resolved_target = str_starts_with( $target, '/' ) + ? \WordPress\Reprint\Exporter\normalize_path( $target ) + : \WordPress\Reprint\Exporter\normalize_path( $symlink_parent_directory . '/' . $target ); + $filesystem_root = $this->get_root_path(); + if ( ! path_is_within_root( $resolved_target, $filesystem_root ) ) { + throw new RuntimeException( + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + "Security: symlink target escapes filesystem root: {$target} " + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- CLI exception, not HTML output. + . "(resolves to {$resolved_target}, root is {$filesystem_root})" + ); + } + } + + private function path_traverses_symlink( string $path ): bool { + $filesystem_root = $this->get_root_path(); + $relative_path = ltrim( substr( $path, strlen( $filesystem_root ) ), '/' ); + if ( '' === $relative_path ) { + return false; + } + $current_path = $filesystem_root; + foreach ( explode( '/', $relative_path ) as $path_component ) { + if ( '' === $path_component ) { + continue; + } + $current_path .= '/' . $path_component; + if ( is_link( $current_path ) ) { + return true; + } + if ( ! file_exists( $current_path ) ) { + break; + } + } + return false; + } + + private function is_within_original_export_scope( string $remote_absolute_path ): bool { + foreach ( $this->original_export_directories as $remote_root ) { + if ( path_is_within_root( $remote_absolute_path, $remote_root ) ) { + return true; + } + } + return false; + } + + private static function path_remainder_under( string $path, string $prefix ): ?string { + $path = rtrim( $path, '/' ); + $prefix = rtrim( $prefix, '/' ); + if ( $path === $prefix ) { + return ''; + } + if ( str_starts_with( $path, $prefix . '/' ) ) { + return substr( $path, strlen( $prefix ) ); + } + return null; + } +} diff --git a/packages/reprint-importer/src/lib/remote/class-streaming-context.php b/packages/reprint-importer/src/lib/remote/class-streaming-context.php new file mode 100644 index 00000000..8f06d90a --- /dev/null +++ b/packages/reprint-importer/src/lib/remote/class-streaming-context.php @@ -0,0 +1,19 @@ +|null Last response statistics from the completion part. */ + public $response_stats = array(); + + /** @var bool Whether the response contained its completion part. */ + public $saw_completion = false; +} diff --git a/tests/Import/CurlTimeoutRecoveryTest.php b/tests/Import/CurlTimeoutRecoveryTest.php index a8cf20c7..6a77322d 100644 --- a/tests/Import/CurlTimeoutRecoveryTest.php +++ b/tests/Import/CurlTimeoutRecoveryTest.php @@ -131,6 +131,17 @@ private function prepareClient(string $clientClass = TimeoutTestClient::class): $ttyProperty = $reflection->getProperty('is_tty'); $ttyProperty->setValue($client, false); + $reflection->getProperty('pulled_filesystem')->setValue( + $client, + new \Reprint\Importer\Filesystem\PulledFilesystem( + $this->filesystem_root, + [], + null, + 'preserve-local', + [], + ), + ); + return [$client, $reflection]; } @@ -578,7 +589,7 @@ class TimeoutTestClient extends \ImportClient protected function fetch_streaming( string $url, ?string $cursor, - \StreamingContext $context, + \Reprint\Importer\Remote\StreamingContext $context, ?array $post_data = null, ?string $endpoint = null ): void { @@ -599,7 +610,7 @@ class InterruptedAfterStreamedPartCloseClient extends \ImportClient protected function fetch_streaming( string $url, ?string $cursor, - \StreamingContext $context, + \Reprint\Importer\Remote\StreamingContext $context, ?array $post_data = null, ?string $endpoint = null ): void { @@ -641,7 +652,7 @@ class SuccessTestClient extends \ImportClient protected function fetch_streaming( string $url, ?string $cursor, - \StreamingContext $context, + \Reprint\Importer\Remote\StreamingContext $context, ?array $post_data = null, ?string $endpoint = null ): void { diff --git a/tests/Import/FileBodyStreamingTest.php b/tests/Import/FileBodyStreamingTest.php index 5736b480..cbbe49f4 100644 --- a/tests/Import/FileBodyStreamingTest.php +++ b/tests/Import/FileBodyStreamingTest.php @@ -27,18 +27,34 @@ protected function tearDown(): void parent::tearDown(); } - public function testFilePartBodiesAreWrittenIncrementally(): void + private function newClient(): \ImportClient { $client = new \ImportClient( 'http://fake.url', $this->tempDir . '/state', $this->tempDir . '/fs-root', ); + (new \ReflectionClass($client))->getProperty('pulled_filesystem')->setValue( + $client, + new \Reprint\Importer\Filesystem\PulledFilesystem( + $this->tempDir . '/fs-root', + [], + null, + 'error', + [], + ), + ); + return $client; + } + + public function testFilePartBodiesAreWrittenIncrementally(): void + { + $client = $this->newClient(); $reflection = new \ReflectionClass($client); $reflection->getProperty('is_tty')->setValue($client, true); $handleFileChunk = $reflection->getMethod('handle_file_chunk'); - $context = new \StreamingContext(); + $context = new \Reprint\Importer\Remote\StreamingContext(); $bodyLengths = []; $context->on_chunk = function (array $chunk) use ($client, $handleFileChunk, $context, &$bodyLengths): void { if (($chunk['headers']['x-chunk-type'] ?? '') === 'file') { @@ -98,11 +114,7 @@ public function testFilePartBodiesAreWrittenIncrementally(): void */ public function testMidFileResumeAppendsRemainingBytesWithoutDuplication(): void { - $client = new \ImportClient( - 'http://fake.url', - $this->tempDir . '/state', - $this->tempDir . '/fs-root', - ); + $client = $this->newClient(); $reflection = new \ReflectionClass($client); $reflection->getProperty('is_tty')->setValue($client, true); @@ -116,7 +128,7 @@ public function testMidFileResumeAppendsRemainingBytesWithoutDuplication(): void // sending it — so on resume we re-receive the whole part body. To // mimic the *intended* behaviour (server cooperates and skips the // already-written prefix), pass 2 sends only the missing tail. - $context1 = new \StreamingContext(); + $context1 = new \Reprint\Importer\Remote\StreamingContext(); $handleFileChunk = $reflection->getMethod('handle_file_chunk'); $context1->on_chunk = function (array $chunk) use ($client, $handleFileChunk, $context1): void { $handleFileChunk->invoke($client, $chunk, $context1); @@ -163,7 +175,7 @@ public function testMidFileResumeAppendsRemainingBytesWithoutDuplication(): void } $trackedBytes = $context1->file_bytes_written; - $context2 = new \StreamingContext(); + $context2 = new \Reprint\Importer\Remote\StreamingContext(); $context2->file_handle = fopen($target, 'ab'); $context2->file_path = $target; $context2->file_ctime = 1234567890; diff --git a/tests/Import/FilesPullStateTest.php b/tests/Import/FilesPullStateTest.php index d103d498..2928d3c5 100644 --- a/tests/Import/FilesPullStateTest.php +++ b/tests/Import/FilesPullStateTest.php @@ -136,6 +136,17 @@ private function prepareClient(): array $behaviorProp = $reflection->getProperty('fs_root_nonempty_behavior'); $behaviorProp->setValue($client, 'preserve-local'); + $reflection->getProperty('pulled_filesystem')->setValue( + $client, + new \Reprint\Importer\Filesystem\PulledFilesystem( + $this->filesystem_root, + [], + null, + 'preserve-local', + [], + ), + ); + return [$client, $reflection]; } @@ -143,6 +154,31 @@ private function prepareClient(): array // State transition tests // --------------------------------------------------------------- + /** Running files-pull without prepared options must fail before changing state. */ + public function testFilesPullRequiresPreparedOptions(): void + { + $this->writeState([ + "active_resumable_command" => [ + "command_name" => null, + "completion_state" => null, + ], + ]); + $state_file = $this->pullStateDirectory . '/state.json'; + $state_before_run = file_get_contents($state_file); + + try { + $this->makeClient()->run_files_pull(); + $this->fail('Expected files-pull to reject unprepared file options.'); + } catch (\LogicException $exception) { + $this->assertSame( + 'File pull options must be prepared before running files-pull.', + $exception->getMessage(), + ); + } + + $this->assertSame($state_before_run, file_get_contents($state_file)); + } + /** * A completed files-pull should refuse to re-run. */ @@ -437,7 +473,7 @@ public function testFetchStageOverwritesPreviouslySyncedFile() // Send a file chunk with new content $method = $reflection->getMethod('handle_file_chunk'); - $context = new \StreamingContext(); + $context = new \Reprint\Importer\Remote\StreamingContext(); $chunk = [ 'headers' => [ 'x-file-path' => base64_encode('/wp-content/themes/flavor/style.css'), @@ -471,7 +507,7 @@ class CompletedFileFetchClient extends \ImportClient protected function fetch_streaming( string $url, ?string $cursor, - \StreamingContext $context, + \Reprint\Importer\Remote\StreamingContext $context, ?array $post_data = null, ?string $endpoint = null ): void { diff --git a/tests/Import/FollowedSymlinksRootTest.php b/tests/Import/FollowedSymlinksRootTest.php index a7d23dc0..a6f984fe 100644 --- a/tests/Import/FollowedSymlinksRootTest.php +++ b/tests/Import/FollowedSymlinksRootTest.php @@ -94,10 +94,9 @@ public function testRelativeIsRejected(): void private function inScope(array $onlyPrefixes, string $path): bool { - $c = $this->newClient(); - $rc = new \ReflectionClass($c); - $rc->getProperty('pull_only_files_with_path_prefixes')->setValue($c, $onlyPrefixes); - return $rc->getMethod('path_is_within_original_export_scope')->invoke($c, $path); + $filesystem = $this->placeFilesystem(null, $onlyPrefixes); + $reflection = new \ReflectionClass($filesystem); + return $reflection->getMethod('is_within_original_export_scope')->invoke($filesystem, $path); } public function testTargetUnderScopeIsInScope(): void @@ -117,45 +116,46 @@ public function testTargetOutsideScopeEscapes(): void * @param array $scopePrefixes Original export scope (--only prefixes). * @param array $remapRules source => absolute target. */ - private function placeClient(?string $followedSymlinksRootSub, array $scopePrefixes, array $remapRules = []): \ImportClient + private function placeFilesystem(?string $followedSymlinksRootSub, array $scopePrefixes, array $remapRules = []): \Reprint\Importer\Filesystem\PulledFilesystem { - $c = $this->newClient(); - $rc = new \ReflectionClass($c); - $rc->getProperty('local_followed_symlinks_root')->setValue($c, $followedSymlinksRootSub === null ? null : $this->root . $followedSymlinksRootSub); - $rc->getProperty('pull_only_files_with_path_prefixes')->setValue($c, $scopePrefixes); - $rc->getProperty('resolved_path_mappings')->setValue($c, $remapRules); - return $c; + return new \Reprint\Importer\Filesystem\PulledFilesystem( + $this->fsRoot, + $remapRules, + $followedSymlinksRootSub === null ? null : $this->root . $followedSymlinksRootSub, + 'error', + $scopePrefixes, + ); } - private function place(\ImportClient $c, string $path): string + private function place(\Reprint\Importer\Filesystem\PulledFilesystem $filesystem, string $path): string { - return (new \ReflectionClass($c))->getMethod('map_remote_absolute_path_to_local_absolute_path')->invoke($c, $path); + return $filesystem->map_remote_absolute_path($path); } public function testEscapingTargetRoutesIntoLocalFollowedSymlinksRoot(): void { - $c = $this->placeClient('/.followed-symlinks-root', ['/var/www/html']); + $filesystem = $this->placeFilesystem('/.followed-symlinks-root', ['/var/www/html']); $this->assertSame( $this->root . '/.followed-symlinks-root/tmp/shared/foo/style.css', - $this->place($c, '/tmp/shared/foo/style.css') + $this->place($filesystem, '/tmp/shared/foo/style.css') ); } public function testInScopePathDoesNotUseLocalFollowedSymlinksRoot(): void { - $c = $this->placeClient('/.followed-symlinks-root', ['/var/www/html']); + $filesystem = $this->placeFilesystem('/.followed-symlinks-root', ['/var/www/html']); $this->assertSame( $this->root . '/var/www/html/index.php', - $this->place($c, '/var/www/html/index.php') + $this->place($filesystem, '/var/www/html/index.php') ); } public function testDefaultPlacementWhenNoLocalFollowedSymlinksRoot(): void { - $c = $this->placeClient(null, []); + $filesystem = $this->placeFilesystem(null, []); $this->assertSame( $this->root . '/tmp/shared/foo/style.css', - $this->place($c, '/tmp/shared/foo/style.css') + $this->place($filesystem, '/tmp/shared/foo/style.css') ); } @@ -163,15 +163,15 @@ public function testDefaultPlacementWhenNoLocalFollowedSymlinksRoot(): void // (/shared/wp-content) must not move the in-scope subtree. public function testAncestorEscapingRootLeavesInScopeContentInPlace(): void { - $c = $this->placeClient('/.followed-symlinks-root', ['/shared/wp-content']); + $filesystem = $this->placeFilesystem('/.followed-symlinks-root', ['/shared/wp-content']); $this->assertSame( $this->root . '/shared/wp-content/plugins/foo.php', - $this->place($c, '/shared/wp-content/plugins/foo.php'), + $this->place($filesystem, '/shared/wp-content/plugins/foo.php'), 'in-scope content must not use the local followed symlinks root' ); $this->assertSame( $this->root . '/.followed-symlinks-root/shared/other/bar.php', - $this->place($c, '/shared/other/bar.php'), + $this->place($filesystem, '/shared/other/bar.php'), 'genuinely escaping content uses the local followed symlinks root' ); } @@ -180,10 +180,10 @@ public function testAncestorEscapingRootLeavesInScopeContentInPlace(): void // and the symlink repoint (which share this seam) agree — no dangling link. public function testRemapWinsOverBundle(): void { - $c = $this->placeClient('/.followed-symlinks-root', ['/var/www/html'], ['/escaped' => $this->root . '/x']); + $filesystem = $this->placeFilesystem('/.followed-symlinks-root', ['/var/www/html'], ['/escaped' => $this->root . '/x']); $this->assertSame( $this->root . '/x/foo', - $this->place($c, '/escaped/foo'), + $this->place($filesystem, '/escaped/foo'), 'remap target wins; the path does not use the local followed symlinks root' ); } @@ -242,6 +242,13 @@ public function testWithinFsRootTargetRepointsToRemappedLocation(): void $rc = new \ReflectionClass($c); $rc->getProperty('resolved_path_mappings')->setValue($c, [$this->root . '/wp-content' => $this->root . '/custom']); $rc->getProperty('follow_symlinks')->setValue($c, true); + $rc->getProperty('pulled_filesystem')->setValue($c, new \Reprint\Importer\Filesystem\PulledFilesystem( + $this->fsRoot, + [$this->root . '/wp-content' => $this->root . '/custom'], + null, + 'error', + [$this->root], + )); $target = $this->root . '/wp-content/themes/x'; // realpath-clean, so it is its own cache key // Pretend the target subtree was followed + indexed. $rc->getProperty('next_remote_index_prefix_cache')->setValue($c, [$target => true]); @@ -271,6 +278,13 @@ public function testIntermediateSymlinkRepointsIntoLocalFollowedSymlinksRoot(): $rc->getProperty('local_followed_symlinks_root')->setValue($c, $this->root . '/.followed-symlinks-root'); $rc->getProperty('pull_only_files_with_path_prefixes')->setValue($c, ['/src/wp-content']); $rc->getProperty('follow_symlinks')->setValue($c, true); + $rc->getProperty('pulled_filesystem')->setValue($c, new \Reprint\Importer\Filesystem\PulledFilesystem( + $this->fsRoot, + [], + $this->root . '/.followed-symlinks-root', + 'error', + ['/src/wp-content'], + )); $rc->getProperty('next_remote_index_prefix_cache')->setValue($c, ['/opt/data' => true]); $entry = json_encode([ diff --git a/tests/Import/OnlyFilesPathPrefixDiffTest.php b/tests/Import/OnlyFilesPathPrefixDiffTest.php index cfa7d14f..5e33f978 100644 --- a/tests/Import/OnlyFilesPathPrefixDiffTest.php +++ b/tests/Import/OnlyFilesPathPrefixDiffTest.php @@ -130,6 +130,16 @@ private function prepareClient(array $pull_only_files_with_path_prefixes): array $r->getProperty('is_tty')->setValue($client, false); $r->getProperty('fs_root_nonempty_behavior')->setValue($client, 'preserve-local'); $r->getProperty('pull_only_files_with_path_prefixes')->setValue($client, $pull_only_files_with_path_prefixes); + $r->getProperty('pulled_filesystem')->setValue( + $client, + new \Reprint\Importer\Filesystem\PulledFilesystem( + $this->filesystem_root, + [], + null, + 'preserve-local', + $pull_only_files_with_path_prefixes, + ), + ); return [$client, $r]; } diff --git a/tests/Import/PullSymlinkTest.php b/tests/Import/PullSymlinkTest.php index b7e604c0..48338f9d 100644 --- a/tests/Import/PullSymlinkTest.php +++ b/tests/Import/PullSymlinkTest.php @@ -54,9 +54,25 @@ private function recursiveDelete(string $dir): void rmdir($dir); } - public function testSymlinkIsCreated() + private function newClient(): \ImportClient { $client = new \ImportClient('http://fake.url', $this->tempDir, $this->tempDir . '/fs-root'); + (new \ReflectionClass($client))->getProperty('pulled_filesystem')->setValue( + $client, + new \Reprint\Importer\Filesystem\PulledFilesystem( + $this->tempDir . '/fs-root', + [], + null, + 'error', + [], + ), + ); + return $client; + } + + public function testSymlinkIsCreated() + { + $client = $this->newClient(); $reflection = new \ReflectionClass($client); $method = $reflection->getMethod('handle_symlink_chunk'); @@ -83,7 +99,7 @@ public function testSymlinkIsCreated() */ public function testRelativeSymlinkEscapingRootRejected() { - $client = new \ImportClient('http://fake.url', $this->tempDir, $this->tempDir . '/fs-root'); + $client = $this->newClient(); $reflection = new \ReflectionClass($client); $method = $reflection->getMethod('handle_symlink_chunk'); @@ -109,7 +125,7 @@ public function testRelativeSymlinkEscapingRootRejected() */ public function testChainedSymlinksEscapingRootRejected() { - $client = new \ImportClient('http://fake.url', $this->tempDir, $this->tempDir . '/fs-root'); + $client = $this->newClient(); $reflection = new \ReflectionClass($client); $method = $reflection->getMethod('handle_symlink_chunk'); @@ -151,7 +167,7 @@ public function testChainedSymlinksEscapingRootRejected() */ public function testAbsoluteSymlinkOutsideRootRejected() { - $client = new \ImportClient('http://fake.url', $this->tempDir, $this->tempDir . '/fs-root'); + $client = $this->newClient(); $reflection = new \ReflectionClass($client); $method = $reflection->getMethod('handle_symlink_chunk'); @@ -176,7 +192,7 @@ public function testAbsoluteSymlinkOutsideRootRejected() */ public function testRelativeSymlinkWithinRootCreated() { - $client = new \ImportClient('http://fake.url', $this->tempDir, $this->tempDir . '/fs-root'); + $client = $this->newClient(); $reflection = new \ReflectionClass($client); $method = $reflection->getMethod('handle_symlink_chunk'); @@ -202,7 +218,7 @@ public function testRelativeSymlinkWithinRootCreated() */ public function testAbsoluteSymlinkWithinRootCreated() { - $client = new \ImportClient('http://fake.url', $this->tempDir, $this->tempDir . '/fs-root'); + $client = $this->newClient(); $root = realpath($this->tempDir . '/fs-root'); $reflection = new \ReflectionClass($client); @@ -225,7 +241,7 @@ public function testAbsoluteSymlinkWithinRootCreated() public function testSymlinkWithMissingDataSkipped() { - $client = new \ImportClient('http://fake.url', $this->tempDir, $this->tempDir . '/fs-root'); + $client = $this->newClient(); $reflection = new \ReflectionClass($client); $method = $reflection->getMethod('handle_symlink_chunk'); @@ -263,7 +279,7 @@ public function testSymlinkWithMissingDataSkipped() public function testSymlinkReplacesExistingFile() { - $client = new \ImportClient('http://fake.url', $this->tempDir, $this->tempDir . '/fs-root'); + $client = $this->newClient(); // Create a regular file $filePath = $this->tempDir . '/fs-root/test/link'; diff --git a/tests/Import/PulledFilesystemTest.php b/tests/Import/PulledFilesystemTest.php new file mode 100644 index 00000000..38fe2faa --- /dev/null +++ b/tests/Import/PulledFilesystemTest.php @@ -0,0 +1,238 @@ +temp_dir = sys_get_temp_dir() . '/pulled-filesystem-test-' . uniqid(); + $this->filesystem_root = $this->temp_dir . '/root'; + mkdir($this->temp_dir, 0755, true); + } + + protected function tearDown(): void + { + $this->removeTestPath($this->temp_dir); + parent::tearDown(); + } + + private function removeTestPath(string $path): void + { + if (is_link($path) || is_file($path)) { + unlink($path); + return; + } + if (!is_dir($path)) { + return; + } + foreach (scandir($path) as $entry) { + if ($entry === '.' || $entry === '..') { + continue; + } + $this->removeTestPath($path . '/' . $entry); + } + rmdir($path); + } + + private function newFilesystem( + string $nonempty_behavior = 'error', + array $resolved_path_mappings = [], + ?string $local_followed_symlinks_root = null, + array $original_export_directories = [] + ): PulledFilesystem { + return new PulledFilesystem( + $this->filesystem_root, + $resolved_path_mappings, + $local_followed_symlinks_root, + $nonempty_behavior, + $original_export_directories, + ); + } + + public function testConstructorAppliesCompleteRoutingConfiguration(): void + { + $mapped_root = $this->temp_dir . '/mapped'; + $followed_root = $this->temp_dir . '/followed'; + $filesystem = $this->newFilesystem( + 'preserve-local', + ['/remote/mapped' => $mapped_root], + $followed_root, + ['/remote/original'], + ); + + $this->assertSame( + $mapped_root . '/file.txt', + $filesystem->map_remote_absolute_path('/remote/mapped/file.txt'), + ); + $this->assertSame( + $filesystem->get_root_path() . '/remote/original/file.txt', + $filesystem->map_remote_absolute_path('/remote/original/file.txt'), + ); + $this->assertSame( + $followed_root . '/remote/outside/file.txt', + $filesystem->map_remote_absolute_path('/remote/outside/file.txt'), + ); + } + + public function testGetRootPathCreatesAndResolvesRoot(): void + { + $filesystem = $this->newFilesystem(); + + $root_path = $filesystem->get_root_path(); + + $this->assertTrue(is_dir($this->filesystem_root)); + $this->assertSame(realpath($this->filesystem_root), $root_path); + } + + public function testMapRemoteAbsolutePathUsesConfiguredMapping(): void + { + $mapped_root = $this->temp_dir . '/mapped'; + $filesystem = $this->newFilesystem('error', ['/remote/content' => $mapped_root]); + + $this->assertSame( + $mapped_root . '/plugin/file.php', + $filesystem->map_remote_absolute_path('/remote/content/plugin/file.php'), + ); + } + + public function testPreserveLocalSkipReasonUsesConfiguredBehavior(): void + { + $preserving_filesystem = $this->newFilesystem('preserve-local'); + $local_path = $preserving_filesystem->map_remote_absolute_path('/existing.txt'); + file_put_contents($local_path, 'local'); + + $this->assertSame( + 'PRESERVE-LOCAL skip file (exists): /existing.txt', + $preserving_filesystem->preserve_local_skip_reason('/existing.txt'), + ); + $this->assertNull( + $this->newFilesystem()->preserve_local_skip_reason('/existing.txt'), + ); + } + + public function testWriteFileChunkWritesAndReportsCompletedFile(): void + { + $filesystem = $this->newFilesystem(); + $context = new PulledFileContext(); + $result = $filesystem->write_file_chunk([ + 'headers' => [ + 'x-file-path' => base64_encode('/content/file.txt'), + 'x-first-chunk' => '1', + 'x-last-chunk' => '1', + 'x-file-ctime' => '1234', + 'x-file-size' => '7', + ], + 'body' => 'content', + ], $context); + + $this->assertSame('content', file_get_contents($this->filesystem_root . '/content/file.txt')); + $this->assertTrue($result['completed']); + $this->assertSame(7, $result['final_size']); + $this->assertSame([ + 'path' => '/content/file.txt', + 'ctime' => 1234, + 'size' => 7, + 'type' => 'file', + ], $result['index_entry']); + $this->assertNull($context->file_handle); + } + + public function testCreateRemoteDirectoryCreatesDecodedPath(): void + { + $filesystem = $this->newFilesystem(); + $result = $filesystem->create_remote_directory([ + 'headers' => [ + 'x-directory-path' => base64_encode('/content/directory'), + 'x-directory-ctime' => '2345', + ], + ]); + + $this->assertSame('/content/directory', $result['remote_absolute_path']); + $this->assertSame(2345, $result['ctime']); + $this->assertTrue(is_dir($this->filesystem_root . '/content/directory')); + } + + public function testCreateRemoteSymlinkCreatesDecodedPath(): void + { + $filesystem = $this->newFilesystem(); + $filesystem->create_directory($filesystem->get_root_path() . '/links'); + $result = $filesystem->create_remote_symlink([ + 'headers' => [ + 'x-symlink-path' => base64_encode('/links/link'), + 'x-symlink-target' => base64_encode('../target'), + 'x-symlink-ctime' => '0', + ], + ], '../target'); + + $local_path = $this->filesystem_root . '/links/link'; + $this->assertNull($result['error']); + $this->assertSame('/links/link', $result['remote_absolute_path']); + $this->assertTrue(is_link($local_path)); + $this->assertSame('../target', readlink($local_path)); + } + + public function testRemovePathRecursivelyRemovesTreeWithoutFollowingSymlinks(): void + { + $filesystem = $this->newFilesystem(); + $root_path = $filesystem->get_root_path(); + $external_path = $this->temp_dir . '/external'; + mkdir($external_path); + file_put_contents($external_path . '/keep.txt', 'keep'); + mkdir($root_path . '/tree/nested', 0755, true); + file_put_contents($root_path . '/tree/nested/remove.txt', 'remove'); + symlink($external_path, $root_path . '/tree/external-link'); + + $this->assertTrue($filesystem->remove_path($root_path . '/tree')); + $this->assertFileDoesNotExist($root_path . '/tree'); + $this->assertSame('keep', file_get_contents($external_path . '/keep.txt')); + } + + public function testCreateDirectoryReplacesBlockingFile(): void + { + $filesystem = $this->newFilesystem(); + $root_path = $filesystem->get_root_path(); + file_put_contents($root_path . '/blocked', 'blocker'); + + $filesystem->create_directory($root_path . '/blocked/child'); + + $this->assertTrue(is_dir($root_path . '/blocked/child')); + } + + public function testDrainOperationMessagesReturnsAndClearsMessages(): void + { + $filesystem = $this->newFilesystem(); + $root_path = $filesystem->get_root_path(); + file_put_contents($root_path . '/blocked', 'blocker'); + $filesystem->create_directory($root_path . '/blocked/child'); + + $this->assertSame( + ["Removing file blocking directory: {$root_path}/blocked"], + $filesystem->drain_operation_messages(), + ); + $this->assertSame([], $filesystem->drain_operation_messages()); + } + + public function testAssertSymlinkTargetWithinRootAcceptsInsideAndRejectsOutside(): void + { + $filesystem = $this->newFilesystem(); + $root_path = $filesystem->get_root_path(); + $filesystem->assert_symlink_target_within_root($root_path . '/links', '../target'); + $this->addToAssertionCount(1); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('Security: symlink target escapes filesystem root'); + $filesystem->assert_symlink_target_within_root($root_path . '/links', '../../outside'); + } +} diff --git a/tests/Import/RemapSeamTest.php b/tests/Import/RemapSeamTest.php index 134e2678..bb93b1ea 100644 --- a/tests/Import/RemapSeamTest.php +++ b/tests/Import/RemapSeamTest.php @@ -7,7 +7,7 @@ require_once __DIR__ . '/../../importer/import.php'; /** - * --remap: the single write seam (map_remote_absolute_path_to_local_absolute_path) + * --remap: the PulledFilesystem write seam * routes remote absolute paths to local absolute paths and leaves the rest nested. */ class RemapSeamTest extends TestCase @@ -54,26 +54,23 @@ private function call($c, string $m, array $a = array()) return (new \ReflectionClass($c))->getMethod($m)->invoke($c, ...$a); } - private function set($c, string $p, $v): void + private function filesystemWithRules(array $rules): \Reprint\Importer\Filesystem\PulledFilesystem { - (new \ReflectionClass($c))->getProperty($p)->setValue($c, $v); - } - - private function clientWithRules(array $rules): \ImportClient - { - $c = new \ImportClient('https://src.example/export.php', $this->stateDir, $this->fsRoot); - $this->set($c, 'resolved_path_mappings', $rules); - return $c; + return new \Reprint\Importer\Filesystem\PulledFilesystem( + $this->fsRoot, + $rules, + null, + 'error', + [], + ); } public function testRemoteAbsolutePathMapsToLocalAbsolutePath(): void { - $c = $this->clientWithRules(array( + $filesystem = $this->filesystemWithRules(array( '/var/www/html/wp-content' => $this->root . '/wp-content', )); - $local_absolute_path = $this->call($c, 'map_remote_absolute_path_to_local_absolute_path', array( - '/var/www/html/wp-content/plugins/woo/woo.php', - )); + $local_absolute_path = $filesystem->map_remote_absolute_path('/var/www/html/wp-content/plugins/woo/woo.php'); $this->assertSame($this->root . '/wp-content/plugins/woo/woo.php', $local_absolute_path); } @@ -82,13 +79,11 @@ public function testDeeperRemotePrefixWinsRegardlessOfLocalPrefixLength(): void // Two nested remote prefixes; the deeper (more specific) one has the // shorter local prefix. It must still win — specificity is ranked by // remote-prefix length, not local-prefix length. - $c = $this->clientWithRules(array( + $filesystem = $this->filesystemWithRules(array( '/srv/wp-content' => $this->root . '/archive-of-everything', '/srv/wp-content/plugins' => $this->root . '/p', )); - $local_absolute_path = $this->call($c, 'map_remote_absolute_path_to_local_absolute_path', array( - '/srv/wp-content/plugins/woo/woo.php', - )); + $local_absolute_path = $filesystem->map_remote_absolute_path('/srv/wp-content/plugins/woo/woo.php'); $this->assertSame($this->root . '/p/woo/woo.php', $local_absolute_path); } @@ -96,32 +91,26 @@ public function testLocalAbsolutePrefixPlacesFilesAtItsRoot(): void { // A local absolute prefix that is the filesystem root: files land directly at the root, // no double slash. - $c = $this->clientWithRules(array( + $filesystem = $this->filesystemWithRules(array( '/var/www/html/wp-content' => $this->root, )); - $local_absolute_path = $this->call($c, 'map_remote_absolute_path_to_local_absolute_path', array( - '/var/www/html/wp-content/plugins/woo/woo.php', - )); + $local_absolute_path = $filesystem->map_remote_absolute_path('/var/www/html/wp-content/plugins/woo/woo.php'); $this->assertSame($this->root . '/plugins/woo/woo.php', $local_absolute_path); } public function testOutOfScopePathFallsBackToNestedIdentity(): void { - $c = $this->clientWithRules(array( + $filesystem = $this->filesystemWithRules(array( '/var/www/html/wp-content' => $this->root . '/wp-content', )); - $local_absolute_path = $this->call($c, 'map_remote_absolute_path_to_local_absolute_path', array( - '/var/www/html/wp-admin/index.php', - )); + $local_absolute_path = $filesystem->map_remote_absolute_path('/var/www/html/wp-admin/index.php'); $this->assertSame($this->root . '/var/www/html/wp-admin/index.php', $local_absolute_path); } public function testNoRulesIsLegacyMapping(): void { - $c = $this->clientWithRules(array()); - $local_absolute_path = $this->call($c, 'map_remote_absolute_path_to_local_absolute_path', array( - '/var/www/html/wp-content/x.txt', - )); + $filesystem = $this->filesystemWithRules(array()); + $local_absolute_path = $filesystem->map_remote_absolute_path('/var/www/html/wp-content/x.txt'); $this->assertSame($this->root . '/var/www/html/wp-content/x.txt', $local_absolute_path); } @@ -133,8 +122,8 @@ public function testNoRulesIsLegacyMapping(): void */ public function testPathRemainderUnder(?string $expected, string $path, string $prefix): void { - $c = $this->clientWithRules(array()); - $this->assertSame($expected, $this->call($c, 'path_remainder_under', array($path, $prefix))); + $filesystem = $this->filesystemWithRules(array()); + $this->assertSame($expected, $this->call($filesystem, 'path_remainder_under', array($path, $prefix))); } public static function providePathRemainderCases(): array diff --git a/tests/Import/TypeSwapTest.php b/tests/Import/TypeSwapTest.php index 44163dca..c52a916c 100644 --- a/tests/Import/TypeSwapTest.php +++ b/tests/Import/TypeSwapTest.php @@ -56,18 +56,37 @@ private function recursiveDelete(string $dir): void rmdir($dir); } + private function newClient(): \ImportClient + { + $client = new \ImportClient('http://fake.url', $this->tempDir, $this->tempDir . '/fs-root'); + (new \ReflectionClass($client))->getProperty('pulled_filesystem')->setValue( + $client, + new \Reprint\Importer\Filesystem\PulledFilesystem( + $this->tempDir . '/fs-root', + [], + null, + 'error', + [], + ), + ); + return $client; + } + /** - * create_directory_if_missing should remove a symlink that blocks directory creation. + * PulledFilesystem should remove a symlink that blocks directory creation. */ public function testEnsureDirectoryPathRemovesBlockingSymlink() { - $client = new \ImportClient('http://fake.url', $this->tempDir, $this->tempDir . '/fs-root'); - - $reflection = new \ReflectionClass($client); - $method = $reflection->getMethod('create_directory_if_missing'); + $filesystem = new \Reprint\Importer\Filesystem\PulledFilesystem( + $this->tempDir . '/fs-root', + [], + null, + 'error', + [], + ); // Resolve the fs-root path so it matches the realpath() check - // inside create_directory_if_missing (on macOS, /var -> /private/var). + // inside PulledFilesystem (on macOS, /var -> /private/var). $fsRoot = realpath($this->tempDir . '/fs-root'); // Create a symlink at a path where we want a real directory @@ -77,8 +96,8 @@ public function testEnsureDirectoryPathRemovesBlockingSymlink() symlink($targetDir, $symlinkPath); $this->assertTrue(is_link($symlinkPath), 'Precondition: symlink exists'); - // create_directory_if_missing for a child should replace the symlink with a real dir - $method->invoke($client, $fsRoot . '/some-dir/child'); + // Creating a child should replace the symlink with a real directory. + $filesystem->create_directory($fsRoot . '/some-dir/child'); $this->assertFalse(is_link($symlinkPath), 'Symlink should be removed'); $this->assertTrue(is_dir($symlinkPath), 'Should be a real directory now'); @@ -90,7 +109,7 @@ public function testEnsureDirectoryPathRemovesBlockingSymlink() */ public function testFileChunkReplacesSymlinkToDirectory() { - $client = new \ImportClient('http://fake.url', $this->tempDir, $this->tempDir . '/fs-root'); + $client = $this->newClient(); $fsRoot = $this->tempDir . '/fs-root'; @@ -105,7 +124,7 @@ public function testFileChunkReplacesSymlinkToDirectory() $reflection = new \ReflectionClass($client); $method = $reflection->getMethod('handle_file_chunk'); - $context = new \StreamingContext(); + $context = new \Reprint\Importer\Remote\StreamingContext(); $chunk = [ 'headers' => [ 'x-file-path' => base64_encode('/swapped-path'), @@ -134,7 +153,7 @@ public function testFileChunkReplacesSymlinkToDirectory() */ public function testDirectoryChunkReplacesSymlinkToFile() { - $client = new \ImportClient('http://fake.url', $this->tempDir, $this->tempDir . '/fs-root'); + $client = $this->newClient(); $fsRoot = $this->tempDir . '/fs-root'; @@ -170,7 +189,7 @@ public function testDirectoryChunkReplacesSymlinkToFile() */ public function testFileChunkUnderFormerSymlink() { - $client = new \ImportClient('http://fake.url', $this->tempDir, $this->tempDir . '/fs-root'); + $client = $this->newClient(); $fsRoot = $this->tempDir . '/fs-root'; @@ -196,7 +215,7 @@ public function testFileChunkUnderFormerSymlink() // Step 2: file chunk writes a nested file $fileMethod = $reflection->getMethod('handle_file_chunk'); - $context = new \StreamingContext(); + $context = new \Reprint\Importer\Remote\StreamingContext(); $fileMethod->invoke($client, [ 'headers' => [ 'x-file-path' => base64_encode('/parent/sub/file.txt'), @@ -218,15 +237,21 @@ public function testFileChunkUnderFormerSymlink() } /** - * create_directory_if_missing should replace a symlink with a full real directory + * PulledFilesystem should replace a symlink with a full real directory * hierarchy when creating deeply nested paths. */ public function testNestedFileUnderExistingSymlinkViaEnsureDirectory() { - $client = new \ImportClient('http://fake.url', $this->tempDir, $this->tempDir . '/fs-root'); + $filesystem = new \Reprint\Importer\Filesystem\PulledFilesystem( + $this->tempDir . '/fs-root', + [], + null, + 'error', + [], + ); // Resolve the fs-root path so it matches the realpath() check - // inside create_directory_if_missing (on macOS, /var -> /private/var). + // inside PulledFilesystem (on macOS, /var -> /private/var). $fsRoot = realpath($this->tempDir . '/fs-root'); // Create a symlink at the top-level path component @@ -236,10 +261,7 @@ public function testNestedFileUnderExistingSymlinkViaEnsureDirectory() symlink($targetDir, $symlinkPath); $this->assertTrue(is_link($symlinkPath), 'Precondition: symlink exists'); - // Call create_directory_if_missing for a deeply nested path - $reflection = new \ReflectionClass($client); - $method = $reflection->getMethod('create_directory_if_missing'); - $method->invoke($client, $fsRoot . '/top/sub/deep'); + $filesystem->create_directory($fsRoot . '/top/sub/deep'); $this->assertFalse(is_link($symlinkPath), 'Symlink should be removed'); $this->assertTrue(is_dir($symlinkPath), 'top should be a real directory');