From 981ea5b72af9968fdceb9cc79ee97f853044a11c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20S=C3=A1enz?= Date: Tue, 7 Apr 2026 22:28:36 +0200 Subject: [PATCH] Maximize cached diff reuse to eliminate redundant directory traversals (#43) Closes #43 --- CHANGELOG.md | 10 ++++ Gemfile.lock | 2 +- .../actions/concerns/mappings_transfer.rb | 11 ++-- lib/dotsync/models/diff.rb | 5 +- lib/dotsync/utils/directory_differ.rb | 52 +++++++------------ lib/dotsync/utils/file_transfer.rb | 40 +++++++++++--- lib/dotsync/version.rb | 2 +- spec/dotsync/actions/pull_action_spec.rb | 14 +++-- spec/dotsync/actions/push_action_spec.rb | 6 +-- spec/dotsync/models/diff_spec.rb | 11 ++++ spec/dotsync/utils/file_transfer_spec.rb | 47 +++++++++++++++++ 11 files changed, 146 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abdeba8..84baaad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## [0.4.6] - 2026-04-07 + +### Changed + +- Maximize cached diff reuse to eliminate redundant directory traversals (#43) + - Build source index during first source walk in `DirectoryDiffer`, removing duplicate `build_source_index` traversal + - Pass pre-computed removal paths from diff to `FileTransfer`, skipping second destination scan in force mode + - Add `removal_rel_paths` field to `Diff` model to carry relative paths for `FileTransfer` + - Skip `show_orphan_preview` directory scan in display phase when no differences exist + ## [0.4.5] - 2026-04-07 ### Fixed diff --git a/Gemfile.lock b/Gemfile.lock index 9c1d221..eddc5c3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - dotsync (0.4.5) + dotsync (0.4.6) fileutils (~> 1.7.3) find (~> 0.2.0) listen (~> 3.9.0) diff --git a/lib/dotsync/actions/concerns/mappings_transfer.rb b/lib/dotsync/actions/concerns/mappings_transfer.rb index 4cd1eb4..8d48198 100644 --- a/lib/dotsync/actions/concerns/mappings_transfer.rb +++ b/lib/dotsync/actions/concerns/mappings_transfer.rb @@ -94,7 +94,7 @@ def show_differences(diff_content: false) show_content_diffs if diff_content && has_modifications? - show_orphan_preview if respond_to?(:manifests_xdg_data_home, true) + show_orphan_preview if has_differences? && respond_to?(:manifests_xdg_data_home, true) end def show_content_diffs @@ -115,13 +115,14 @@ def transfer_mappings mutex = Mutex.new # Only transfer mappings that have actual differences (uses cached diffs) - changed_mappings = valid_mappings.each_with_index.filter_map do |mapping, idx| - mapping if differs[idx].any? + changed_pairs = valid_mappings.each_with_index.filter_map do |mapping, idx| + [mapping, differs[idx]] if differs[idx].any? end # Process mappings in parallel - each mapping is independent - Dotsync::Parallel.each(changed_mappings) do |mapping| - Dotsync::FileTransfer.new(mapping).transfer + Dotsync::Parallel.each(changed_pairs) do |mapping, diff| + removals = diff.removal_rel_paths.map { |rel| File.join(mapping.dest, rel) } + Dotsync::FileTransfer.new(mapping, removals: removals).transfer rescue Dotsync::PermissionError => e mutex.synchronize { errors << ["Permission denied: #{e.message}", "Try: chmod +w or check file permissions"] } rescue Dotsync::DiskFullError => e diff --git a/lib/dotsync/models/diff.rb b/lib/dotsync/models/diff.rb index 9b62418..3f66586 100644 --- a/lib/dotsync/models/diff.rb +++ b/lib/dotsync/models/diff.rb @@ -3,13 +3,14 @@ module Dotsync # Represents the differences between two directories class Diff - attr_reader :additions, :modifications, :removals, :modification_pairs + attr_reader :additions, :modifications, :removals, :modification_pairs, :removal_rel_paths - def initialize(additions: [], modifications: [], removals: [], modification_pairs: []) + def initialize(additions: [], modifications: [], removals: [], modification_pairs: [], removal_rel_paths: []) @additions = additions @modifications = modifications @removals = removals @modification_pairs = modification_pairs + @removal_rel_paths = removal_rel_paths end def any? diff --git a/lib/dotsync/utils/directory_differ.rb b/lib/dotsync/utils/directory_differ.rb index 9bc28ae..87d80e7 100644 --- a/lib/dotsync/utils/directory_differ.rb +++ b/lib/dotsync/utils/directory_differ.rb @@ -11,11 +11,11 @@ module Dotsync # # This class implements several optimizations to handle large directory trees efficiently: # - # 1. **Pre-indexed source tree** (see #build_source_index) - # Instead of calling File.exist? for each destination file (disk I/O per file), - # we build a Set of all source paths upfront. Checking Set#include? is O(1) in memory - # vs O(1) disk I/O, which is orders of magnitude faster for large trees. - # Impact: ~100x faster for directories with thousands of files. + # 1. **Source index built during first walk** (see #diff_mapping_directories) + # The source tree walk that finds additions/modifications also builds a Set of all + # source paths. In force mode, this Set enables O(1) lookups during the destination + # walk — replacing per-file File.exist? calls (disk I/O) with hash lookups (memory). + # Impact: ~100x faster for directories with thousands of files, single source traversal. # # 2. **Early directory pruning with Find.prune** (see #diff_mapping_directories) # When an `only` filter is configured, we prune entire directory subtrees that @@ -58,6 +58,12 @@ def diff_mapping_directories modification_pairs = [] removals = [] + # OPTIMIZATION: Build the source index during the first walk. + # This Set is used in force mode for O(1) lookups during the destination walk, + # replacing per-file File.exist? calls (disk I/O) with hash lookups (memory). + # Building it here avoids a second traversal of the source tree. + source_index = Set.new + # Walk the source tree to find additions and modifications. # Uses bidirectional_include? with Find.prune to skip directories # that are outside the `only` filter, avoiding unnecessary traversal. @@ -72,6 +78,8 @@ def diff_mapping_directories next end + source_index << src_path + dest_path = File.join(mapping_dest, rel_path) if !File.exist?(dest_path) @@ -86,11 +94,6 @@ def diff_mapping_directories # In force mode, also find files in destination that don't exist in source (removals). if force? - # OPTIMIZATION: Pre-index source tree into a Set for O(1) lookups. - # This replaces per-file File.exist? calls (disk I/O) with hash lookups (memory). - # For a destination with thousands of files, this is orders of magnitude faster. - source_index = build_source_index - Find.find(mapping_dest) do |dest_path| rel_path = dest_path.sub(/^#{Regexp.escape(mapping_dest)}\/?/, "") next if rel_path.empty? @@ -118,11 +121,16 @@ def diff_mapping_directories filtered_modifications = filter_ignores(modifications) modification_pairs = modification_pairs.select { |pair| filtered_modifications.include?(pair[:rel_path]) } + filtered_removals = filter_ignores(removals) + additions = relative_to_absolute(filter_ignores(additions), mapping_original_dest) modifications = relative_to_absolute(filtered_modifications, mapping_original_dest) - removals = relative_to_absolute(filter_ignores(removals), mapping_original_dest) + removals = relative_to_absolute(filtered_removals, mapping_original_dest) - Dotsync::Diff.new(additions: additions, modifications: modifications, removals: removals, modification_pairs: modification_pairs) + Dotsync::Diff.new( + additions: additions, modifications: modifications, removals: removals, + modification_pairs: modification_pairs, removal_rel_paths: filtered_removals + ) end def diff_mapping_files @@ -140,26 +148,6 @@ def diff_mapping_files Dotsync::Diff.new(additions: additions, modifications: modifications, modification_pairs: modification_pairs) end - # Builds a Set of all source paths for O(1) existence checks. - # - # This is used during the destination walk (force mode) to check if a destination - # file exists in the source. Using a Set avoids repeated File.exist? calls, - # replacing disk I/O with memory lookups. - # - # @return [Set] Set of absolute source paths - def build_source_index - index = Set.new - Find.find(mapping_src) do |src_path| - # Apply the same pruning logic as the main source walk - unless @mapping.bidirectional_include?(src_path) - Find.prune - next - end - index << src_path - end - index - end - def filter_ignores(all_paths) return all_paths unless ignores.any? all_paths.reject do |path| diff --git a/lib/dotsync/utils/file_transfer.rb b/lib/dotsync/utils/file_transfer.rb index f5e5dd0..f9409de 100644 --- a/lib/dotsync/utils/file_transfer.rb +++ b/lib/dotsync/utils/file_transfer.rb @@ -5,17 +5,17 @@ class FileTransfer # Initializes a new FileTransfer instance # # @param mapping [Dotsync::Mapping] the mapping object containing source, destination, force, and ignore details - # @option mapping [String] :src the source directory path - # @option mapping [String] :dest the destination directory path - # @option mapping [Boolean] :force? optional flag to force actions - # @option mapping [Array] :ignores optional list of files/directories to ignore - def initialize(mapping) + # @param removals [Array, nil] pre-computed absolute paths to remove in force mode. + # When provided (from cached diff), skips the destination tree traversal in cleanup_folder. + # When nil, falls back to scanning the destination tree (used by backup transfers). + def initialize(mapping, removals: nil) @mapping = mapping @src = mapping.src @dest = mapping.dest @force = mapping.force? @inclusions = mapping.inclusions || [] @ignores = mapping.ignores || [] + @removals = removals end def transfer @@ -43,7 +43,13 @@ def transfer end transfer_file(@src, target_dest) else - cleanup_folder(@dest) if @force + if @force + if @removals + remove_precomputed_files + else + cleanup_folder(@dest) + end + end transfer_folder(@src, @dest) end end @@ -131,6 +137,28 @@ def transfer_symlink(symlink_src, symlink_dest) end end + # Removes files using pre-computed removal paths from the cached diff, + # avoiding a full destination tree traversal. After deleting files, walks + # up to clean empty parent directories. + def remove_precomputed_files + dest_expanded = File.expand_path(@dest) + + @removals.each do |path| + next unless File.file?(path) + + FileUtils.rm(path) + + # Clean up empty parent directories up to the destination root + dir = File.dirname(path) + while dir != dest_expanded && dir.start_with?(dest_expanded) + break unless Dir.empty?(dir) + + FileUtils.rmdir(dir) + dir = File.dirname(dir) + end + end + end + def cleanup_folder(target_dir) target_dir = File.expand_path(target_dir) diff --git a/lib/dotsync/version.rb b/lib/dotsync/version.rb index 334d0d2..254fe39 100644 --- a/lib/dotsync/version.rb +++ b/lib/dotsync/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Dotsync - VERSION = "0.4.5" + VERSION = "0.4.6" end diff --git a/spec/dotsync/actions/pull_action_spec.rb b/spec/dotsync/actions/pull_action_spec.rb index 3a9425a..45418a8 100644 --- a/spec/dotsync/actions/pull_action_spec.rb +++ b/spec/dotsync/actions/pull_action_spec.rb @@ -70,8 +70,8 @@ before do allow(Dotsync::FileTransfer).to receive(:new).and_call_original - allow(Dotsync::FileTransfer).to receive(:new).with(mappings[0]).and_return(file_transfer1) - allow(Dotsync::FileTransfer).to receive(:new).with(mappings[1]).and_return(file_transfer2) + allow(Dotsync::FileTransfer).to receive(:new).with(mappings[0], any_args).and_return(file_transfer1) + allow(Dotsync::FileTransfer).to receive(:new).with(mappings[1], any_args).and_return(file_transfer2) allow(file_transfer1).to receive(:transfer) allow(file_transfer2).to receive(:transfer) end @@ -104,6 +104,12 @@ action.execute end + + it "skips orphan preview scan" do + expect(action).not_to receive(:dest_files_matching_inclusions) + + action.execute + end end context "with apply option" do @@ -228,7 +234,7 @@ let(:mappings) { [mapping1, mapping_with_hooks] } before do - allow(Dotsync::FileTransfer).to receive(:new).with(mappings[1]).and_return(file_transfer2) + allow(Dotsync::FileTransfer).to receive(:new).with(mappings[1], any_args).and_return(file_transfer2) end it "executes hooks after transfer when files changed" do @@ -581,7 +587,7 @@ FileUtils.mkdir_p(File.dirname(manifest_path)) File.write(manifest_path, '{"files": ["elasticctl", "grafanactl", "setup-grafana"]}') - allow(Dotsync::FileTransfer).to receive(:new).with(mapping_with_inclusions).and_return( + allow(Dotsync::FileTransfer).to receive(:new).with(mapping_with_inclusions, any_args).and_return( instance_double("Dotsync::FileTransfer", transfer: nil) ) end diff --git a/spec/dotsync/actions/push_action_spec.rb b/spec/dotsync/actions/push_action_spec.rb index 474655d..2fec6e5 100644 --- a/spec/dotsync/actions/push_action_spec.rb +++ b/spec/dotsync/actions/push_action_spec.rb @@ -60,8 +60,8 @@ let(:color_diff_updated) { Dotsync::Colors.diff_modifications } before do - allow(Dotsync::FileTransfer).to receive(:new).with(mappings[0]).and_return(file_transfer1) - allow(Dotsync::FileTransfer).to receive(:new).with(mappings[1]).and_return(file_transfer2) + allow(Dotsync::FileTransfer).to receive(:new).with(mappings[0], any_args).and_return(file_transfer1) + allow(Dotsync::FileTransfer).to receive(:new).with(mappings[1], any_args).and_return(file_transfer2) allow(file_transfer1).to receive(:transfer) allow(file_transfer2).to receive(:transfer) end @@ -230,7 +230,7 @@ before do File.write(mapping_with_hooks.src, "#{mapping_with_hooks.src} content") File.write(mapping_with_hooks.dest, "#{mapping_with_hooks.dest} content") - allow(Dotsync::FileTransfer).to receive(:new).with(mappings[1]).and_return(file_transfer2) + allow(Dotsync::FileTransfer).to receive(:new).with(mappings[1], any_args).and_return(file_transfer2) end it "executes hooks after transfer when files changed" do diff --git a/spec/dotsync/models/diff_spec.rb b/spec/dotsync/models/diff_spec.rb index 1f9d17c..176871b 100644 --- a/spec/dotsync/models/diff_spec.rb +++ b/spec/dotsync/models/diff_spec.rb @@ -10,6 +10,7 @@ expect(diff.additions).to eq([]) expect(diff.modifications).to eq([]) expect(diff.removals).to eq([]) + expect(diff.removal_rel_paths).to eq([]) end end @@ -52,6 +53,16 @@ expect(diff.removals).to eq(["deleted.txt"]) end end + + context "with removal_rel_paths" do + it "stores relative removal paths for FileTransfer" do + diff = described_class.new( + removals: ["/dest/fold/file.txt"], + removal_rel_paths: ["fold/file.txt"] + ) + expect(diff.removal_rel_paths).to eq(["fold/file.txt"]) + end + end end describe "#any?" do diff --git a/spec/dotsync/utils/file_transfer_spec.rb b/spec/dotsync/utils/file_transfer_spec.rb index 907335d..c7a3f60 100644 --- a/spec/dotsync/utils/file_transfer_spec.rb +++ b/spec/dotsync/utils/file_transfer_spec.rb @@ -499,6 +499,53 @@ end end end + + context "with pre-computed removals" do + let(:force) { true } + let(:sanitized_dest) { config.dest } + + before do + FileUtils.mkdir_p(File.join(src, "fold")) + File.write(File.join(src, "fold", "kept.txt"), "kept content") + + FileUtils.mkdir_p(File.join(dest, "fold")) + File.write(File.join(dest, "fold", "kept.txt"), "old content") + File.write(File.join(dest, "fold", "removed.txt"), "to remove") + File.write(File.join(dest, "other.txt"), "other content") + end + + it "uses pre-computed removals instead of scanning destination" do + removals = [File.join(sanitized_dest, "fold", "removed.txt"), File.join(sanitized_dest, "other.txt")] + transfer = described_class.new(config, removals: removals) + transfer.transfer + + expect(File.exist?(File.join(dest, "fold", "removed.txt"))).to be false + expect(File.exist?(File.join(dest, "other.txt"))).to be false + expect(File.read(File.join(dest, "fold", "kept.txt"))).to eq("kept content") + end + + it "cleans up empty parent directories after removal" do + FileUtils.mkdir_p(File.join(dest, "empty_parent")) + File.write(File.join(dest, "empty_parent", "only_file.txt"), "content") + + removals = [File.join(sanitized_dest, "empty_parent", "only_file.txt")] + transfer = described_class.new(config, removals: removals) + transfer.transfer + + expect(File.exist?(File.join(dest, "empty_parent", "only_file.txt"))).to be false + expect(Dir.exist?(File.join(dest, "empty_parent"))).to be false + end + + it "preserves non-empty parent directories" do + removals = [File.join(sanitized_dest, "fold", "removed.txt")] + transfer = described_class.new(config, removals: removals) + transfer.transfer + + expect(File.exist?(File.join(dest, "fold", "removed.txt"))).to be false + expect(Dir.exist?(File.join(dest, "fold"))).to be true + expect(File.exist?(File.join(dest, "fold", "kept.txt"))).to be true + end + end end end