Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
## [0.4.6] - 2026-04-07

### Changed

- Maximize cached diff reuse to eliminate redundant directory traversals (#43)
- Build source index during first source walk in `DirectoryDiffer`, removing duplicate `build_source_index` traversal
- Pass pre-computed removal paths from diff to `FileTransfer`, skipping second destination scan in force mode
- Add `removal_rel_paths` field to `Diff` model to carry relative paths for `FileTransfer`
- Skip `show_orphan_preview` directory scan in display phase when no differences exist

## [0.4.5] - 2026-04-07

### Fixed
Expand Down
2 changes: 1 addition & 1 deletion Gemfile.lock
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
PATH
remote: .
specs:
dotsync (0.4.5)
dotsync (0.4.6)
fileutils (~> 1.7.3)
find (~> 0.2.0)
listen (~> 3.9.0)
Expand Down
11 changes: 6 additions & 5 deletions lib/dotsync/actions/concerns/mappings_transfer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ def show_differences(diff_content: false)

show_content_diffs if diff_content && has_modifications?

show_orphan_preview if respond_to?(:manifests_xdg_data_home, true)
show_orphan_preview if has_differences? && respond_to?(:manifests_xdg_data_home, true)
end

def show_content_diffs
Expand All @@ -115,13 +115,14 @@ def transfer_mappings
mutex = Mutex.new

# Only transfer mappings that have actual differences (uses cached diffs)
changed_mappings = valid_mappings.each_with_index.filter_map do |mapping, idx|
mapping if differs[idx].any?
changed_pairs = valid_mappings.each_with_index.filter_map do |mapping, idx|
[mapping, differs[idx]] if differs[idx].any?
end

# Process mappings in parallel - each mapping is independent
Dotsync::Parallel.each(changed_mappings) do |mapping|
Dotsync::FileTransfer.new(mapping).transfer
Dotsync::Parallel.each(changed_pairs) do |mapping, diff|
removals = diff.removal_rel_paths.map { |rel| File.join(mapping.dest, rel) }
Dotsync::FileTransfer.new(mapping, removals: removals).transfer
rescue Dotsync::PermissionError => e
mutex.synchronize { errors << ["Permission denied: #{e.message}", "Try: chmod +w <path> or check file permissions"] }
rescue Dotsync::DiskFullError => e
Expand Down
5 changes: 3 additions & 2 deletions lib/dotsync/models/diff.rb
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@
module Dotsync
# Represents the differences between two directories
class Diff
attr_reader :additions, :modifications, :removals, :modification_pairs
attr_reader :additions, :modifications, :removals, :modification_pairs, :removal_rel_paths

def initialize(additions: [], modifications: [], removals: [], modification_pairs: [])
def initialize(additions: [], modifications: [], removals: [], modification_pairs: [], removal_rel_paths: [])
@additions = additions
@modifications = modifications
@removals = removals
@modification_pairs = modification_pairs
@removal_rel_paths = removal_rel_paths
end

def any?
Expand Down
52 changes: 20 additions & 32 deletions lib/dotsync/utils/directory_differ.rb
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@ module Dotsync
#
# This class implements several optimizations to handle large directory trees efficiently:
#
# 1. **Pre-indexed source tree** (see #build_source_index)
# Instead of calling File.exist? for each destination file (disk I/O per file),
# we build a Set of all source paths upfront. Checking Set#include? is O(1) in memory
# vs O(1) disk I/O, which is orders of magnitude faster for large trees.
# Impact: ~100x faster for directories with thousands of files.
# 1. **Source index built during first walk** (see #diff_mapping_directories)
# The source tree walk that finds additions/modifications also builds a Set of all
# source paths. In force mode, this Set enables O(1) lookups during the destination
# walk — replacing per-file File.exist? calls (disk I/O) with hash lookups (memory).
# Impact: ~100x faster for directories with thousands of files, single source traversal.
#
# 2. **Early directory pruning with Find.prune** (see #diff_mapping_directories)
# When an `only` filter is configured, we prune entire directory subtrees that
Expand Down Expand Up @@ -58,6 +58,12 @@ def diff_mapping_directories
modification_pairs = []
removals = []

# OPTIMIZATION: Build the source index during the first walk.
# This Set is used in force mode for O(1) lookups during the destination walk,
# replacing per-file File.exist? calls (disk I/O) with hash lookups (memory).
# Building it here avoids a second traversal of the source tree.
source_index = Set.new

# Walk the source tree to find additions and modifications.
# Uses bidirectional_include? with Find.prune to skip directories
# that are outside the `only` filter, avoiding unnecessary traversal.
Expand All @@ -72,6 +78,8 @@ def diff_mapping_directories
next
end

source_index << src_path

dest_path = File.join(mapping_dest, rel_path)

if !File.exist?(dest_path)
Expand All @@ -86,11 +94,6 @@ def diff_mapping_directories

# In force mode, also find files in destination that don't exist in source (removals).
if force?
# OPTIMIZATION: Pre-index source tree into a Set for O(1) lookups.
# This replaces per-file File.exist? calls (disk I/O) with hash lookups (memory).
# For a destination with thousands of files, this is orders of magnitude faster.
source_index = build_source_index

Find.find(mapping_dest) do |dest_path|
rel_path = dest_path.sub(/^#{Regexp.escape(mapping_dest)}\/?/, "")
next if rel_path.empty?
Expand Down Expand Up @@ -118,11 +121,16 @@ def diff_mapping_directories
filtered_modifications = filter_ignores(modifications)
modification_pairs = modification_pairs.select { |pair| filtered_modifications.include?(pair[:rel_path]) }

filtered_removals = filter_ignores(removals)

additions = relative_to_absolute(filter_ignores(additions), mapping_original_dest)
modifications = relative_to_absolute(filtered_modifications, mapping_original_dest)
removals = relative_to_absolute(filter_ignores(removals), mapping_original_dest)
removals = relative_to_absolute(filtered_removals, mapping_original_dest)

Dotsync::Diff.new(additions: additions, modifications: modifications, removals: removals, modification_pairs: modification_pairs)
Dotsync::Diff.new(
additions: additions, modifications: modifications, removals: removals,
modification_pairs: modification_pairs, removal_rel_paths: filtered_removals
)
end

def diff_mapping_files
Expand All @@ -140,26 +148,6 @@ def diff_mapping_files
Dotsync::Diff.new(additions: additions, modifications: modifications, modification_pairs: modification_pairs)
end

# Builds a Set of all source paths for O(1) existence checks.
#
# This is used during the destination walk (force mode) to check if a destination
# file exists in the source. Using a Set avoids repeated File.exist? calls,
# replacing disk I/O with memory lookups.
#
# @return [Set<String>] Set of absolute source paths
def build_source_index
index = Set.new
Find.find(mapping_src) do |src_path|
# Apply the same pruning logic as the main source walk
unless @mapping.bidirectional_include?(src_path)
Find.prune
next
end
index << src_path
end
index
end

def filter_ignores(all_paths)
return all_paths unless ignores.any?
all_paths.reject do |path|
Expand Down
40 changes: 34 additions & 6 deletions lib/dotsync/utils/file_transfer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,17 @@ class FileTransfer
# Initializes a new FileTransfer instance
#
# @param mapping [Dotsync::Mapping] the mapping object containing source, destination, force, and ignore details
# @option mapping [String] :src the source directory path
# @option mapping [String] :dest the destination directory path
# @option mapping [Boolean] :force? optional flag to force actions
# @option mapping [Array<String>] :ignores optional list of files/directories to ignore
def initialize(mapping)
# @param removals [Array<String>, nil] pre-computed absolute paths to remove in force mode.
# When provided (from cached diff), skips the destination tree traversal in cleanup_folder.
# When nil, falls back to scanning the destination tree (used by backup transfers).
def initialize(mapping, removals: nil)
@mapping = mapping
@src = mapping.src
@dest = mapping.dest
@force = mapping.force?
@inclusions = mapping.inclusions || []
@ignores = mapping.ignores || []
@removals = removals
end

def transfer
Expand Down Expand Up @@ -43,7 +43,13 @@ def transfer
end
transfer_file(@src, target_dest)
else
cleanup_folder(@dest) if @force
if @force
if @removals
remove_precomputed_files
else
cleanup_folder(@dest)
end
end
transfer_folder(@src, @dest)
end
end
Expand Down Expand Up @@ -131,6 +137,28 @@ def transfer_symlink(symlink_src, symlink_dest)
end
end

# Removes files using pre-computed removal paths from the cached diff,
# avoiding a full destination tree traversal. After deleting files, walks
# up to clean empty parent directories.
def remove_precomputed_files
dest_expanded = File.expand_path(@dest)

@removals.each do |path|
next unless File.file?(path)

FileUtils.rm(path)

# Clean up empty parent directories up to the destination root
dir = File.dirname(path)
while dir != dest_expanded && dir.start_with?(dest_expanded)
break unless Dir.empty?(dir)

FileUtils.rmdir(dir)
dir = File.dirname(dir)
end
end
end

def cleanup_folder(target_dir)
target_dir = File.expand_path(target_dir)

Expand Down
2 changes: 1 addition & 1 deletion lib/dotsync/version.rb
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# frozen_string_literal: true

module Dotsync
VERSION = "0.4.5"
VERSION = "0.4.6"
end
14 changes: 10 additions & 4 deletions spec/dotsync/actions/pull_action_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,8 @@

before do
allow(Dotsync::FileTransfer).to receive(:new).and_call_original
allow(Dotsync::FileTransfer).to receive(:new).with(mappings[0]).and_return(file_transfer1)
allow(Dotsync::FileTransfer).to receive(:new).with(mappings[1]).and_return(file_transfer2)
allow(Dotsync::FileTransfer).to receive(:new).with(mappings[0], any_args).and_return(file_transfer1)
allow(Dotsync::FileTransfer).to receive(:new).with(mappings[1], any_args).and_return(file_transfer2)
allow(file_transfer1).to receive(:transfer)
allow(file_transfer2).to receive(:transfer)
end
Expand Down Expand Up @@ -104,6 +104,12 @@

action.execute
end

it "skips orphan preview scan" do
expect(action).not_to receive(:dest_files_matching_inclusions)

action.execute
end
end

context "with apply option" do
Expand Down Expand Up @@ -228,7 +234,7 @@
let(:mappings) { [mapping1, mapping_with_hooks] }

before do
allow(Dotsync::FileTransfer).to receive(:new).with(mappings[1]).and_return(file_transfer2)
allow(Dotsync::FileTransfer).to receive(:new).with(mappings[1], any_args).and_return(file_transfer2)
end

it "executes hooks after transfer when files changed" do
Expand Down Expand Up @@ -581,7 +587,7 @@
FileUtils.mkdir_p(File.dirname(manifest_path))
File.write(manifest_path, '{"files": ["elasticctl", "grafanactl", "setup-grafana"]}')

allow(Dotsync::FileTransfer).to receive(:new).with(mapping_with_inclusions).and_return(
allow(Dotsync::FileTransfer).to receive(:new).with(mapping_with_inclusions, any_args).and_return(
instance_double("Dotsync::FileTransfer", transfer: nil)
)
end
Expand Down
6 changes: 3 additions & 3 deletions spec/dotsync/actions/push_action_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@
let(:color_diff_updated) { Dotsync::Colors.diff_modifications }

before do
allow(Dotsync::FileTransfer).to receive(:new).with(mappings[0]).and_return(file_transfer1)
allow(Dotsync::FileTransfer).to receive(:new).with(mappings[1]).and_return(file_transfer2)
allow(Dotsync::FileTransfer).to receive(:new).with(mappings[0], any_args).and_return(file_transfer1)
allow(Dotsync::FileTransfer).to receive(:new).with(mappings[1], any_args).and_return(file_transfer2)
allow(file_transfer1).to receive(:transfer)
allow(file_transfer2).to receive(:transfer)
end
Expand Down Expand Up @@ -230,7 +230,7 @@
before do
File.write(mapping_with_hooks.src, "#{mapping_with_hooks.src} content")
File.write(mapping_with_hooks.dest, "#{mapping_with_hooks.dest} content")
allow(Dotsync::FileTransfer).to receive(:new).with(mappings[1]).and_return(file_transfer2)
allow(Dotsync::FileTransfer).to receive(:new).with(mappings[1], any_args).and_return(file_transfer2)
end

it "executes hooks after transfer when files changed" do
Expand Down
11 changes: 11 additions & 0 deletions spec/dotsync/models/diff_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
expect(diff.additions).to eq([])
expect(diff.modifications).to eq([])
expect(diff.removals).to eq([])
expect(diff.removal_rel_paths).to eq([])
end
end

Expand Down Expand Up @@ -52,6 +53,16 @@
expect(diff.removals).to eq(["deleted.txt"])
end
end

context "with removal_rel_paths" do
it "stores relative removal paths for FileTransfer" do
diff = described_class.new(
removals: ["/dest/fold/file.txt"],
removal_rel_paths: ["fold/file.txt"]
)
expect(diff.removal_rel_paths).to eq(["fold/file.txt"])
end
end
end

describe "#any?" do
Expand Down
47 changes: 47 additions & 0 deletions spec/dotsync/utils/file_transfer_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -499,6 +499,53 @@
end
end
end

context "with pre-computed removals" do
let(:force) { true }
let(:sanitized_dest) { config.dest }

before do
FileUtils.mkdir_p(File.join(src, "fold"))
File.write(File.join(src, "fold", "kept.txt"), "kept content")

FileUtils.mkdir_p(File.join(dest, "fold"))
File.write(File.join(dest, "fold", "kept.txt"), "old content")
File.write(File.join(dest, "fold", "removed.txt"), "to remove")
File.write(File.join(dest, "other.txt"), "other content")
end

it "uses pre-computed removals instead of scanning destination" do
removals = [File.join(sanitized_dest, "fold", "removed.txt"), File.join(sanitized_dest, "other.txt")]
transfer = described_class.new(config, removals: removals)
transfer.transfer

expect(File.exist?(File.join(dest, "fold", "removed.txt"))).to be false
expect(File.exist?(File.join(dest, "other.txt"))).to be false
expect(File.read(File.join(dest, "fold", "kept.txt"))).to eq("kept content")
end

it "cleans up empty parent directories after removal" do
FileUtils.mkdir_p(File.join(dest, "empty_parent"))
File.write(File.join(dest, "empty_parent", "only_file.txt"), "content")

removals = [File.join(sanitized_dest, "empty_parent", "only_file.txt")]
transfer = described_class.new(config, removals: removals)
transfer.transfer

expect(File.exist?(File.join(dest, "empty_parent", "only_file.txt"))).to be false
expect(Dir.exist?(File.join(dest, "empty_parent"))).to be false
end

it "preserves non-empty parent directories" do
removals = [File.join(sanitized_dest, "fold", "removed.txt")]
transfer = described_class.new(config, removals: removals)
transfer.transfer

expect(File.exist?(File.join(dest, "fold", "removed.txt"))).to be false
expect(Dir.exist?(File.join(dest, "fold"))).to be true
expect(File.exist?(File.join(dest, "fold", "kept.txt"))).to be true
end
end
end
end

Expand Down
Loading