Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 38 additions & 10 deletions copier/_main.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import os
import platform
import re
import stat
import subprocess
import sys
Expand Down Expand Up @@ -1305,16 +1306,43 @@ def _apply_update(self) -> None: # noqa: C901
with local.cwd(subproject_top):
apply_cmd = git["apply", "--reject", "--exclude", self.answers_relpath]
ignored_files = git["status", "--ignored", "--porcelain"]()
# returns "!! file1\n !! file2\n"
# extra_exclude will contain: ["file1", file2"]
extra_exclude = [
filename.split("!! ").pop()
for filename in ignored_files.splitlines()
if filename.startswith("!! ")
]
for skip_pattern in chain(
map(self._render_string, self.all_skip_if_exists), extra_exclude
):
gitignore_path = Path(subproject_top, ".gitignore")
dir_patterns = set()
if gitignore_path.exists():
with gitignore_path.open() as gitignore_file:
for line in gitignore_file:
line = line.strip()
if not line or line.startswith("#"):
continue
# Matches foo/, foo/*, foo/**/, **/test/, **/test/*, etc.
if (
line.endswith("/")
or line.endswith("/*")
or line.endswith("/**/")
or re.match(r".*\*\*/.*", line)
):
cleaned = re.sub(r"(\/\*|\/\*\*\/|\/$)", "", line)
dir_patterns.add(cleaned)
# Single loop: process ignored files and build exclude lists
ignored_dirs = set()
extra_exclude = []
for filename in ignored_files.splitlines():
if filename.startswith("!! "):
fname = filename.split("!! ").pop()
matched_dir = False
for p in dir_patterns:
if fname == p or fname.startswith(p + "/"):
if fname.endswith("/"):
ignored_dirs.add(fname.rstrip("/"))
matched_dir = True
break
if not matched_dir:
extra_exclude.append(fname)
for dir_pattern in ignored_dirs:
apply_cmd = apply_cmd["--exclude", dir_pattern + "/*"]
for skip_pattern in map(self._render_string, self.all_skip_if_exists):
apply_cmd = apply_cmd["--exclude", skip_pattern]
for skip_pattern in extra_exclude:
Comment on lines +1309 to +1345
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TBH, this looks quite complex for a problem, I'm a bit worried that we're missing some edge cases.

I haven't looked into it in detail, but I think there are two ways to exclude files in the update:

  1. Exclude files when applying the patch via git apply --exclude, which has the limitation you've encountered.
  2. Omit the files to be excluded from the patch.

Option 2 sounds potentially simpler and more robust. WDYT?

apply_cmd = apply_cmd["--exclude", skip_pattern]
(apply_cmd << diff)(retcode=None)
if self.conflict == "inline":
Expand Down
116 changes: 116 additions & 0 deletions tests/test_updatediff.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ def test_updatediff(tmp_path_factory: pytest.TempPathFactory) -> None:
Thanks for your attention.
"""
),
(repo / ".gitignore.jinja"): ".venv/\n**/.cache/\n",
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think this added file serves any purpose in this test case.

Suggested change
(repo / ".gitignore.jinja"): ".venv/\n**/.cache/\n",

}
)
with local.cwd(repo):
Expand Down Expand Up @@ -2089,3 +2090,118 @@ def test_skip_if_exists_templated(tmp_path_factory: pytest.TempPathFactory) -> N

run_update(str(dst), overwrite=True)
assert (dst / "skip-if-exists.txt").read_text() == "baz"


def test_exclude_with_gitignore(tmp_path_factory: pytest.TempPathFactory) -> None:
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test passes even without the changes in the update algorithm, so it doesn't seem to cover the real scenario under test.

This test looks way too complex for the scenario under test. Could you please scope it down to the scenario where too many exclude patterns cause the OSError: [Errno 7] Argument list too long error?

src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
# Prepare repo bundle
repo = src / "repo"
bundle = src / "demo_updatediff_repo.bundle"
build_file_tree(
{
(repo / ".copier-answers.yml.jinja"): (
"""\
# Changes here will be overwritten by Copier
{{ _copier_answers|to_nice_yaml }}
"""
),
(repo / "copier.yml"): (
"""\
_envops:
"keep_trailing_newline": True
project_name: to become a pirate
author_name: Guybrush
"""
),
(repo / "README.txt.jinja"): (
"""
Let me introduce myself.

My name is {{author_name}}, and my project is {{project_name}}.

Thanks for your attention.
"""
),
(repo / ".gitignore.jinja"): ".venv/\n**/.cache/\n",
}
)
with local.cwd(repo):
git("init")
git("add", ".")
git("commit", "-m", "Guybrush wants to be a pirate")
git("tag", "v0.0.1")
build_file_tree(
{
(repo / "README.txt.jinja"): (
"""
Let me introduce myself.

My name is {{author_name}}.

My project is {{project_name}}.

Thanks for your attention.
"""
),
}
)
with local.cwd(repo):
git("init")
git("add", ".")
git("commit", "-m", "Update README format")
git("bundle", "create", bundle, "--all")
git("tag", "v0.0.2")

# Generate repo bundle
target = dst / "target"
readme = target / "README.txt"
commit = git["commit", "--all"]
# Run copier 1st time, with specific tag
CopierApp.run(
[
"copier",
"copy",
str(bundle),
str(target),
"--defaults",
"--overwrite",
"--vcs-ref=v0.0.1",
],
exit=False,
)
# Check it's copied OK
assert load_answersfile_data(target) == {
"_commit": "v0.0.1",
"_src_path": str(bundle),
"author_name": "Guybrush",
"project_name": "to become a pirate",
}
assert readme.read_text() == dedent(
"""
Let me introduce myself.

My name is Guybrush, and my project is to become a pirate.

Thanks for your attention.
"""
)
# Init destination as a new independent git repo
with local.cwd(target):
git("init")
# Commit changes
# Test .gitignore by creating files in the ignored directories
venv_dir = target / ".venv"
venv_dir.mkdir()
cache_dir = target / "test_cache/.cache"
cache_dir.mkdir(parents=True)
for i in range(3000):
(venv_dir / f"file_{i}.txt").write_text(f"dummy {i}")
(cache_dir / f"file_{i}.txt").write_text(f"dummy cache {i}")
git("add", ".")
commit("-m", "hello world")
# Update target to latest tag and check it's updated in answers file
CopierApp.run(["copier", "update", "--defaults", "--UNSAFE"], exit=False)
assert venv_dir.exists()
assert (venv_dir / "file_0.txt").exists()
assert cache_dir.exists()
assert (cache_dir / "file_0.txt").exists()