diff --git a/doc/architecture/diagrams.md b/doc/architecture/diagrams.md index 82512788..b3c1749c 100644 --- a/doc/architecture/diagrams.md +++ b/doc/architecture/diagrams.md @@ -349,8 +349,11 @@ sequenceDiagram end end - opt Submodules of upstream that were nested, or --recurse-submodules - Note over git-nested: Nest each at the commit upstream pins now (as in clone) + opt Submodules of upstream that are nested + Note over git-nested: Put them back from HEAD, with your changes.
Then pull each to the commit upstream pins now + end + opt --recurse-submodules + Note over git-nested: Nest each new one at the commit upstream pins (as in clone) end Note over git-nested: The commit message lists the new upstream commits @@ -691,6 +694,7 @@ sequenceDiagram git-nested->>Git: ls-files -- git-nested->>Git: rm -r -- git-nested->>Git: read-tree --prefix= -u + Note over git-nested: Inner nested repos: checkout HEAD -- , not the gitlink Note over git-nested: Write .gitnested opt patches in use git-nested->>Git: checkout HEAD -- @@ -708,6 +712,10 @@ sequenceDiagram Note over git-nested: Remove empty parent dirs up to .git/tmp git-nested->>Git: update-ref refs/nested//commit end + + opt Inner nested repos + Note over git-nested: Pull each to the commit upstream pins now (as in pull) + end ``` --- diff --git a/doc/cli/commit.md b/doc/cli/commit.md index f98e1f92..9be6bad2 100644 --- a/doc/cli/commit.md +++ b/doc/cli/commit.md @@ -48,6 +48,9 @@ git nested commit --file=msg.txt ext/lib it. Or pass `-F`. - The branch must contain the fetched upstream commit. Otherwise upstream changes would be lost. `--force` skips this check. +- Nested repositories inside `` stay nested, with your changes. + Then each one is pulled to the commit that upstream pins now, as `pull` + does. ## See also diff --git a/doc/cli/pull.md b/doc/cli/pull.md index 026f3c3a..af1cc1fa 100644 --- a/doc/cli/pull.md +++ b/doc/cli/pull.md @@ -138,9 +138,13 @@ This pull always rebases. Without `--pin`, a merge pull never goes back. is recorded. - A pin stays where it is on a normal pull. Only `--pin` moves it, forward or back. -- Submodules of upstream that are nested already stay nested. Each one moves - to the commit upstream pins after the pull. `--recurse-submodules` also - nests new ones. +- Submodules of upstream that are nested already stay nested. Each one is + pulled to the commit upstream pins now. Your changes in it stay. + `--recurse-submodules` also nests new ones. +- If one of them stops on conflicts, finish it. Then run the same pull + again. It updates the other ones. +- With `--force`, nested repositories inside lose their changes too. See + [Upstream submodules](../guides/submodules.md). - A worktree left from an earlier pull blocks a new one. Finish it, or run [`clean`](clean.md). - Once the nested repository has [patches](../configuration/gitnested-file.md#patches), diff --git a/doc/guides/submodules.md b/doc/guides/submodules.md index 184652e5..85cc8d69 100644 --- a/doc/guides/submodules.md +++ b/doc/guides/submodules.md @@ -16,8 +16,14 @@ Each one stays at the commit upstream pins. ### Pull -`pull` keeps them nested. After the pull, each one moves to the commit -upstream pins now. +`pull` keeps them nested. Each one is pulled to the commit upstream pins +now. Your changes in it stay. + +If one of them stops on conflicts, finish it as the message says. Then run +the same `pull` again. It updates the other ones. + +`pull --force` takes upstream as it is. Inner nested repositories lose +their changes too. Add `--recurse-submodules` to nest them again. If upstream adds a new submodule, a plain pull leaves it empty. To nest it too: diff --git a/src/git_nested/commands/clone.py b/src/git_nested/commands/clone.py index 4b2f2e66..5fa5f9a3 100644 --- a/src/git_nested/commands/clone.py +++ b/src/git_nested/commands/clone.py @@ -6,6 +6,7 @@ from .. import content, discovery, gitfile, output, refs, submodules from ..cli import setup +from ..constants import GITNESTED_FILENAME from ..errors import GitNestedError from ..git import GitRunner from ..models import CommandContext, Flags, NestedConfig @@ -178,31 +179,41 @@ def _clone_into( command='clone', ) if flags.recurse_submodules: - nest_submodules(git, flags, subdir, config.remote) + nest_submodules(git, flags, config, subdir) else: submodules.warn_if_any(git, subdir) -def nest_submodules(git: GitRunner, flags: Flags, subdir: Path, remote: str, only: set[str] | None = None) -> None: +def nest_submodules(git: GitRunner, flags: Flags, config: NestedConfig, subdir: Path) -> None: """Turn each submodule under subdir into a nested repository, at the commit it pins. - Each one is cloned with -b , so it stays at that commit. With - `only`, just the submodules at those paths. + Each one is cloned with -b , so it stays at that commit. The + submodules come from the tree subdir got its content from. Nested ones + are skipped. """ - for path, pinned in submodules.gitlinks(git, subdir): - if only is not None and path not in only: + in_index = dict(submodules.gitlinks(git, subdir)) + for relative, pinned in sorted(submodules.gitlinks_at(git, content.nested_tree(git, config, subdir)).items()): + path = f'{subdir.as_posix()}/{relative}' + if (Path(path) / GITNESTED_FILENAME).is_file() or ((config.filter or config.exclude) and path not in in_index): continue - url = submodules.url_of(git, subdir / '.gitmodules', path.removeprefix(f'{subdir.as_posix()}/'), remote) + url = submodules.url_of(git, subdir / '.gitmodules', relative, config.remote) if url is None: output.warn(f"{path}: .gitmodules has no url for it, so it stays empty") continue - git.run(['rm', '-q', '--cached', '--', path]) + git.run(['rm', '-q', '--cached', '--', path], may_fail=True) clone_at(git, flags, Path(path), url, pinned) def clone_at(git: GitRunner, flags: Flags, subdir: Path, url: str, commit: str) -> None: """Clone `url` at `commit` into subdir, and its submodules too.""" - inner = Flags(branch=commit, recurse_submodules=True, quiet=flags.quiet, verbose=flags.verbose) + inner = Flags( + branch=commit, + recurse_submodules=True, + gpg_sign=flags.gpg_sign, + signoff=flags.signoff, + quiet=flags.quiet, + verbose=flags.verbose, + ) subdir, gitnested, subref, config = setup.setup_command(git, 'clone', inner, subdir, url) do_clone(git, inner, config, subdir, gitnested, subref, git.check_output(['rev-parse', 'HEAD'])) output.say(f"{subdir.as_posix()}: cloned from {url} ({commit})") diff --git a/src/git_nested/commands/commit.py b/src/git_nested/commands/commit.py index 77c6ec9e..1134c19b 100644 --- a/src/git_nested/commands/commit.py +++ b/src/git_nested/commands/commit.py @@ -5,7 +5,7 @@ from .. import content, output, patches, refs, worktree from ..cli import setup from ..models import CommandContext -from . import fetch +from . import fetch, pull def cmd_commit(ctx: CommandContext) -> None: @@ -37,4 +37,5 @@ def cmd_commit(ctx: CommandContext) -> None: # After a pull that stopped on a conflict, this commit finishes it command=worktree.stopped(git, subdir_worktree) or 'commit', ) + pull.follow_inner(ctx, config, subdir) output.say(f"{subdir.as_posix()}: committed {nested_commit_ref}") diff --git a/src/git_nested/commands/pull.py b/src/git_nested/commands/pull.py index 348b5ad8..cd8bf656 100644 --- a/src/git_nested/commands/pull.py +++ b/src/git_nested/commands/pull.py @@ -4,11 +4,14 @@ import os import shutil +from dataclasses import replace from pathlib import Path +from typing import cast from uuid import uuid4 from .. import content, discovery, filters, gitfile, history, messages, output, patches, refs, submodules, worktree from ..cli import setup +from ..constants import GITNESTED_FILENAME from ..errors import GitNestedError from ..git import GitRunner from ..models import CommandContext, Flags, NestedConfig @@ -204,6 +207,8 @@ def _commit_without_squash( git.run(['merge', '--no-commit', '--no-ff', '-s', 'ours', '--allow-unrelated-histories', rewritten]) content._place_content(git, config, subdir, nested_commit_ref) gitfile.create_level_gitnested_files(git, subdir) + # After the level files: an inner nested repo of our own needs none + content.keep_inner_nested(git, flags, config, subdir, head_commit) regular = gitnested.parent / '.gitnested' for path in [gitnested] + ([regular] if regular != gitnested and regular.exists() else []): @@ -306,16 +311,55 @@ def cmd_pull(ctx: CommandContext) -> None: output.say(f"{subdir.as_posix()}: {verb} {config.remote} ({config.branch})") return - # Submodules of upstream that are nested now stay nested, at the commit upstream pins after the pull - nested_before = {f'{subdir.as_posix()}/{path}' for path in history.gitlinked_inner(git, config, subdir)} pulled = do_pull(git, flags, config, subdir, gitnested, ctx.tmp, subref) if pulled is None: + # An inner nested repo may still lag behind, if an earlier pull stopped there + follow_inner(ctx, config, subdir) output.say(f"{subdir.as_posix()}: already up to date with {config.remote} ({config.branch})") return branch, subdir_worktree = pulled output.verbose(f"committing the new {branch} content") _commit_pull(ctx, config, subdir, gitnested, subref, branch, subdir_worktree) - clone.nest_submodules(git, flags, subdir, config.remote, None if flags.recurse_submodules else nested_before) + follow_inner(ctx, config, subdir) + if flags.recurse_submodules: + clone.nest_submodules(git, flags, config, subdir) submodules.warn_if_any(git, subdir) output.say(f"{subdir.as_posix()}: pulled from {config.remote} ({config.branch})") + + +def follow_inner(ctx: CommandContext, config: NestedConfig, subdir: Path) -> None: + """Pull each inner nested repo to the commit that upstream pins now. Local changes stay. + + If one stops on conflicts, a new pull of subdir does the rest. + """ + git = ctx.git + for relative, pin in sorted(submodules.gitlinks_at(git, content.nested_tree(git, config, subdir)).items()): + path = subdir / relative + if not (path / GITNESTED_FILENAME).is_file(): + continue + try: + pull_to(ctx, path, submodules.url_of(git, subdir / '.gitmodules', relative, config.remote), pin) + except GitNestedError as e: + raise GitNestedError( + f"{e.message.rstrip()}\n\nThen run 'git nested pull {subdir.as_posix()}' for the other inner repositories." + ) from None + + +def pull_to(ctx: CommandContext, subdir: Path, url: str | None, pin: str) -> None: + """Pull subdir to `pin`, and from `url` if that is new. Local changes stay.""" + current = NestedConfig.from_file(subdir / GITNESTED_FILENAME) + if current.commit == pin: + return + inner = Flags( + branch=pin, + remote=url if url and url != current.remote else None, + update=True, + recurse_submodules=True, + gpg_sign=ctx.flags.gpg_sign, + signoff=ctx.flags.signoff, + quiet=ctx.flags.quiet, + verbose=ctx.flags.verbose, + ) + head = ctx.git.check_output(['rev-parse', 'HEAD']) + cmd_pull(cast('CommandContext', replace(ctx, flags=inner, subdir=subdir, head_commit=head))) diff --git a/src/git_nested/content.py b/src/git_nested/content.py index 12081f5a..30867384 100644 --- a/src/git_nested/content.py +++ b/src/git_nested/content.py @@ -6,7 +6,7 @@ from pathlib import Path, PurePosixPath from typing import cast -from . import discovery, filters, gitfile, history, output, patches, refs, worktree +from . import discovery, filters, gitfile, history, output, patches, refs, submodules, worktree from .constants import GITNESTED_FILENAME from .errors import GitNestedError from .git import GitRunner @@ -79,6 +79,8 @@ def commit_nested_branch( local = place_with_patches(git, config, subdir, nested_commit_ref) gitfile.create_level_gitnested_files(git, subdir) + # After the level files: an inner nested repo of our own needs none + keep_inner_nested(git, flags, config, subdir, head_commit) # A level file has the regular .gitnested next to it, which is kept in # step so that operating on the nested repo directly still sees current info. @@ -127,6 +129,40 @@ def place_with_patches(git: GitRunner, config: NestedConfig, subdir: Path, neste return local +def keep_inner_nested(git: GitRunner, flags: Flags, config: NestedConfig, subdir: Path, head: str | None) -> None: + """Do not commit a gitlink where a nested repo belongs. + + An inner nested repo comes back from `head`, with its local changes. + Not with --force: that takes upstream as it is. + With --recurse-submodules a new submodule is left out. It is nested next. + """ + drop_new = flags.recurse_submodules and not (config.filter or config.exclude) + for path, _ in submodules.gitlinks(git, subdir): + if not flags.force and _was_nested(git, head, path): + git.run(['update-index', '--force-remove', '--', path]) + git.run(['checkout', cast('str', head), '--', path]) + elif drop_new: + _drop_gitlink(git, Path(path)) + + +def _was_nested(git: GitRunner, head: str | None, path: str) -> bool: + return ( + bool(head) and git.run(['cat-file', '-e', f'{head}:{path}/{GITNESTED_FILENAME}'], may_fail=True).returncode == 0 + ) + + +def _drop_gitlink(git: GitRunner, path: Path) -> None: + git.run(['update-index', '--force-remove', '--', path]) + if path.is_dir() and not any(path.iterdir()): + path.rmdir() + + +def nested_tree(git: GitRunner, config: NestedConfig, subdir: Path) -> str: + """The tree that subdir got its content from in the last clone, pull or commit.""" + tree = f'refs/nested/{refs.sanitize_subref(git, subdir.as_posix())}/commit' + return f'{tree}:{config.path}' if config.path else tree + + def _place_content(git: GitRunner, config: NestedConfig, subdir: Path, nested_commit_ref: str) -> None: """Replace whatever subdir/ holds with the (filtered) tree of nested_commit_ref.""" if git.check_output(['ls-files', '--', subdir], may_fail=True): diff --git a/src/git_nested/submodules.py b/src/git_nested/submodules.py index 83c4162b..008ad44c 100644 --- a/src/git_nested/submodules.py +++ b/src/git_nested/submodules.py @@ -19,6 +19,16 @@ def gitlinks(git: GitRunner, subdir: Path) -> list[tuple[str, str]]: return links +def gitlinks_at(git: GitRunner, tree: str) -> dict[str, str]: + """The submodules in `tree`, as {path: pinned commit}. Empty if there is no such tree.""" + links = {} + for line in git.check_output(['ls-tree', '-r', '-z', tree], may_fail=True).split('\0'): + meta, _, path = line.partition('\t') + if meta.startswith('160000 '): + links[path] = meta.split()[2] + return links + + def entry(git: GitRunner, gitmodules: Path, path: str) -> tuple[str, str] | None: """The name and url `gitmodules` gives for the submodule at `path`, if any.""" if not gitmodules.is_file(): diff --git a/tests/e2e/test_submodules.py b/tests/e2e/test_submodules.py index fbfbcece..6d9740b1 100644 --- a/tests/e2e/test_submodules.py +++ b/tests/e2e/test_submodules.py @@ -55,6 +55,16 @@ def test_init_from_submodule(foo_bar_cloned): assert env.run(['git', 'status', '--porcelain'], cwd=foo).output == '' +def test_clone_recurse_submodules_leaves_an_excluded_one_out(foo_bar_cloned): + env = foo_bar_cloned + foo = env.workspace / 'foo' + _bar_with_a_submodule(env) + + cmd_git_nested(['clone', '--recurse-submodules', '--exclude', 'sub', (env.upstream / 'bar').as_posix()], foo) + + assert not (foo / 'bar' / 'sub').exists() + + def test_a_submodule_without_url_stays_empty(foo_bar_cloned): """A gitlink that .gitmodules does not describe cannot be nested; say so.""" env = foo_bar_cloned @@ -121,3 +131,134 @@ def test_push_keeps_the_submodule_of_upstream(foo_bar_cloned): entry = env.run(['git', 'ls-tree', 'master', 'sub'], cwd=env.upstream / 'bar').output.split() assert entry[:3] == ['160000', 'commit', pinned] + + +# ============================================================================ +# Inner nested repos keep their changes +# ============================================================================ + + +def _nest_bar_with_sub(env): + """Clone bar with its submodule 'sub' into foo, all nested. Return foo.""" + foo = env.workspace / 'foo' + _bar_with_a_submodule(env) + cmd_git_nested(['clone', '--recurse-submodules', (env.upstream / 'bar').as_posix()], foo) + return foo + + +def _assert_no_gitlink(env, repo): + modes = {line.split()[0] for line in env.run(['git', 'ls-files', '-s'], cwd=repo).output.splitlines()} + assert '160000' not in modes + + +def _resolve_in_worktree(env, repo, output, file, text): + """Fix a conflict where the help text says, and commit it there.""" + worktree = repo / next(line.split('cd ', 1)[1] for line in output.splitlines() if '1. cd ' in line) + (worktree / file).write_text(text) + env.run(['git', 'add', file], cwd=worktree) + env.run(['git', 'commit', '-q', '--no-edit'], cwd=worktree) + + +def test_clone_recurse_submodules_commits_no_gitlink(foo_bar_cloned): + env = foo_bar_cloned + foo = _nest_bar_with_sub(env) + + for commit in env.run(['git', 'rev-list', 'HEAD'], cwd=foo).output.split(): + tree = env.run(['git', 'ls-tree', '-r', commit], cwd=foo).output + assert '160000 commit' not in tree + + +def test_pull_keeps_changes_in_an_inner_nested_repo(foo_bar_cloned): + env = foo_bar_cloned + foo = _nest_bar_with_sub(env) + env.modify_files('bar/sub/Foo', text='local', cwd=foo) + pinned = _bump_the_submodule(env) + + cmd_git_nested('pull bar', cwd=foo) + + assert 'local' in (foo / 'bar' / 'sub' / 'Foo').read_text() + assert (foo / 'bar' / 'sub' / 'FooNew').is_file() + assert_gitnested_field(foo / 'bar' / 'sub' / '.gitnested', commit=pinned) + _assert_no_gitlink(env, foo) + + +def test_pull_no_squash_keeps_changes_in_an_inner_nested_repo(foo_bar_cloned): + env = foo_bar_cloned + foo = _nest_bar_with_sub(env) + env.modify_files('bar/sub/Foo', text='local', cwd=foo) + _bump_the_submodule(env) + + cmd_git_nested('pull --no-squash bar', cwd=foo) + + assert 'local' in (foo / 'bar' / 'sub' / 'Foo').read_text() + assert (foo / 'bar' / 'sub' / 'FooNew').is_file() + _assert_no_gitlink(env, foo) + + +def test_pull_force_drops_changes_in_an_inner_nested_repo(foo_bar_cloned): + """--force takes upstream as it is. --recurse-submodules nests sub again.""" + env = foo_bar_cloned + foo = _nest_bar_with_sub(env) + env.modify_files('bar/sub/Foo', text='local', cwd=foo) + pinned = _bump_the_submodule(env) + + cmd_git_nested('pull --force --recurse-submodules bar', cwd=foo) + + assert 'local' not in (foo / 'bar' / 'sub' / 'Foo').read_text() + assert_gitnested_field(foo / 'bar' / 'sub' / '.gitnested', commit=pinned) + _assert_no_gitlink(env, foo) + + +def test_commit_after_a_conflict_keeps_inner_nested_repos(foo_bar_cloned): + """You finish a pull by hand. sub stays nested. It moves to the new pin.""" + env = foo_bar_cloned + foo = _nest_bar_with_sub(env) + env.modify_files('bar/sub/Foo', text='local', cwd=foo) + env.modify_files('bar/Bar', text='mine', cwd=foo) + env.modify_files('Bar', text='theirs', cwd=env.workspace / 'bar') + env.run(['git', 'push', '-q'], cwd=env.workspace / 'bar') + pinned = _bump_the_submodule(env) + + result = cmd_git_nested('pull bar', cwd=foo, check=False) + assert result.returncode == 1 + _resolve_in_worktree(env, foo, result.output, 'Bar', 'both\n') + cmd_git_nested('commit bar', cwd=foo) + + assert (foo / 'bar' / 'Bar').read_text() == 'both\n' + assert 'local' in (foo / 'bar' / 'sub' / 'Foo').read_text() + assert_gitnested_field(foo / 'bar' / 'sub' / '.gitnested', commit=pinned) + _assert_no_gitlink(env, foo) + + +def test_pull_again_after_a_conflict_in_an_inner_nested_repo(foo_bar_cloned): + """The first inner repo stops on a conflict. A new pull updates the second one.""" + env = foo_bar_cloned + bar = env.workspace / 'bar' + _bar_with_a_submodule(env) + env.run(['git', *ALLOW_FILE, 'submodule', 'add', '-q', (env.upstream / 'foo').as_posix(), 'sub2'], cwd=bar) + env.run(['git', 'commit', '-qm', 'add sub2'], cwd=bar) + env.run(['git', 'push', '-q'], cwd=bar) + foo = env.workspace / 'foo' + cmd_git_nested(['clone', '--recurse-submodules', (env.upstream / 'bar').as_posix()], foo) + env.modify_files('bar/sub/Foo', text='local', cwd=foo) + other = env.workspace / 'foo-other' + env.run(['git', 'clone', '-q', (env.upstream / 'foo').as_posix(), other]) + env.modify_files('Foo', text='upstream', cwd=other) + env.run(['git', 'push', '-q'], cwd=other) + foo_upstream = git_rev_parse(['HEAD'], cwd=other) + for path in ('sub', 'sub2'): + env.run(['git', 'pull', '-q'], cwd=bar / path) + env.run(['git', 'commit', '-qam', 'bump both'], cwd=bar) + env.run(['git', 'push', '-q'], cwd=bar) + + result = cmd_git_nested('pull bar', cwd=foo, check=False) + + assert result.returncode == 1 + assert "git nested pull bar" in result.output + _resolve_in_worktree(env, foo, result.output, 'Foo', 'both\n') + cmd_git_nested('commit bar/sub', cwd=foo) + cmd_git_nested('pull bar', cwd=foo) + + assert_gitnested_field(foo / 'bar' / 'sub' / '.gitnested', commit=foo_upstream) + assert_gitnested_field(foo / 'bar' / 'sub2' / '.gitnested', commit=foo_upstream) + _assert_no_gitlink(env, foo)