diff --git a/doc/architecture/diagrams.md b/doc/architecture/diagrams.md
index 82512788..b3c1749c 100644
--- a/doc/architecture/diagrams.md
+++ b/doc/architecture/diagrams.md
@@ -349,8 +349,11 @@ sequenceDiagram
end
end
- opt Submodules of upstream that were nested, or --recurse-submodules
- Note over git-nested: Nest each at the commit upstream pins now (as in clone)
+ opt Submodules of upstream that are nested
+ Note over git-nested: Put them back from HEAD, with your changes.
Then pull each to the commit upstream pins now
+ end
+ opt --recurse-submodules
+ Note over git-nested: Nest each new one at the commit upstream pins (as in clone)
end
Note over git-nested: The commit message lists the new upstream commits
@@ -691,6 +694,7 @@ sequenceDiagram
git-nested->>Git: ls-files --
git-nested->>Git: rm -r --
git-nested->>Git: read-tree --prefix= -u
+ Note over git-nested: Inner nested repos: checkout HEAD -- , not the gitlink
Note over git-nested: Write .gitnested
opt patches in use
git-nested->>Git: checkout HEAD --
@@ -708,6 +712,10 @@ sequenceDiagram
Note over git-nested: Remove empty parent dirs up to .git/tmp
git-nested->>Git: update-ref refs/nested//commit
end
+
+ opt Inner nested repos
+ Note over git-nested: Pull each to the commit upstream pins now (as in pull)
+ end
```
---
diff --git a/doc/cli/commit.md b/doc/cli/commit.md
index f98e1f92..9be6bad2 100644
--- a/doc/cli/commit.md
+++ b/doc/cli/commit.md
@@ -48,6 +48,9 @@ git nested commit --file=msg.txt ext/lib
it. Or pass `-F`.
- The branch must contain the fetched upstream commit. Otherwise upstream
changes would be lost. `--force` skips this check.
+- Nested repositories inside `` stay nested, with your changes.
+ Then each one is pulled to the commit that upstream pins now, as `pull`
+ does.
## See also
diff --git a/doc/cli/pull.md b/doc/cli/pull.md
index 026f3c3a..af1cc1fa 100644
--- a/doc/cli/pull.md
+++ b/doc/cli/pull.md
@@ -138,9 +138,13 @@ This pull always rebases. Without `--pin`, a merge pull never goes back.
is recorded.
- A pin stays where it is on a normal pull. Only `--pin` moves it,
forward or back.
-- Submodules of upstream that are nested already stay nested. Each one moves
- to the commit upstream pins after the pull. `--recurse-submodules` also
- nests new ones.
+- Submodules of upstream that are nested already stay nested. Each one is
+ pulled to the commit upstream pins now. Your changes in it stay.
+ `--recurse-submodules` also nests new ones.
+- If one of them stops on conflicts, finish it. Then run the same pull
+ again. It updates the other ones.
+- With `--force`, nested repositories inside lose their changes too. See
+ [Upstream submodules](../guides/submodules.md).
- A worktree left from an earlier pull blocks a new one. Finish it, or run
[`clean`](clean.md).
- Once the nested repository has [patches](../configuration/gitnested-file.md#patches),
diff --git a/doc/guides/submodules.md b/doc/guides/submodules.md
index 184652e5..85cc8d69 100644
--- a/doc/guides/submodules.md
+++ b/doc/guides/submodules.md
@@ -16,8 +16,14 @@ Each one stays at the commit upstream pins.
### Pull
-`pull` keeps them nested. After the pull, each one moves to the commit
-upstream pins now.
+`pull` keeps them nested. Each one is pulled to the commit upstream pins
+now. Your changes in it stay.
+
+If one of them stops on conflicts, finish it as the message says. Then run
+the same `pull` again. It updates the other ones.
+
+`pull --force` takes upstream as it is. Inner nested repositories lose
+their changes too. Add `--recurse-submodules` to nest them again.
If upstream adds a new submodule, a plain pull leaves it empty. To nest it
too:
diff --git a/src/git_nested/commands/clone.py b/src/git_nested/commands/clone.py
index 4b2f2e66..5fa5f9a3 100644
--- a/src/git_nested/commands/clone.py
+++ b/src/git_nested/commands/clone.py
@@ -6,6 +6,7 @@
from .. import content, discovery, gitfile, output, refs, submodules
from ..cli import setup
+from ..constants import GITNESTED_FILENAME
from ..errors import GitNestedError
from ..git import GitRunner
from ..models import CommandContext, Flags, NestedConfig
@@ -178,31 +179,41 @@ def _clone_into(
command='clone',
)
if flags.recurse_submodules:
- nest_submodules(git, flags, subdir, config.remote)
+ nest_submodules(git, flags, config, subdir)
else:
submodules.warn_if_any(git, subdir)
-def nest_submodules(git: GitRunner, flags: Flags, subdir: Path, remote: str, only: set[str] | None = None) -> None:
+def nest_submodules(git: GitRunner, flags: Flags, config: NestedConfig, subdir: Path) -> None:
"""Turn each submodule under subdir into a nested repository, at the commit it pins.
- Each one is cloned with -b , so it stays at that commit. With
- `only`, just the submodules at those paths.
+ Each one is cloned with -b , so it stays at that commit. The
+ submodules come from the tree subdir got its content from. Nested ones
+ are skipped.
"""
- for path, pinned in submodules.gitlinks(git, subdir):
- if only is not None and path not in only:
+ in_index = dict(submodules.gitlinks(git, subdir))
+ for relative, pinned in sorted(submodules.gitlinks_at(git, content.nested_tree(git, config, subdir)).items()):
+ path = f'{subdir.as_posix()}/{relative}'
+ if (Path(path) / GITNESTED_FILENAME).is_file() or ((config.filter or config.exclude) and path not in in_index):
continue
- url = submodules.url_of(git, subdir / '.gitmodules', path.removeprefix(f'{subdir.as_posix()}/'), remote)
+ url = submodules.url_of(git, subdir / '.gitmodules', relative, config.remote)
if url is None:
output.warn(f"{path}: .gitmodules has no url for it, so it stays empty")
continue
- git.run(['rm', '-q', '--cached', '--', path])
+ git.run(['rm', '-q', '--cached', '--', path], may_fail=True)
clone_at(git, flags, Path(path), url, pinned)
def clone_at(git: GitRunner, flags: Flags, subdir: Path, url: str, commit: str) -> None:
"""Clone `url` at `commit` into subdir, and its submodules too."""
- inner = Flags(branch=commit, recurse_submodules=True, quiet=flags.quiet, verbose=flags.verbose)
+ inner = Flags(
+ branch=commit,
+ recurse_submodules=True,
+ gpg_sign=flags.gpg_sign,
+ signoff=flags.signoff,
+ quiet=flags.quiet,
+ verbose=flags.verbose,
+ )
subdir, gitnested, subref, config = setup.setup_command(git, 'clone', inner, subdir, url)
do_clone(git, inner, config, subdir, gitnested, subref, git.check_output(['rev-parse', 'HEAD']))
output.say(f"{subdir.as_posix()}: cloned from {url} ({commit})")
diff --git a/src/git_nested/commands/commit.py b/src/git_nested/commands/commit.py
index 77c6ec9e..1134c19b 100644
--- a/src/git_nested/commands/commit.py
+++ b/src/git_nested/commands/commit.py
@@ -5,7 +5,7 @@
from .. import content, output, patches, refs, worktree
from ..cli import setup
from ..models import CommandContext
-from . import fetch
+from . import fetch, pull
def cmd_commit(ctx: CommandContext) -> None:
@@ -37,4 +37,5 @@ def cmd_commit(ctx: CommandContext) -> None:
# After a pull that stopped on a conflict, this commit finishes it
command=worktree.stopped(git, subdir_worktree) or 'commit',
)
+ pull.follow_inner(ctx, config, subdir)
output.say(f"{subdir.as_posix()}: committed {nested_commit_ref}")
diff --git a/src/git_nested/commands/pull.py b/src/git_nested/commands/pull.py
index 348b5ad8..cd8bf656 100644
--- a/src/git_nested/commands/pull.py
+++ b/src/git_nested/commands/pull.py
@@ -4,11 +4,14 @@
import os
import shutil
+from dataclasses import replace
from pathlib import Path
+from typing import cast
from uuid import uuid4
from .. import content, discovery, filters, gitfile, history, messages, output, patches, refs, submodules, worktree
from ..cli import setup
+from ..constants import GITNESTED_FILENAME
from ..errors import GitNestedError
from ..git import GitRunner
from ..models import CommandContext, Flags, NestedConfig
@@ -204,6 +207,8 @@ def _commit_without_squash(
git.run(['merge', '--no-commit', '--no-ff', '-s', 'ours', '--allow-unrelated-histories', rewritten])
content._place_content(git, config, subdir, nested_commit_ref)
gitfile.create_level_gitnested_files(git, subdir)
+ # After the level files: an inner nested repo of our own needs none
+ content.keep_inner_nested(git, flags, config, subdir, head_commit)
regular = gitnested.parent / '.gitnested'
for path in [gitnested] + ([regular] if regular != gitnested and regular.exists() else []):
@@ -306,16 +311,55 @@ def cmd_pull(ctx: CommandContext) -> None:
output.say(f"{subdir.as_posix()}: {verb} {config.remote} ({config.branch})")
return
- # Submodules of upstream that are nested now stay nested, at the commit upstream pins after the pull
- nested_before = {f'{subdir.as_posix()}/{path}' for path in history.gitlinked_inner(git, config, subdir)}
pulled = do_pull(git, flags, config, subdir, gitnested, ctx.tmp, subref)
if pulled is None:
+ # An inner nested repo may still lag behind, if an earlier pull stopped there
+ follow_inner(ctx, config, subdir)
output.say(f"{subdir.as_posix()}: already up to date with {config.remote} ({config.branch})")
return
branch, subdir_worktree = pulled
output.verbose(f"committing the new {branch} content")
_commit_pull(ctx, config, subdir, gitnested, subref, branch, subdir_worktree)
- clone.nest_submodules(git, flags, subdir, config.remote, None if flags.recurse_submodules else nested_before)
+ follow_inner(ctx, config, subdir)
+ if flags.recurse_submodules:
+ clone.nest_submodules(git, flags, config, subdir)
submodules.warn_if_any(git, subdir)
output.say(f"{subdir.as_posix()}: pulled from {config.remote} ({config.branch})")
+
+
+def follow_inner(ctx: CommandContext, config: NestedConfig, subdir: Path) -> None:
+ """Pull each inner nested repo to the commit that upstream pins now. Local changes stay.
+
+ If one stops on conflicts, a new pull of subdir does the rest.
+ """
+ git = ctx.git
+ for relative, pin in sorted(submodules.gitlinks_at(git, content.nested_tree(git, config, subdir)).items()):
+ path = subdir / relative
+ if not (path / GITNESTED_FILENAME).is_file():
+ continue
+ try:
+ pull_to(ctx, path, submodules.url_of(git, subdir / '.gitmodules', relative, config.remote), pin)
+ except GitNestedError as e:
+ raise GitNestedError(
+ f"{e.message.rstrip()}\n\nThen run 'git nested pull {subdir.as_posix()}' for the other inner repositories."
+ ) from None
+
+
+def pull_to(ctx: CommandContext, subdir: Path, url: str | None, pin: str) -> None:
+ """Pull subdir to `pin`, and from `url` if that is new. Local changes stay."""
+ current = NestedConfig.from_file(subdir / GITNESTED_FILENAME)
+ if current.commit == pin:
+ return
+ inner = Flags(
+ branch=pin,
+ remote=url if url and url != current.remote else None,
+ update=True,
+ recurse_submodules=True,
+ gpg_sign=ctx.flags.gpg_sign,
+ signoff=ctx.flags.signoff,
+ quiet=ctx.flags.quiet,
+ verbose=ctx.flags.verbose,
+ )
+ head = ctx.git.check_output(['rev-parse', 'HEAD'])
+ cmd_pull(cast('CommandContext', replace(ctx, flags=inner, subdir=subdir, head_commit=head)))
diff --git a/src/git_nested/content.py b/src/git_nested/content.py
index 12081f5a..30867384 100644
--- a/src/git_nested/content.py
+++ b/src/git_nested/content.py
@@ -6,7 +6,7 @@
from pathlib import Path, PurePosixPath
from typing import cast
-from . import discovery, filters, gitfile, history, output, patches, refs, worktree
+from . import discovery, filters, gitfile, history, output, patches, refs, submodules, worktree
from .constants import GITNESTED_FILENAME
from .errors import GitNestedError
from .git import GitRunner
@@ -79,6 +79,8 @@ def commit_nested_branch(
local = place_with_patches(git, config, subdir, nested_commit_ref)
gitfile.create_level_gitnested_files(git, subdir)
+ # After the level files: an inner nested repo of our own needs none
+ keep_inner_nested(git, flags, config, subdir, head_commit)
# A level file has the regular .gitnested next to it, which is kept in
# step so that operating on the nested repo directly still sees current info.
@@ -127,6 +129,40 @@ def place_with_patches(git: GitRunner, config: NestedConfig, subdir: Path, neste
return local
+def keep_inner_nested(git: GitRunner, flags: Flags, config: NestedConfig, subdir: Path, head: str | None) -> None:
+ """Do not commit a gitlink where a nested repo belongs.
+
+ An inner nested repo comes back from `head`, with its local changes.
+ Not with --force: that takes upstream as it is.
+ With --recurse-submodules a new submodule is left out. It is nested next.
+ """
+ drop_new = flags.recurse_submodules and not (config.filter or config.exclude)
+ for path, _ in submodules.gitlinks(git, subdir):
+ if not flags.force and _was_nested(git, head, path):
+ git.run(['update-index', '--force-remove', '--', path])
+ git.run(['checkout', cast('str', head), '--', path])
+ elif drop_new:
+ _drop_gitlink(git, Path(path))
+
+
+def _was_nested(git: GitRunner, head: str | None, path: str) -> bool:
+ return (
+ bool(head) and git.run(['cat-file', '-e', f'{head}:{path}/{GITNESTED_FILENAME}'], may_fail=True).returncode == 0
+ )
+
+
+def _drop_gitlink(git: GitRunner, path: Path) -> None:
+ git.run(['update-index', '--force-remove', '--', path])
+ if path.is_dir() and not any(path.iterdir()):
+ path.rmdir()
+
+
+def nested_tree(git: GitRunner, config: NestedConfig, subdir: Path) -> str:
+ """The tree that subdir got its content from in the last clone, pull or commit."""
+ tree = f'refs/nested/{refs.sanitize_subref(git, subdir.as_posix())}/commit'
+ return f'{tree}:{config.path}' if config.path else tree
+
+
def _place_content(git: GitRunner, config: NestedConfig, subdir: Path, nested_commit_ref: str) -> None:
"""Replace whatever subdir/ holds with the (filtered) tree of nested_commit_ref."""
if git.check_output(['ls-files', '--', subdir], may_fail=True):
diff --git a/src/git_nested/submodules.py b/src/git_nested/submodules.py
index 83c4162b..008ad44c 100644
--- a/src/git_nested/submodules.py
+++ b/src/git_nested/submodules.py
@@ -19,6 +19,16 @@ def gitlinks(git: GitRunner, subdir: Path) -> list[tuple[str, str]]:
return links
+def gitlinks_at(git: GitRunner, tree: str) -> dict[str, str]:
+ """The submodules in `tree`, as {path: pinned commit}. Empty if there is no such tree."""
+ links = {}
+ for line in git.check_output(['ls-tree', '-r', '-z', tree], may_fail=True).split('\0'):
+ meta, _, path = line.partition('\t')
+ if meta.startswith('160000 '):
+ links[path] = meta.split()[2]
+ return links
+
+
def entry(git: GitRunner, gitmodules: Path, path: str) -> tuple[str, str] | None:
"""The name and url `gitmodules` gives for the submodule at `path`, if any."""
if not gitmodules.is_file():
diff --git a/tests/e2e/test_submodules.py b/tests/e2e/test_submodules.py
index fbfbcece..6d9740b1 100644
--- a/tests/e2e/test_submodules.py
+++ b/tests/e2e/test_submodules.py
@@ -55,6 +55,16 @@ def test_init_from_submodule(foo_bar_cloned):
assert env.run(['git', 'status', '--porcelain'], cwd=foo).output == ''
+def test_clone_recurse_submodules_leaves_an_excluded_one_out(foo_bar_cloned):
+ env = foo_bar_cloned
+ foo = env.workspace / 'foo'
+ _bar_with_a_submodule(env)
+
+ cmd_git_nested(['clone', '--recurse-submodules', '--exclude', 'sub', (env.upstream / 'bar').as_posix()], foo)
+
+ assert not (foo / 'bar' / 'sub').exists()
+
+
def test_a_submodule_without_url_stays_empty(foo_bar_cloned):
"""A gitlink that .gitmodules does not describe cannot be nested; say so."""
env = foo_bar_cloned
@@ -121,3 +131,134 @@ def test_push_keeps_the_submodule_of_upstream(foo_bar_cloned):
entry = env.run(['git', 'ls-tree', 'master', 'sub'], cwd=env.upstream / 'bar').output.split()
assert entry[:3] == ['160000', 'commit', pinned]
+
+
+# ============================================================================
+# Inner nested repos keep their changes
+# ============================================================================
+
+
+def _nest_bar_with_sub(env):
+ """Clone bar with its submodule 'sub' into foo, all nested. Return foo."""
+ foo = env.workspace / 'foo'
+ _bar_with_a_submodule(env)
+ cmd_git_nested(['clone', '--recurse-submodules', (env.upstream / 'bar').as_posix()], foo)
+ return foo
+
+
+def _assert_no_gitlink(env, repo):
+ modes = {line.split()[0] for line in env.run(['git', 'ls-files', '-s'], cwd=repo).output.splitlines()}
+ assert '160000' not in modes
+
+
+def _resolve_in_worktree(env, repo, output, file, text):
+ """Fix a conflict where the help text says, and commit it there."""
+ worktree = repo / next(line.split('cd ', 1)[1] for line in output.splitlines() if '1. cd ' in line)
+ (worktree / file).write_text(text)
+ env.run(['git', 'add', file], cwd=worktree)
+ env.run(['git', 'commit', '-q', '--no-edit'], cwd=worktree)
+
+
+def test_clone_recurse_submodules_commits_no_gitlink(foo_bar_cloned):
+ env = foo_bar_cloned
+ foo = _nest_bar_with_sub(env)
+
+ for commit in env.run(['git', 'rev-list', 'HEAD'], cwd=foo).output.split():
+ tree = env.run(['git', 'ls-tree', '-r', commit], cwd=foo).output
+ assert '160000 commit' not in tree
+
+
+def test_pull_keeps_changes_in_an_inner_nested_repo(foo_bar_cloned):
+ env = foo_bar_cloned
+ foo = _nest_bar_with_sub(env)
+ env.modify_files('bar/sub/Foo', text='local', cwd=foo)
+ pinned = _bump_the_submodule(env)
+
+ cmd_git_nested('pull bar', cwd=foo)
+
+ assert 'local' in (foo / 'bar' / 'sub' / 'Foo').read_text()
+ assert (foo / 'bar' / 'sub' / 'FooNew').is_file()
+ assert_gitnested_field(foo / 'bar' / 'sub' / '.gitnested', commit=pinned)
+ _assert_no_gitlink(env, foo)
+
+
+def test_pull_no_squash_keeps_changes_in_an_inner_nested_repo(foo_bar_cloned):
+ env = foo_bar_cloned
+ foo = _nest_bar_with_sub(env)
+ env.modify_files('bar/sub/Foo', text='local', cwd=foo)
+ _bump_the_submodule(env)
+
+ cmd_git_nested('pull --no-squash bar', cwd=foo)
+
+ assert 'local' in (foo / 'bar' / 'sub' / 'Foo').read_text()
+ assert (foo / 'bar' / 'sub' / 'FooNew').is_file()
+ _assert_no_gitlink(env, foo)
+
+
+def test_pull_force_drops_changes_in_an_inner_nested_repo(foo_bar_cloned):
+ """--force takes upstream as it is. --recurse-submodules nests sub again."""
+ env = foo_bar_cloned
+ foo = _nest_bar_with_sub(env)
+ env.modify_files('bar/sub/Foo', text='local', cwd=foo)
+ pinned = _bump_the_submodule(env)
+
+ cmd_git_nested('pull --force --recurse-submodules bar', cwd=foo)
+
+ assert 'local' not in (foo / 'bar' / 'sub' / 'Foo').read_text()
+ assert_gitnested_field(foo / 'bar' / 'sub' / '.gitnested', commit=pinned)
+ _assert_no_gitlink(env, foo)
+
+
+def test_commit_after_a_conflict_keeps_inner_nested_repos(foo_bar_cloned):
+ """You finish a pull by hand. sub stays nested. It moves to the new pin."""
+ env = foo_bar_cloned
+ foo = _nest_bar_with_sub(env)
+ env.modify_files('bar/sub/Foo', text='local', cwd=foo)
+ env.modify_files('bar/Bar', text='mine', cwd=foo)
+ env.modify_files('Bar', text='theirs', cwd=env.workspace / 'bar')
+ env.run(['git', 'push', '-q'], cwd=env.workspace / 'bar')
+ pinned = _bump_the_submodule(env)
+
+ result = cmd_git_nested('pull bar', cwd=foo, check=False)
+ assert result.returncode == 1
+ _resolve_in_worktree(env, foo, result.output, 'Bar', 'both\n')
+ cmd_git_nested('commit bar', cwd=foo)
+
+ assert (foo / 'bar' / 'Bar').read_text() == 'both\n'
+ assert 'local' in (foo / 'bar' / 'sub' / 'Foo').read_text()
+ assert_gitnested_field(foo / 'bar' / 'sub' / '.gitnested', commit=pinned)
+ _assert_no_gitlink(env, foo)
+
+
+def test_pull_again_after_a_conflict_in_an_inner_nested_repo(foo_bar_cloned):
+ """The first inner repo stops on a conflict. A new pull updates the second one."""
+ env = foo_bar_cloned
+ bar = env.workspace / 'bar'
+ _bar_with_a_submodule(env)
+ env.run(['git', *ALLOW_FILE, 'submodule', 'add', '-q', (env.upstream / 'foo').as_posix(), 'sub2'], cwd=bar)
+ env.run(['git', 'commit', '-qm', 'add sub2'], cwd=bar)
+ env.run(['git', 'push', '-q'], cwd=bar)
+ foo = env.workspace / 'foo'
+ cmd_git_nested(['clone', '--recurse-submodules', (env.upstream / 'bar').as_posix()], foo)
+ env.modify_files('bar/sub/Foo', text='local', cwd=foo)
+ other = env.workspace / 'foo-other'
+ env.run(['git', 'clone', '-q', (env.upstream / 'foo').as_posix(), other])
+ env.modify_files('Foo', text='upstream', cwd=other)
+ env.run(['git', 'push', '-q'], cwd=other)
+ foo_upstream = git_rev_parse(['HEAD'], cwd=other)
+ for path in ('sub', 'sub2'):
+ env.run(['git', 'pull', '-q'], cwd=bar / path)
+ env.run(['git', 'commit', '-qam', 'bump both'], cwd=bar)
+ env.run(['git', 'push', '-q'], cwd=bar)
+
+ result = cmd_git_nested('pull bar', cwd=foo, check=False)
+
+ assert result.returncode == 1
+ assert "git nested pull bar" in result.output
+ _resolve_in_worktree(env, foo, result.output, 'Foo', 'both\n')
+ cmd_git_nested('commit bar/sub', cwd=foo)
+ cmd_git_nested('pull bar', cwd=foo)
+
+ assert_gitnested_field(foo / 'bar' / 'sub' / '.gitnested', commit=foo_upstream)
+ assert_gitnested_field(foo / 'bar' / 'sub2' / '.gitnested', commit=foo_upstream)
+ _assert_no_gitlink(env, foo)