Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions doc/architecture/diagrams.md
Original file line number Diff line number Diff line change
Expand Up @@ -349,8 +349,11 @@ sequenceDiagram
end
end

opt Submodules of upstream that were nested, or --recurse-submodules
Note over git-nested: Nest each at the commit upstream pins now (as in clone)
opt Submodules of upstream that are nested
Note over git-nested: Put them back from HEAD, with your changes.<br/>Then pull each to the commit upstream pins now
end
opt --recurse-submodules
Note over git-nested: Nest each new one at the commit upstream pins (as in clone)
end

Note over git-nested: The commit message lists the new upstream commits
Expand Down Expand Up @@ -691,6 +694,7 @@ sequenceDiagram
git-nested->>Git: ls-files -- <subdir>
git-nested->>Git: rm -r -- <subdir>
git-nested->>Git: read-tree --prefix=<subdir> -u <nested_commit_ref>
Note over git-nested: Inner nested repos: checkout HEAD -- <path>, not the gitlink
Note over git-nested: Write .gitnested
opt patches in use
git-nested->>Git: checkout HEAD -- <inner repos added here>
Expand All @@ -708,6 +712,10 @@ sequenceDiagram
Note over git-nested: Remove empty parent dirs up to .git/tmp
git-nested->>Git: update-ref refs/nested/<subref>/commit <nested_commit_ref>
end

opt Inner nested repos
Note over git-nested: Pull each to the commit upstream pins now (as in pull)
end
```

---
Expand Down
3 changes: 3 additions & 0 deletions doc/cli/commit.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ git nested commit --file=msg.txt ext/lib
it. Or pass `-F`.
- The branch must contain the fetched upstream commit. Otherwise upstream
changes would be lost. `--force` skips this check.
- Nested repositories inside `<subdir>` stay nested, with your changes.
Then each one is pulled to the commit that upstream pins now, as `pull`
does.

## See also

Expand Down
10 changes: 7 additions & 3 deletions doc/cli/pull.md
Original file line number Diff line number Diff line change
Expand Up @@ -138,9 +138,13 @@ This pull always rebases. Without `--pin`, a merge pull never goes back.
is recorded.
- A pin stays where it is on a normal pull. Only `--pin` moves it,
forward or back.
- Submodules of upstream that are nested already stay nested. Each one moves
to the commit upstream pins after the pull. `--recurse-submodules` also
nests new ones.
- Submodules of upstream that are nested already stay nested. Each one is
pulled to the commit upstream pins now. Your changes in it stay.
`--recurse-submodules` also nests new ones.
- If one of them stops on conflicts, finish it. Then run the same pull
again. It updates the other ones.
- With `--force`, nested repositories inside lose their changes too. See
[Upstream submodules](../guides/submodules.md).
- A worktree left from an earlier pull blocks a new one. Finish it, or run
[`clean`](clean.md).
- Once the nested repository has [patches](../configuration/gitnested-file.md#patches),
Expand Down
10 changes: 8 additions & 2 deletions doc/guides/submodules.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,14 @@ Each one stays at the commit upstream pins.

### Pull

`pull` keeps them nested. After the pull, each one moves to the commit
upstream pins now.
`pull` keeps them nested. Each one is pulled to the commit upstream pins
now. Your changes in it stay.

If one of them stops on conflicts, finish it as the message says. Then run
the same `pull` again. It updates the other ones.

`pull --force` takes upstream as it is. Inner nested repositories lose
their changes too. Add `--recurse-submodules` to nest them again.

If upstream adds a new submodule, a plain pull leaves it empty. To nest it
too:
Expand Down
29 changes: 20 additions & 9 deletions src/git_nested/commands/clone.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

from .. import content, discovery, gitfile, output, refs, submodules
from ..cli import setup
from ..constants import GITNESTED_FILENAME
from ..errors import GitNestedError
from ..git import GitRunner
from ..models import CommandContext, Flags, NestedConfig
Expand Down Expand Up @@ -178,31 +179,41 @@ def _clone_into(
command='clone',
)
if flags.recurse_submodules:
nest_submodules(git, flags, subdir, config.remote)
nest_submodules(git, flags, config, subdir)
else:
submodules.warn_if_any(git, subdir)


def nest_submodules(git: GitRunner, flags: Flags, subdir: Path, remote: str, only: set[str] | None = None) -> None:
def nest_submodules(git: GitRunner, flags: Flags, config: NestedConfig, subdir: Path) -> None:
"""Turn each submodule under subdir into a nested repository, at the commit it pins.

Each one is cloned with -b <commit>, so it stays at that commit. With
`only`, just the submodules at those paths.
Each one is cloned with -b <commit>, so it stays at that commit. The
submodules come from the tree subdir got its content from. Nested ones
are skipped.
"""
for path, pinned in submodules.gitlinks(git, subdir):
if only is not None and path not in only:
in_index = dict(submodules.gitlinks(git, subdir))
for relative, pinned in sorted(submodules.gitlinks_at(git, content.nested_tree(git, config, subdir)).items()):
path = f'{subdir.as_posix()}/{relative}'
if (Path(path) / GITNESTED_FILENAME).is_file() or ((config.filter or config.exclude) and path not in in_index):
continue
url = submodules.url_of(git, subdir / '.gitmodules', path.removeprefix(f'{subdir.as_posix()}/'), remote)
url = submodules.url_of(git, subdir / '.gitmodules', relative, config.remote)
if url is None:
output.warn(f"{path}: .gitmodules has no url for it, so it stays empty")
continue
git.run(['rm', '-q', '--cached', '--', path])
git.run(['rm', '-q', '--cached', '--', path], may_fail=True)
clone_at(git, flags, Path(path), url, pinned)


def clone_at(git: GitRunner, flags: Flags, subdir: Path, url: str, commit: str) -> None:
"""Clone `url` at `commit` into subdir, and its submodules too."""
inner = Flags(branch=commit, recurse_submodules=True, quiet=flags.quiet, verbose=flags.verbose)
inner = Flags(
branch=commit,
recurse_submodules=True,
gpg_sign=flags.gpg_sign,
signoff=flags.signoff,
quiet=flags.quiet,
verbose=flags.verbose,
)
subdir, gitnested, subref, config = setup.setup_command(git, 'clone', inner, subdir, url)
do_clone(git, inner, config, subdir, gitnested, subref, git.check_output(['rev-parse', 'HEAD']))
output.say(f"{subdir.as_posix()}: cloned from {url} ({commit})")
Expand Down
3 changes: 2 additions & 1 deletion src/git_nested/commands/commit.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from .. import content, output, patches, refs, worktree
from ..cli import setup
from ..models import CommandContext
from . import fetch
from . import fetch, pull


def cmd_commit(ctx: CommandContext) -> None:
Expand Down Expand Up @@ -37,4 +37,5 @@ def cmd_commit(ctx: CommandContext) -> None:
# After a pull that stopped on a conflict, this commit finishes it
command=worktree.stopped(git, subdir_worktree) or 'commit',
)
pull.follow_inner(ctx, config, subdir)
output.say(f"{subdir.as_posix()}: committed {nested_commit_ref}")
50 changes: 47 additions & 3 deletions src/git_nested/commands/pull.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,14 @@

import os
import shutil
from dataclasses import replace
from pathlib import Path
from typing import cast
from uuid import uuid4

from .. import content, discovery, filters, gitfile, history, messages, output, patches, refs, submodules, worktree
from ..cli import setup
from ..constants import GITNESTED_FILENAME
from ..errors import GitNestedError
from ..git import GitRunner
from ..models import CommandContext, Flags, NestedConfig
Expand Down Expand Up @@ -204,6 +207,8 @@ def _commit_without_squash(
git.run(['merge', '--no-commit', '--no-ff', '-s', 'ours', '--allow-unrelated-histories', rewritten])
content._place_content(git, config, subdir, nested_commit_ref)
gitfile.create_level_gitnested_files(git, subdir)
# After the level files: an inner nested repo of our own needs none
content.keep_inner_nested(git, flags, config, subdir, head_commit)

regular = gitnested.parent / '.gitnested'
for path in [gitnested] + ([regular] if regular != gitnested and regular.exists() else []):
Expand Down Expand Up @@ -306,16 +311,55 @@ def cmd_pull(ctx: CommandContext) -> None:
output.say(f"{subdir.as_posix()}: {verb} {config.remote} ({config.branch})")
return

# Submodules of upstream that are nested now stay nested, at the commit upstream pins after the pull
nested_before = {f'{subdir.as_posix()}/{path}' for path in history.gitlinked_inner(git, config, subdir)}
pulled = do_pull(git, flags, config, subdir, gitnested, ctx.tmp, subref)
if pulled is None:
# An inner nested repo may still lag behind, if an earlier pull stopped there
follow_inner(ctx, config, subdir)
output.say(f"{subdir.as_posix()}: already up to date with {config.remote} ({config.branch})")
return

branch, subdir_worktree = pulled
output.verbose(f"committing the new {branch} content")
_commit_pull(ctx, config, subdir, gitnested, subref, branch, subdir_worktree)
clone.nest_submodules(git, flags, subdir, config.remote, None if flags.recurse_submodules else nested_before)
follow_inner(ctx, config, subdir)
if flags.recurse_submodules:
clone.nest_submodules(git, flags, config, subdir)
submodules.warn_if_any(git, subdir)
output.say(f"{subdir.as_posix()}: pulled from {config.remote} ({config.branch})")


def follow_inner(ctx: CommandContext, config: NestedConfig, subdir: Path) -> None:
"""Pull each inner nested repo to the commit that upstream pins now. Local changes stay.

If one stops on conflicts, a new pull of subdir does the rest.
"""
git = ctx.git
for relative, pin in sorted(submodules.gitlinks_at(git, content.nested_tree(git, config, subdir)).items()):
path = subdir / relative
if not (path / GITNESTED_FILENAME).is_file():
continue
try:
pull_to(ctx, path, submodules.url_of(git, subdir / '.gitmodules', relative, config.remote), pin)
except GitNestedError as e:
raise GitNestedError(
f"{e.message.rstrip()}\n\nThen run 'git nested pull {subdir.as_posix()}' for the other inner repositories."
) from None


def pull_to(ctx: CommandContext, subdir: Path, url: str | None, pin: str) -> None:
"""Pull subdir to `pin`, and from `url` if that is new. Local changes stay."""
current = NestedConfig.from_file(subdir / GITNESTED_FILENAME)
if current.commit == pin:
return
inner = Flags(
branch=pin,
remote=url if url and url != current.remote else None,
update=True,
recurse_submodules=True,
gpg_sign=ctx.flags.gpg_sign,
signoff=ctx.flags.signoff,
quiet=ctx.flags.quiet,
verbose=ctx.flags.verbose,
)
head = ctx.git.check_output(['rev-parse', 'HEAD'])
cmd_pull(cast('CommandContext', replace(ctx, flags=inner, subdir=subdir, head_commit=head)))
38 changes: 37 additions & 1 deletion src/git_nested/content.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from pathlib import Path, PurePosixPath
from typing import cast

from . import discovery, filters, gitfile, history, output, patches, refs, worktree
from . import discovery, filters, gitfile, history, output, patches, refs, submodules, worktree
from .constants import GITNESTED_FILENAME
from .errors import GitNestedError
from .git import GitRunner
Expand Down Expand Up @@ -79,6 +79,8 @@ def commit_nested_branch(

local = place_with_patches(git, config, subdir, nested_commit_ref)
gitfile.create_level_gitnested_files(git, subdir)
# After the level files: an inner nested repo of our own needs none
keep_inner_nested(git, flags, config, subdir, head_commit)

# A level file has the regular .gitnested next to it, which is kept in
# step so that operating on the nested repo directly still sees current info.
Expand Down Expand Up @@ -127,6 +129,40 @@ def place_with_patches(git: GitRunner, config: NestedConfig, subdir: Path, neste
return local


def keep_inner_nested(git: GitRunner, flags: Flags, config: NestedConfig, subdir: Path, head: str | None) -> None:
"""Do not commit a gitlink where a nested repo belongs.

An inner nested repo comes back from `head`, with its local changes.
Not with --force: that takes upstream as it is.
With --recurse-submodules a new submodule is left out. It is nested next.
"""
drop_new = flags.recurse_submodules and not (config.filter or config.exclude)
for path, _ in submodules.gitlinks(git, subdir):
if not flags.force and _was_nested(git, head, path):
git.run(['update-index', '--force-remove', '--', path])
git.run(['checkout', cast('str', head), '--', path])
elif drop_new:
_drop_gitlink(git, Path(path))


def _was_nested(git: GitRunner, head: str | None, path: str) -> bool:
return (
bool(head) and git.run(['cat-file', '-e', f'{head}:{path}/{GITNESTED_FILENAME}'], may_fail=True).returncode == 0
)


def _drop_gitlink(git: GitRunner, path: Path) -> None:
git.run(['update-index', '--force-remove', '--', path])
if path.is_dir() and not any(path.iterdir()):
path.rmdir()


def nested_tree(git: GitRunner, config: NestedConfig, subdir: Path) -> str:
"""The tree that subdir got its content from in the last clone, pull or commit."""
tree = f'refs/nested/{refs.sanitize_subref(git, subdir.as_posix())}/commit'
return f'{tree}:{config.path}' if config.path else tree


def _place_content(git: GitRunner, config: NestedConfig, subdir: Path, nested_commit_ref: str) -> None:
"""Replace whatever subdir/ holds with the (filtered) tree of nested_commit_ref."""
if git.check_output(['ls-files', '--', subdir], may_fail=True):
Expand Down
10 changes: 10 additions & 0 deletions src/git_nested/submodules.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,16 @@ def gitlinks(git: GitRunner, subdir: Path) -> list[tuple[str, str]]:
return links


def gitlinks_at(git: GitRunner, tree: str) -> dict[str, str]:
"""The submodules in `tree`, as {path: pinned commit}. Empty if there is no such tree."""
links = {}
for line in git.check_output(['ls-tree', '-r', '-z', tree], may_fail=True).split('\0'):
meta, _, path = line.partition('\t')
if meta.startswith('160000 '):
links[path] = meta.split()[2]
return links


def entry(git: GitRunner, gitmodules: Path, path: str) -> tuple[str, str] | None:
"""The name and url `gitmodules` gives for the submodule at `path`, if any."""
if not gitmodules.is_file():
Expand Down
Loading
Loading