diff --git a/README.md b/README.md index bfd405d6..51938994 100644 --- a/README.md +++ b/README.md @@ -31,10 +31,12 @@ A small `.gitnested` file in the folder remembers the URL, the branch and the upstream commit. So the commands stay short. Your repository has submodules already? Turn them all into nested -repositories: +repositories, and keep following upstream: ```bash git nested submodules init --recursive +git fetch origin +git nested submodules update origin/main ``` ## Why git-nested? diff --git a/doc/architecture/diagrams.md b/doc/architecture/diagrams.md index 3a738e45..ed7c39f9 100644 --- a/doc/architecture/diagrams.md +++ b/doc/architecture/diagrams.md @@ -783,7 +783,50 @@ sequenceDiagram ## submodules -`init` runs `init --from-submodule` for each submodule. +`init` runs `init --from-submodule` for each submodule. `update` looks +like this: + +```mermaid +sequenceDiagram + participant User + participant git-nested + participant Git + + User->>git-nested: git nested submodules update + git-nested->>Git: merge-base --is-ancestor HEAD + Note over git-nested: Merged already: only update the nested repos that lag behind + opt --dry-run + Note over git-nested: Print the plan, also deeper down, then stop + end + git-nested->>Git: merge --no-commit --no-ff + Note over git-nested: Write .git/nested-submodules-update.json for --continue and --abort + git-nested->>Git: update-index --force-remove + Note over git-nested: Drop each gitlink and its part of .gitmodules + Note over git-nested: Merge .gitattributes without git-nested's block, then add the block again + git-nested->>Git: commit --no-edit + Note over git-nested: Stop on other conflicts. --continue goes on here + opt A submodule pull stopped before + Note over git-nested: --continue finishes it there, then runs commit + end + + loop For each submodule upstream changed + alt moved + git-nested->>Git: mv + else removed + git-nested->>Git: rm -r + else added + Note over git-nested: Same as clone. Not for update = none + else new pin + Note over git-nested: Same as pull -b -u. Skipped if it has a pin + end + git-nested->>Git: commit + end + + opt --single-commit + git-nested->>Git: reset --soft + git-nested->>Git: commit --amend + end +``` ## diff diff --git a/doc/cli/clone.md b/doc/cli/clone.md index 66cec6f4..8b8c50f5 100644 --- a/doc/cli/clone.md +++ b/doc/cli/clone.md @@ -114,7 +114,9 @@ If nothing changed, it does nothing and says "already up to date". - If your `.gitattributes` would change files of the nested repository, for example `*.bat text eol=crlf`, `clone` and `pull` add `-text` for these files to `.gitattributes`, in a commit of their own. So the files - stay as upstream has them. + stay as upstream has them, in git and in your checkout. +- With `--recurse-submodules`, a submodule with `update = none` in + `.gitmodules` stays a submodule, as in git. - If upstream is empty, `clone` stops. Use [`init`](init.md) and then [`push`](push.md) to fill it. - With [patches](../guides/patches.md), `--force` starts over: it takes diff --git a/doc/cli/index.md b/doc/cli/index.md index ff1b7967..f9691854 100644 --- a/doc/cli/index.md +++ b/doc/cli/index.md @@ -16,7 +16,7 @@ Each command has its own page. This page lists what they share. | [`push`](push.md) | push a nested repository's local changes upstream | | [`fetch`](fetch.md) | fetch upstream content for a nested repository | | [`status`](status.md) | report on the nested repositories of this repository | -| [`submodules`](submodules.md) | turn all submodules into nested repositories | +| [`submodules`](submodules.md) | turn all submodules into nested repositories and keep them up to date | | [`diff`](diff.md) | show a nested repository's local changes against upstream | | [`patches`](patches.md) | write, check, drop or cherry-pick the patches of a nested repository | | [`verify`](verify.md) | check that a nested repository is its upstream commit plus patches or diffs | @@ -117,7 +117,8 @@ The details are on each command's page. In short: | `-f`, `--force` | force the operation; what it does differs per command | | `-S`, `--gpg-sign` | GPG-sign the commits git-nested makes | | `--signoff` | add a `Signed-off-by:` trailer | -| `--single-commit` | with `--all`: one commit for all | +| `--single-commit` | with `--all`: one commit for all. With `submodules`: one merge commit | +| `--dry-run` | show what would happen; change nothing | ### `--all` and `-b` diff --git a/doc/cli/submodules.md b/doc/cli/submodules.md index 386ea737..e6d53369 100644 --- a/doc/cli/submodules.md +++ b/doc/cli/submodules.md @@ -1,9 +1,12 @@ # `git nested submodules` -Turn all submodules into nested repositories. +Turn all submodules into nested repositories and keep them up to date. ```text git nested submodules init [--recursive] [--resolve-relative ] [--only | --ignore ]... +git nested submodules update [] +git nested submodules update --continue +git nested submodules update --abort ``` ## What it does @@ -11,23 +14,53 @@ git nested submodules init [--recursive] [--resolve-relative ] [--only ` merges an upstream commit, for example `upstream/main`. + Then it changes the nested repositories the way upstream changed its + submodules. Each changed submodule gets one commit. It does not fetch. + If the commit is merged already, it only updates the nested + repositories that lag behind. -No gitlink and no `.gitmodules` entry stays for a nested submodule. +`update` handles these upstream changes: + +| Upstream did | `update` does | +|---|---| +| moved a pin | pull the nested repository to the new commit. Your changes stay | +| added a submodule | nest it. Its submodules too, unless a `.gitnested` says `submodules: keep` | +| removed a submodule | remove the nested repository. A warning says if you changed it | +| moved a submodule | move the folder. Your changes stay | +| changed a url | write the new `remote` to `.gitnested` | + +No gitlink and no `.gitmodules` entry comes back for a nested submodule. +Nested repositories that you added yourself are not touched. + +A submodule with `update = none` in `.gitmodules` stays a submodule, as in +git. git does not check those out either. + +## Arguments + +| Argument | Meaning | +|---|---| +| `` | `update`: what to merge. A branch like `upstream/main`, a tag, or a SHA. Fetch it first | ## Options | Option | Meaning | |---|---| -| `--only ` | nest only the submodules at or below this path. The others stay submodules. Give it more than once for more paths | -| `--ignore ` | nest all submodules but the ones at or below this path. Give it more than once for more paths. Not with `--only` | -| `--recursive` | nest the submodules of the submodules too, at any depth. Without it, they stay submodules. The choice goes into each `.gitnested` as [`submodules`](../configuration/gitnested-file.md) | -| `--resolve-relative ` | resolve relative submodule urls against this url. Without it, `init` stops at a relative url | +| `--only ` | `init`: nest only the submodules at or below this path. The others stay submodules. Give it more than once for more paths | +| `--ignore ` | `init`: nest all submodules but the ones at or below this path. Give it more than once for more paths. Not with `--only` | +| `--recursive` | `init`: nest the submodules of the submodules too, at any depth. Without it, they stay submodules. The choice goes into each `.gitnested` as [`submodules`](../configuration/gitnested-file.md), and `update` follows it | +| `--resolve-relative ` | `init`: resolve relative submodule urls against this url. Without it, `init` stops at a relative url | +| `--dry-run` | show what would change, also deeper down. Change nothing. It may fetch the commits it needs to look at | +| `--single-commit` | `update `: make one merge commit with all submodule changes in it | +| `--continue` | go on after a stop on conflicts | +| `--abort` | throw away an update that stopped on conflicts | | `-S`, `--gpg-sign` | GPG-sign the commits | | `--signoff` | add a `Signed-off-by:` trailer to the commits | @@ -37,6 +70,38 @@ No gitlink and no `.gitmodules` entry stays for a nested submodule. git clone https://github.com/user/project cd project git nested submodules init --recursive +git fetch origin +git nested submodules update origin/main +``` + +### A fork + +Your fork is `origin`. The project you forked is `upstream`: + +```bash +git remote add upstream https://github.com/org/project +git nested submodules init --recursive +git fetch upstream +git nested submodules update upstream/main +git push origin main +``` + +`update` takes a tag or a SHA too. For example, to follow releases: + +```bash +git fetch upstream --tags +git nested submodules update v2.0 +``` + +### Not with `git merge` or `git pull` + +Merge upstream with `update`, not with git itself. git can't put a gitlink +where a nested folder is, so a plain `git merge` stops, or brings gitlinks +into your history. If you started one, throw it away and use `update`: + +```bash +git merge --abort +git nested submodules update upstream/main ``` ### Relative urls @@ -50,7 +115,8 @@ git nested submodules init --resolve-relative https://github.com/org/project Now `../lib.git` is `https://github.com/org/lib.git`. This is only for the first level. With `--recursive`, a deeper relative url is relative to -the url of its parent submodule, as in git. +the url of its parent submodule, as in git. Later, `update` resolves new +relative urls against the remote of the commit it merges. ### Only some submodules @@ -61,7 +127,13 @@ git nested submodules init --only lib/core --only third_party ``` This nests `lib/core` and every submodule below `third_party`. The others -stay submodules. +stay submodules. `update` updates them as submodules. + +`init` keeps `--only` and `--ignore` in `.gitmodules`, in a `[nested]` +section. So `update` leaves a new submodule out too, if `init` would have +left it out. Upstream's changes to `.gitmodules` come in for the +submodules that stay. Your own changes there stay too. On a conflict +there, the conflict lines start with `#`. So git can still read the file. Or name the ones to keep as submodules: @@ -69,17 +141,98 @@ Or name the ones to keep as submodules: git nested submodules init --ignore vendor/big ``` -`init` keeps `--only` and `--ignore` in `.gitmodules`, in a `[nested]` -section. +### See first + +```bash +git nested submodules update --dry-run upstream/main +``` + +```text +would merge 3 commits of upstream/main +modules/core: would pull to 1a2b3c4d5e6f +modules/core/third_party/lib: would pull to 7d8e9f0a1b2c +docs: would nest at 9f8e7d6c5b4a +tools: would remove (your changes in it go too. They stay in the history) +``` + +### Conflicts + +A conflict outside the submodules stops the merge: + +```bash +git nested submodules update upstream/main +# fix the files, then +git add +git nested submodules update --continue +``` + +A conflict inside a submodule stops it too. The message says where: + +```text +modules/core: the update stopped on conflicts in src/main.c. +Fix them there, and 'git add' them: + + cd .git/tmp/nested/modules/core + +Then run 'git nested submodules update --continue'. It finishes modules/core and goes on with the others. +Or throw the update away: 'git nested submodules update --abort'. +``` + +To give up instead: + +```bash +git nested submodules update --abort +``` + +Nested repositories that were updated already stay updated. The stopped +one goes back. Run `git nested submodules update upstream/main` again to +update the rest. + +### Keep a submodule back + +Set a pin. Then `update` leaves it where it is, with a warning: + +```bash +git nested config modules/core pin 1a2b3c4 +``` + +To follow upstream again, remove the `pin` line from +`modules/core/.gitnested`. + +## History + +The merge keeps the upstream commits as they are. So they still have their +submodules: + +- `git checkout` of an upstream commit or tag shows empty submodule + folders. +- `git log -p` of an upstream commit shows the submodule changes as gitlink + changes. + +Your own commits never have a gitlink. Use them for builds and releases. To search for a bug with `git bisect`, add +`--first-parent`. + +By default, the merge commit comes first. One commit per submodule follows. +At the merge commit, the submodules are still at their old commits. If every +commit must build, use `--single-commit`. Its message lists the submodule +changes. ## Notes -- `init` needs a clean working tree. +- `init` and `update` need a clean working tree. `update` needs a branch, + not a detached HEAD. +- If upstream moves a submodule to a path you use already, it stays where + it is. A warning says so. +- If upstream adds a submodule at a path you use already, it is not nested. + A warning says so. - If your `.gitattributes` would change files of a nested repository, for example `*.bat text eol=crlf`, git-nested adds `-text` for these files to `.gitattributes`. So they stay as upstream has them, as in a submodule. + `update` keeps this block at the end of the file, also when upstream adds + rules there. - Push your changes to each submodule's upstream with [`push`](push.md), as for any nested repository. +- Changes to the upstream superproject itself are not pushed by this command. ## See also diff --git a/doc/configuration/gitnested-file.md b/doc/configuration/gitnested-file.md index 6333a512..18c136d7 100644 --- a/doc/configuration/gitnested-file.md +++ b/doc/configuration/gitnested-file.md @@ -201,8 +201,8 @@ What happens to the submodules inside the nested repository: - `nest`: they become nested repositories too. - `keep`: they stay submodules. -`submodules init` writes it: `keep`, or `nest` with `--recursive`. Unset -means `nest`. +`submodules init` writes it: `keep`, or `nest` with `--recursive`. Later, +`submodules update` follows it. Unset means `nest`. ### `cmdver` diff --git a/doc/guides/submodules.md b/doc/guides/submodules.md index 6a3ded4f..687031ad 100644 --- a/doc/guides/submodules.md +++ b/doc/guides/submodules.md @@ -57,14 +57,35 @@ repository. ## Turn a whole repository into one Your repository has many submodules, maybe submodules in submodules. You -want all of them as plain files: +want all of them as plain files, and still follow upstream: ```bash git nested submodules init --recursive +git fetch origin +git nested submodules update origin/main ``` `init --recursive` nests all of them, at any depth. Without `--recursive`, it nests only the first level. `--only ` nests only -some. +some. `update` merges upstream, keeps its commits, and +changes the nested repositories the way upstream changed its submodules. -See [`submodules`](../cli/submodules.md). +To see what `update` would do first: + +```bash +git nested submodules update --dry-run origin/main +``` + +In a fork, update from the project you forked: + +```bash +git nested submodules init --recursive +git fetch upstream +git nested submodules update upstream/main +``` + +Use `update` instead of `git merge` or `git pull`. A plain git merge +can't put a gitlink where a nested folder is. + +See [`submodules`](../cli/submodules.md). It also explains how the merged +history looks, and how to keep a submodule back. diff --git a/man/man1/git-nested.1 b/man/man1/git-nested.1 index bbb0610e..78110c06 100644 --- a/man/man1/git-nested.1 +++ b/man/man1/git-nested.1 @@ -100,7 +100,7 @@ run a git command in the worktree of a nested repository report on the nested repositories of this repository .TP .B submodules -turn all submodules into nested repositories +turn all submodules into nested repositories and keep them up to date .TP .B verify check that a nested repository is its upstream commit plus patches or diffs diff --git a/src/git_nested/cli/parser.py b/src/git_nested/cli/parser.py index e9364293..03dbd70b 100644 --- a/src/git_nested/cli/parser.py +++ b/src/git_nested/cli/parser.py @@ -86,6 +86,8 @@ def _flags_from_args(args: argparse.Namespace) -> Flags: squash=getattr(args, 'squash', False), update=getattr(args, 'update', False), keep=getattr(args, 'keep', False), + resume=getattr(args, 'resume', False), + abort=getattr(args, 'abort', False), quiet=getattr(args, 'quiet', False), # argparse's `count` action leaves the attribute at None when the flag # is absent, and the output layer wants a real count. @@ -179,7 +181,7 @@ def parse_args(git: GitRunner, args_list: list[str]) -> tuple[str, CommandContex tuple: (command, context). The context's git_tmp/head_commit are left unset -- only the caller knows whether the command runs in a repository. """ - args = build_arg_parser().parse_args(_options_last(args_list, 'patches')) + args = build_arg_parser().parse_args(_options_last(_options_last(args_list, 'patches'), 'submodules')) if args.version: args.command = 'version' @@ -188,7 +190,7 @@ def parse_args(git: GitRunner, args_list: list[str]) -> tuple[str, CommandContex flags = _flags_from_args(args) _check_flags(flags) - if flags.single_commit and not flags.all: + if flags.single_commit and not flags.all and args.command != 'submodules': output.usage_error("--single-commit needs --all") patches, verify_mode = _verify_files(args) @@ -214,5 +216,6 @@ def parse_args(git: GitRunner, args_list: list[str]) -> tuple[str, CommandContex patches=patches, verify_mode=verify_mode, submodules_action=getattr(args, 'submodules_action', None), + submodules_args=getattr(args, 'submodules_args', None), ) return args.command, context diff --git a/src/git_nested/cli/spec.py b/src/git_nested/cli/spec.py index d7d7db36..a1612a73 100644 --- a/src/git_nested/cli/spec.py +++ b/src/git_nested/cli/spec.py @@ -79,7 +79,18 @@ 'update', ], 'status': ['ALL', 'all', 'fetch', 'short'], - 'submodules': ['gpg_sign', 'ignore', 'only', 'recursive', 'resolve_relative', 'signoff'], + 'submodules': [ + 'abort', + 'dry_run', + 'gpg_sign', + 'ignore', + 'only', + 'recursive', + 'resolve_relative', + 'resume', + 'signoff', + 'single_commit', + ], 'verify': ['diffs', 'ignore', 'keep', 'output_dir', 'patches'], 'version': [], } @@ -102,7 +113,7 @@ 'repair': "point a nested repository's sync point back into the current history", 'run': "run a git command in the worktree of a nested repository", 'status': "report on the nested repositories of this repository", - 'submodules': "turn all submodules into nested repositories", + 'submodules': "turn all submodules into nested repositories and keep them up to date", 'verify': "check that a nested repository is its upstream commit plus patches or diffs", 'version': "print version information", } @@ -143,7 +154,11 @@ ('-c', '--commit'), {'action': 'store_true', 'help': "record the pushed commit in .gitnested; needs a push to the tracked branch"}, ), - ('dry_run', ('--dry-run',), {'action': 'store_true', 'dest': 'dry_run', 'help': "show what would be pushed"}), + ( + 'dry_run', + ('--dry-run',), + {'action': 'store_true', 'dest': 'dry_run', 'help': "show what would happen; change nothing"}, + ), ( 'diffs', ('--diffs',), @@ -259,6 +274,16 @@ 'help': "patches: folder for the missing patch; diff: file for the whole diff", }, ), + ( + 'abort', + ('--abort',), + {'action': 'store_true', 'help': "throw away a 'submodules update' that stopped on conflicts"}, + ), + ( + 'resume', + ('--continue',), + {'action': 'store_true', 'dest': 'resume', 'help': "finish a 'submodules update' that stopped on conflicts"}, + ), ('remote', ('-r', '--remote'), {'dest': 'remote', 'help': "upstream remote to push/pull/fetch"}), ( 'short', @@ -268,7 +293,7 @@ ( 'single_commit', ('--single-commit',), - {'action': 'store_true', 'help': "with --all: one commit for all nested repositories"}, + {'action': 'store_true', 'help': "one commit for all nested repositories"}, ), ( 'squash', @@ -355,7 +380,20 @@ # everything after goes to git untouched, options included. ('git_args', {'nargs': '...', 'help': "the git command to run, with its arguments"}), ], - 'submodules': [('submodules_action', {'metavar': 'action', 'choices': ['init'], 'help': "init"})], + 'submodules': [ + ( + 'submodules_action', + {'metavar': 'action', 'choices': ['init', 'update'], 'help': "init or update"}, + ), + ( + 'submodules_args', + { + 'metavar': 'arg', + 'nargs': '*', + 'help': "update: the commit to merge", + }, + ), + ], 'verify': [('subdir', {'help': _SUBDIR_HELP})], } @@ -453,7 +491,12 @@ "'init' nests all submodules at the commits they pin, one commit each. " "With --recursive, their submodules too. " "--only and --ignore pick some of them. " - "A relative submodule url needs --resolve-relative." + "A relative submodule url needs --resolve-relative. " + "'update' merges an upstream commit and keeps its commits. It does not fetch. " + "Then it changes the nested repositories the way upstream changed its submodules. " + "Each changed submodule gets one commit. " + "On conflicts, fix them and run --continue. Or give up with --abort. " + "--dry-run shows the changes and makes none." ), 'verify': ( "Apply the files to the upstream commit in .gitnested, in a temporary worktree, and compare " @@ -555,6 +598,10 @@ "git nested submodules init --recursive", "git nested submodules init --only lib/core", "git nested submodules init --resolve-relative https://github.com/org/project", + "git nested submodules update upstream/main", + "git nested submodules update --dry-run upstream/main", + "git nested submodules update v2.0", + "git nested submodules update --continue", ], 'verify': [ "git nested verify ext/lib --patches patches/lib/*.patch", diff --git a/src/git_nested/commands/clone.py b/src/git_nested/commands/clone.py index 0b6ac237..a05de16b 100644 --- a/src/git_nested/commands/clone.py +++ b/src/git_nested/commands/clone.py @@ -186,9 +186,11 @@ def _clone_into( def _warn_about_submodules(git: GitRunner, flags: Flags, subdir: Path) -> None: - """Say which submodules stay empty, and what nests them.""" + """Say which submodules stay empty, and why or what nests them.""" if flags.hint: submodules.warn_if_any(git, subdir, f"use {flags.hint} to nest them") + elif flags.submodules == 'keep': + submodules.warn_if_any(git, subdir, "its .gitnested says 'submodules: keep'") else: submodules.warn_if_any(git, subdir) @@ -198,19 +200,29 @@ def nest_submodules(git: GitRunner, flags: Flags, config: NestedConfig, subdir: Each one is cloned with -b , so it stays at that commit. The submodules come from the tree subdir got its content from. Nested ones - are skipped. + are skipped, and so are the ones git skips: `update = none`. """ in_index = dict(submodules.gitlinks(git, subdir)) for relative, pinned in sorted(submodules.gitlinks_at(git, content.nested_tree(config, subdir)).items()): path = f'{subdir.as_posix()}/{relative}' if (Path(path) / GITNESTED_FILENAME).is_file() or ((config.filter or config.exclude) and path not in in_index): continue - url = submodules.url_of(git, subdir / '.gitmodules', relative, config.remote) - if url is None: - output.warn(f"{path}: .gitmodules has no url for it, so it stays empty") - continue - git.run(['rm', '-q', '--cached', '--', path], may_fail=True) - clone_at(git, flags, Path(path), url, pinned) + url = _url_to_nest(git, config, subdir, relative) + if url: + git.run(['rm', '-q', '--cached', '--', path], may_fail=True) + clone_at(git, flags, Path(path), url, pinned) + + +def _url_to_nest(git: GitRunner, config: NestedConfig, subdir: Path, relative: str) -> str | None: + """The url to nest the submodule at `relative` from. None, with a word why, if it stays a submodule.""" + path = f'{subdir.as_posix()}/{relative}' + if submodules.skipped_by_git(git, subdir / '.gitmodules', relative): + output.say(f"{path}: stays a submodule, .gitmodules says 'update = none'") + return None + url = submodules.url_of(git, subdir / '.gitmodules', relative, config.remote) + if url is None: + output.warn(f"{path}: .gitmodules has no url for it, so it stays empty") + return url def clone_at(git: GitRunner, flags: Flags, subdir: Path, url: str, commit: str) -> None: diff --git a/src/git_nested/commands/submodules.py b/src/git_nested/commands/submodules.py index 35e96996..25c426ed 100644 --- a/src/git_nested/commands/submodules.py +++ b/src/git_nested/commands/submodules.py @@ -1,35 +1,92 @@ -"""Turn all submodules into nested repos. +"""Turn all submodules into nested repos, and keep them in step with upstream. The root repo stays a normal clone of the upstream superproject. Its submodules are nested repos. - init: nest every submodule, or the ones named, one commit each. +- update: merge an upstream commit, drop the gitlinks it brings, then + update the nested repos to match. One commit per nested repo. """ from __future__ import annotations +import json +import os import posixpath -from dataclasses import replace +import re +from collections.abc import Callable +from dataclasses import dataclass, replace from pathlib import Path from typing import cast -from .. import checks, output, submodules +from .. import checks, configio, content, output, refs, submodules, worktree +from ..constants import GITNESTED_FILENAME from ..errors import GitNestedError from ..git import GitRunner -from ..models import CommandContext, Flags -from . import init +from ..models import CommandContext, Flags, NestedConfig +from . import clone, commit, config, init, pull _GITMODULES = '.gitmodules' -# --only and --ignore, kept in .gitmodules +_GITATTRIBUTES = '.gitattributes' +# Where a merge that stopped keeps what --continue needs. Written once. +_STATE_FILE = 'nested-submodules-update.json' +_MAX_ARGS = {'init': 0, 'update': 1} +# What init left out, in .gitmodules: update leaves new submodules there out too _ONLY, _IGNORE = 'nested.only', 'nested.ignore' +# The lines git merge-file marks a conflict with +_CONFLICT_MARKER = re.compile(r'^(<{7}|={7}|>{7})', re.MULTILINE) +# A SHA-1 or SHA-256 commit id +_COMMIT_ID = re.compile(r'[0-9a-f]{40}|[0-9a-f]{64}') def cmd_submodules(ctx: CommandContext) -> None: - """Run `submodules init`.""" - if ctx.flags.only and ctx.flags.ignore: + """Run `submodules init` or `submodules update`.""" + action = ctx.submodules_action or '' + args = ctx.submodules_args or [] + _check_args(ctx.flags, action, args) + if action == 'init': + checks.check_worktree_clean(ctx.git, 'init') + _nest_all(ctx.git, ctx.flags) + else: + _update(ctx, args) + + +def _given(flags: Flags) -> list[str]: + """The options of update that were given.""" + options = { + '--continue': flags.resume, + '--abort': flags.abort, + '--dry-run': flags.dry_run, + '--single-commit': flags.single_commit, + } + return [name for name, on in options.items() if on] + + +def _check_args(flags: Flags, action: str, args: list[str]) -> None: + given = _given(flags) + if len(args) > _MAX_ARGS[action]: + output.usage_error(f"too many arguments for 'submodules {action}'") + _check_init_options(flags, action) + if action == 'init' and given: + output.usage_error(f"{given[0]} only works with 'submodules update'") + stop = next((name for name in given if name in ('--continue', '--abort')), None) + if stop and (args or len(given) > 1): + output.usage_error(f"{stop} takes no other arguments, and no --dry-run or --single-commit") + + +def _check_init_options(flags: Flags, action: str) -> None: + """The options of init: only with init, and not --only with --ignore.""" + if flags.only and flags.ignore: output.usage_error("--only and --ignore can't be used together") - checks.check_worktree_clean(ctx.git, 'init') - _nest_all(ctx.git, ctx.flags) + init_only = { + '--only': flags.only, + '--ignore': flags.ignore, + '--resolve-relative': flags.resolve_relative, + '--recursive': flags.recursive, + } + picked = next((name for name, value in init_only.items() if value), None) + if picked and action != 'init': + output.usage_error(f"{picked} only works with 'submodules init'") # ============================================================================ @@ -51,15 +108,18 @@ def _nest_all(git: GitRunner, flags: Flags) -> None: if not flags.resolve_relative: _refuse_relative_urls(git, links) _keep_selection(git, flags) - # .gitnested keeps the choice + # .gitnested keeps the choice, for update choice = cast('Flags', replace(flags, submodules='nest' if flags.recursive else 'keep', hint='--recursive')) for path in links: init.init_from_submodule(git, choice, Path(path), flags.resolve_relative) def _selected(git: GitRunner, flags: Flags) -> list[str]: - """The submodules that --only and --ignore leave in.""" - links = [path for path, _ in submodules.gitlinks(git, Path())] + """The submodules that --only and --ignore leave in. Not the ones git skips: `update = none`.""" + gitmodules = Path(_GITMODULES) + links = [ + path for path, _ in submodules.gitlinks(git, Path()) if not submodules.skipped_by_git(git, gitmodules, path) + ] if flags.only: return _below_any(links, flags.only) skipped = set(_below_any(links, flags.ignore)) if flags.ignore else set() @@ -83,6 +143,51 @@ def _add_selection(git: GitRunner, file: Path, selection: tuple[list[str], list[ git.run(['config', '-f', file, '--add', key, path]) +def _selection_at(git: GitRunner, rev: str) -> tuple[list[str], list[str]]: + """The --only and --ignore paths that .gitmodules of `rev` keeps.""" + text = git.check_output(['show', f'{rev}:{_GITMODULES}'], may_fail=True) + return cast( + 'tuple[list[str], list[str]]', + tuple( + git.check_output(['config', '-f', '-', '--get-all', key], may_fail=True, input=text).split('\n') + if text + else [] + for key in (_ONLY, _IGNORE) + ), + ) + + +def _left_out(path: str, only: list[str], ignore: list[str]) -> bool: + """Whether init's --only or --ignore leaves `path` out.""" + only, ignore = [p for p in only if p], [p for p in ignore if p] + return (bool(only) and not any(_below(path, p) for p in only)) or any(_below(path, p) for p in ignore) + + +def _kept(git: GitRunner, ours: str, target: str) -> set[str]: + """The submodules of `target` that stay submodules. + + The ones `ours` has as a submodule, also under a new path, and also + inside nested repos. The ones git skips: `update = none`. And new ones + that init's --only or --ignore leaves out. + """ + ours_links = submodules.gitlinks_at(git, ours) + names = {name for path, (name, _) in _modules_at(git, ours).items() if path in ours_links} + selection = _selection_at(git, ours) + target_modules = _modules_at(git, target) + kept = set(ours_links) | _skipped_by_git_at(git, target) + for path in submodules.gitlinks_at(git, target): + if _stays(path, target_modules.get(path, ('', ''))[0], names, selection): + kept.add(path) + return kept + + +def _stays(path: str, name: str, names: set[str], selection: tuple[list[str], list[str]]) -> bool: + """Whether the submodule at `path` stays one: its name is in `names`, or init left it out.""" + if name and name in names: + return True + return not _is_nested(path) and _left_out(path, *selection) + + def _refuse_relative_urls(git: GitRunner, links: list[str]) -> None: """Fail if a submodule in `links` has a relative url. @@ -114,3 +219,667 @@ def _below_any(links: list[str], paths: list[str]) -> list[str]: def _below(link: str, path: str) -> bool: """Whether `link` is `path` or below it.""" return path in ('.', link) or link.startswith(f'{path}/') + + +# ============================================================================ +# update +# ============================================================================ + + +def _update(ctx: CommandContext, args: list[str]) -> None: + git = ctx.git + state_file = Path(git.check_output(['rev-parse', '--git-dir'])) / _STATE_FILE + if ctx.flags.abort: + _abort(ctx, state_file) + return + state = _resume(git, state_file) if ctx.flags.resume else _start(ctx, state_file, args) + if state is None: + return + _finish_merge(git, ctx.flags) + _run_steps(ctx, state) + if state['single']: + _fold_into_merge(git, ctx.flags, _merge_commit(git, state['target'])) + state_file.unlink() + + +def _run_steps(ctx: CommandContext, state: dict[str, str]) -> None: + """Do the plan. With --continue, first finish a submodule pull that stopped.""" + # Each step makes its own commit. --single-commit folds them in at the end. + steps_ctx = cast('CommandContext', replace(ctx, flags=replace(ctx.flags, single_commit=False))) + for step in _plan(ctx.git, state): + if step.what == 'pull' and ctx.flags.resume: + _finish_pending(steps_ctx, step.path) + _APPLY[step.what](steps_ctx, step) + + +def _merge_commit(git: GitRunner, target: str) -> str: + """The merge commit of `target` on this branch.""" + log = git.check_output(['log', '--first-parent', '--merges', '--format=%H %P', f'{target}..HEAD']) + for line in log.splitlines(): + commit_id, *parents = line.split() + if target in parents[1:]: + return commit_id + raise GitNestedError(f"the merge commit of {target} is gone") + + +def _abort(ctx: CommandContext, state_file: Path) -> None: + """Throw away a merge that stopped, and a nested repo's pull that stopped. Nested repos that it updated stay.""" + git = ctx.git + if not state_file.is_file(): + raise GitNestedError("there is nothing to abort") + if _merging(git): + git.run(['merge', '--abort']) + _drop_stopped_pulls(git, ctx.tmp) + state_file.unlink() + output.say( + "aborted. The nested repos that were updated stay. 'git nested submodules update ' updates the rest." + ) + + +def _drop_stopped_pulls(git: GitRunner, tmp: Path) -> None: + """Remove the worktrees and branches of the nested repos whose pull stopped on conflicts.""" + for path, _ in worktree.list_worktrees(git): + try: + branch = path.resolve().relative_to(tmp.resolve()).as_posix() + except ValueError: + continue + if worktree.stopped(git, path): + worktree.delete_branch(git, branch, tmp) + + +def _resume(git: GitRunner, state_file: Path) -> dict[str, str]: + """The state of the merge that stopped. The merge must still be there, or be done.""" + if not state_file.is_file(): + raise GitNestedError("there is nothing to continue") + state = _load(state_file) + if not _merging(git) and not _is_ancestor(git, state['target'], 'HEAD'): + raise GitNestedError("The merge of upstream is gone. Start over with 'git nested submodules update --abort'.") + return state + + +def _load(state_file: Path) -> dict[str, str]: + """Read the state file, and check what it says.""" + broken = GitNestedError(f"{state_file.as_posix()} is broken. Start over with --abort.") + try: + raw = json.loads(state_file.read_text()) + state = { + 'base': str(raw['base']), + 'target': str(raw['target']), + 'url': str(raw['url']), + 'single': 'yes' if raw.get('single') else '', + } + except (ValueError, KeyError, TypeError): + raise broken from None + if not all(_COMMIT_ID.fullmatch(state[key]) for key in ('base', 'target')): + raise broken + return state + + +def _start(ctx: CommandContext, state_file: Path, args: list[str]) -> dict[str, str] | None: + """Start the merge. None if there is nothing to merge, or on --dry-run.""" + git = ctx.git + if state_file.is_file(): + raise GitNestedError( + "A 'submodules update' stopped before. Finish it with 'git nested submodules update --continue'." + ) + if not args: + _needs_a_commit(git) + checks.check_worktree_clean(git, 'update') + ref, remote_url, label = _resolve(git, args) + target = git.check_output(['rev-parse', f'{ref}^{{commit}}']) + if _is_ancestor(git, target, 'HEAD'): + return _catch_up(ctx, state_file, {'base': target, 'target': target, 'url': remote_url, 'single': ''}, label) + base = git.check_output(['merge-base', 'HEAD', target], may_fail=True) + if not base: + raise GitNestedError(f"{label} has no history in common with this branch") + state = { + 'base': base, + 'target': target, + 'url': remote_url, + 'single': 'yes' if ctx.flags.single_commit else '', + } + if ctx.flags.dry_run: + count = git.check_output(['rev-list', '--count', f'HEAD..{target}']) + _show_plan(git, state, f"would merge {count} commits of {label}") + return None + _start_merge(git, ref) + # Only now: --continue needs a merge to continue + state_file.write_text(json.dumps(state)) + _drop_gitlinks(git, state) + _merge_gitattributes(git, state) + return state + + +def _catch_up(ctx: CommandContext, state_file: Path, state: dict[str, str], label: str) -> dict[str, str] | None: + """The target is merged. Bring the nested repos to its gitlinks, if some are not there yet. + + For example after --abort. Only for the upstream commit of the last + merge, so an older one does not take the nested repos back. + """ + git = ctx.git + if ctx.flags.single_commit: + output.usage_error("--single-commit only works when 'submodules update' merges") + last = git.check_output(['log', '-1', '--first-parent', '--merges', '--format=%P', 'HEAD']).split() + if state['target'] not in last[1:] or all(step.what == 'skip' for step in _plan(git, state)): + output.say(f"already up to date with {label}") + return None + if ctx.flags.dry_run: + _show_plan(git, state, f"would update the nested repos to {label}") + return None + state_file.write_text(json.dumps(state)) + return state + + +def _needs_a_commit(git: GitRunner) -> None: + """Fail: update needs a commit. Say what to do about a git merge that is in progress.""" + if _merging(git): + raise GitNestedError( + "a git merge is in progress. Run 'git merge --abort'. " + "Then run 'git nested submodules update ' instead." + ) + output.usage_error("'submodules update' needs a commit, for example upstream/main") + + +def _current_branch(git: GitRunner) -> str: + branch = git.check_output(['symbolic-ref', '--short', '-q', 'HEAD'], may_fail=True) + if not branch: + raise GitNestedError("HEAD is detached. Check out a branch first.") + return branch + + +def _resolve(git: GitRunner, args: list[str]) -> tuple[str, str, str]: + """The commit to merge, as given. Return it, the url of its remote, and a name to show.""" + _current_branch(git) + ref = args[0] + if not _is_commit(git, ref): + raise GitNestedError(f"{ref}: no such commit. Fetch it first.") + return ref, _remote_url_of(git, ref), ref + + +def _remote_url_of(git: GitRunner, ref: str) -> str: + """The url of the remote that `ref` comes from. Else of the current branch's remote, else of origin.""" + full = git.check_output(['rev-parse', '--symbolic-full-name', ref], may_fail=True) + remote = '' + if full.startswith('refs/remotes/'): + rest = full.removeprefix('refs/remotes/') + # A remote name may hold a slash. The longest match wins. + names = sorted(git.check_output(['remote']).split(), key=len, reverse=True) + remote = next((name for name in names if rest.startswith(f'{name}/')), '') + if not remote: + branch = _current_branch(git) + remote = git.check_output(['config', f'branch.{branch}.remote'], may_fail=True) or 'origin' + return git.check_output(['remote', 'get-url', remote], may_fail=True) + + +def _is_commit(git: GitRunner, ref: str) -> bool: + return git.run(['rev-parse', '-q', '--verify', f'{ref}^{{commit}}'], may_fail=True).returncode == 0 + + +def _is_ancestor(git: GitRunner, commit: str, of: str) -> bool: + return git.run(['merge-base', '--is-ancestor', commit, of], may_fail=True).returncode == 0 + + +def _merging(git: GitRunner) -> bool: + return git.run(['rev-parse', '-q', '--verify', 'MERGE_HEAD'], may_fail=True).returncode == 0 + + +def _start_merge(git: GitRunner, ref: str) -> None: + result = git.run(['merge', '-q', '--no-commit', '--no-ff', ref], may_fail=True) + if not _merging(git): + raise GitNestedError(f"the merge of {ref} failed:\n{result.stdout}{result.stderr}") + + +def _drop_gitlinks(git: GitRunner, state: dict[str, str]) -> None: + """Keep the nested folders. Drop the gitlinks that the merge brought. + + The kept submodules stay, see _kept. .gitmodules is merged for them only. + """ + kept = _kept(git, 'HEAD', state['target']) + for entry in git.check_output(['ls-files', '-s', '-z']).split('\0'): + meta, _, path = entry.partition('\t') + if meta.startswith('160000 ') and path not in kept: + git.run(['update-index', '--force-remove', '--', path]) + _remove_if_empty(Path(path)) + _merge_gitmodules(git, state, kept) + + +def _merge_gitmodules(git: GitRunner, state: dict[str, str], kept: set[str]) -> None: + """Merge .gitmodules of base, HEAD and target, with the kept submodules only. + + So upstream's changes to them come in, and yours stay. A conflict stays + in the file, like any merge conflict. + """ + names = {name for path, (name, _) in _modules_at(git, state['target']).items() if path in kept} + names |= {name for name, _ in _modules_at(git, 'HEAD').values()} + selection = _selection_at(git, 'HEAD') + scratch = Path(git.check_output(['rev-parse', '--git-dir'])) / 'nested-gitmodules' + scratch.mkdir(exist_ok=True) + files = [ + _kept_modules(git, rev, names, selection, scratch / rev) for rev in ('HEAD', state['base'], state['target']) + ] + merged = git.run(['merge-file', '-p', '-L', 'HEAD', '-L', 'base', '-L', 'upstream', *files], may_fail=True) + git.run(['update-index', '--force-remove', '--', _GITMODULES]) + Path(_GITMODULES).unlink(missing_ok=True) + if merged.stdout.strip(): + # Markers as comments: git reads .gitmodules as config, and a marker breaks every git command + text = _CONFLICT_MARKER.sub(r'# \1', merged.stdout) + Path(_GITMODULES).write_text(text, encoding='utf-8', newline='\n') + _stage_merged(git, files, conflict=merged.returncode != 0) + for file in files: + Path(file).unlink() + + +def _stage_merged(git: GitRunner, files: list[str], conflict: bool) -> None: + """Stage the merged .gitmodules. A conflict gets stages 1-3, so git status and --continue know it. + + `files` are the versions of HEAD, base and target. + """ + if not conflict: + git.run(['add', '--', _GITMODULES]) + return + blobs = [git.check_output(['hash-object', '-w', file]) for file in (files[1], files[0], files[2])] + # -z: on Windows, text mode turns \n into \r\n, and git would read the path as '.gitmodules\r' + info = ''.join(f'100644 {blob} {stage}\t{_GITMODULES}\0' for stage, blob in enumerate(blobs, start=1)) + git.run(['update-index', '-z', '--index-info'], input=info) + + +def _kept_modules(git: GitRunner, rev: str, names: set[str], selection: tuple[list[str], list[str]], file: Path) -> str: + """Write .gitmodules of `rev` to `file`, with only the submodules in `names`, and `selection`.""" + text = git.check_output(['show', f'{rev}:{_GITMODULES}'], may_fail=True) + file.write_text(f'{text}\n' if text else '', encoding='utf-8', newline='\n') + others = {name for name, _ in _modules_at(git, rev).values()} - names + for section in [*(f'submodule.{name}' for name in others), 'nested']: + git.run(['config', '-f', file, '--remove-section', section], may_fail=True) + # No section left: the same empty file for each side, so they do not differ by a blank line + if not file.read_text(encoding='utf-8').strip(): + file.write_text('', encoding='utf-8') + _add_selection(git, file, selection) + return file.as_posix() + + +def _merge_gitattributes(git: GitRunner, state: dict[str, str]) -> None: + """Merge .gitattributes again without git-nested's block, then put the block back at the end. + + git-nested owns the block. Upstream adds its rules at the end too, so + a plain merge stops there. A real conflict in the other rules stays. + """ + if _GITATTRIBUTES not in _conflicts(git): + return + files, block = _attributes_without_block(git, state) + merged = git.run(['merge-file', '-p', *files], may_fail=True) + for file in files: + Path(file).unlink() + if merged.returncode == 0: + rules = merged.stdout if merged.stdout.endswith('\n') or not merged.stdout else f'{merged.stdout}\n' + Path(_GITATTRIBUTES).write_text(rules + block, encoding='utf-8', newline='\n') + git.run(['add', '--', _GITATTRIBUTES]) + + +def _attributes_without_block(git: GitRunner, state: dict[str, str]) -> tuple[list[str], str]: + """.gitattributes of HEAD, base and target without git-nested's block, in files. Also the block of HEAD.""" + scratch = Path(git.check_output(['rev-parse', '--git-dir'])) / 'nested-gitattributes' + scratch.mkdir(exist_ok=True) + files, blocks = [], [] + for rev in ('HEAD', state['base'], state['target']): + text = git.check_output(['show', f'{rev}:{_GITATTRIBUTES}'], may_fail=True) + rest, block = content.split_keep_block(f'{text}\n' if text else '') + (scratch / rev).write_text(rest, encoding='utf-8', newline='\n') + files.append((scratch / rev).as_posix()) + blocks.append(block) + return files, blocks[0] + + +def _conflicts(git: GitRunner, cwd: Path | None = None) -> list[str]: + """The files with conflicts left, in the worktree at `cwd` or here.""" + return sorted(set(git.check_output(['diff', '--name-only', '--diff-filter=U'], cwd=cwd).split('\n')) - {''}) + + +def _remove_if_empty(path: Path) -> None: + """A gitlink leaves an empty folder behind.""" + if path.is_dir() and not any(path.iterdir()): + path.rmdir() + + +def _finish_merge(git: GitRunner, flags: Flags) -> None: + """Commit the merge. Stop if there are conflicts left.""" + if not _merging(git): + return + conflicts = _conflicts(git) + if conflicts: + raise GitNestedError( + f"the merge has conflicts in: {', '.join(conflicts)}\n" + "Fix them and 'git add' them. Then run 'git nested submodules update --continue'." + ) + git.run(['commit', '-q', '--no-edit', *_signing(flags)]) + + +def _signing(flags: Flags) -> list[str]: + return [*(['--signoff'] if flags.signoff else []), *(['-S'] if flags.gpg_sign else [])] + + +def _fold_into_merge(git: GitRunner, flags: Flags, merge: str) -> None: + """Put the submodule commits into the merge commit. Its message lists them.""" + if git.check_output(['rev-parse', 'HEAD']) == merge: + return + message = git.check_output(['log', '-1', '--format=%B', merge]) + changes = git.check_output(['log', '--reverse', '--format=- %s', f'{merge}..HEAD']) + git.run(['reset', '-q', '--soft', merge]) + git.run(['commit', '-q', '--amend', '--no-verify', '-F', '-', *_signing(flags)], input=f"{message}\n\n{changes}\n") + + +def _finish_stopped(git: GitRunner, path: str, tree: Path) -> None: + """Finish the merge, `git am` or rebase that stopped in the worktree of `path`.""" + if git.run(['rev-parse', '-q', '--verify', 'MERGE_HEAD'], cwd=tree, may_fail=True).returncode == 0: + finish = ['commit', '-q', '--no-edit'] + elif Path(tree, git.check_output(['rev-parse', '--git-path', 'rebase-apply'], cwd=tree)).is_dir(): + # A pull with patches applies them with git am + finish = ['am', '--continue'] + elif Path(tree, git.check_output(['rev-parse', '--git-path', 'rebase-merge'], cwd=tree)).is_dir(): + finish = ['rebase', '--continue'] + else: + return + left = _conflicts(git, tree) + if left: + raise GitNestedError(f"{path}: fix the conflicts in {tree.as_posix()} first: {', '.join(left)}") + git.run(['-c', 'core.editor=true', *finish], cwd=tree) + + +def _finish_pending(ctx: CommandContext, path: str) -> None: + """Commit the submodule pull that stopped on conflicts, if you have not done it yet.""" + git = ctx.git + branch = refs.nested_branch(git, refs.sanitize_subref(path)) + # A rebase that stopped has no branch checked out + tree = worktree.find_worktree(git, branch) or ctx.tmp / branch + if not tree.is_dir(): + return + _finish_stopped(git, path, tree) + inner = Flags( + gpg_sign=ctx.flags.gpg_sign, signoff=ctx.flags.signoff, quiet=ctx.flags.quiet, verbose=ctx.flags.verbose + ) + head = git.check_output(['rev-parse', 'HEAD']) + commit.cmd_commit( + cast( + 'CommandContext', + replace(ctx, flags=inner, subdir=Path(path), upstream=None, nested_commit_ref=None, head_commit=head), + ) + ) + + +# ============================================================================ +# The plan: what changes in the nested repos +# ============================================================================ + + +@dataclass +class _Step: + """One change to one nested repo. + + `what` is move, remove, nest, pull, url or skip. `to` is the new path of + a move, or the pin of a nest or pull. `url` is the url to nest from, or + a new url. `note` is the reason to skip, or a warning. + """ + + what: str + path: str + to: str = '' + url: str | None = None + note: str = '' + + def text(self) -> str: + """What the step does, in a few words.""" + return { + 'move': f"move to {self.to}", + 'remove': "remove", + 'nest': f"nest at {self.to[:12]}", + 'pull': f"pull to {self.to[:12]}" + (f" from {self.url}" if self.url else ''), + 'url': f"set remote to {self.url}", + 'skip': "stay as it is", + }[self.what] + + +def _module_fields(git: GitRunner, commit: str) -> dict[str, dict[str, str]]: + """The .gitmodules of `commit`, as {name: {field: value}}, with path, url and update.""" + text = git.check_output(['show', f'{commit}:{_GITMODULES}'], may_fail=True) + fields: dict[str, dict[str, str]] = {} + lines = git.check_output( + ['config', '-f', '-', '--get-regexp', r'^submodule\..*\.(path|url|update)$'], may_fail=True, input=text + ) + for line in lines.splitlines(): + key, _, value = line.partition(' ') + name, _, field = key.removeprefix('submodule.').rpartition('.') + fields.setdefault(name, {})[field] = value + return fields + + +def _modules_at(git: GitRunner, commit: str) -> dict[str, tuple[str, str]]: + """The .gitmodules of `commit`, as {path: (name, url)}.""" + fields = _module_fields(git, commit) + return {f['path']: (name, f['url']) for name, f in fields.items() if 'path' in f and 'url' in f} + + +def _skipped_by_git_at(git: GitRunner, commit: str) -> set[str]: + """The submodules that .gitmodules of `commit` gives `update = none`. git does not check those out.""" + return {f['path'] for f in _module_fields(git, commit).values() if 'path' in f and f.get('update') == 'none'} + + +def _is_nested(path: str | Path) -> bool: + return (Path(path) / GITNESTED_FILENAME).is_file() + + +def _plan(git: GitRunner, state: dict[str, str]) -> list[_Step]: + """What to do so that the nested repos match the submodules of the target.""" + old, new = submodules.gitlinks_at(git, state['base']), submodules.gitlinks_at(git, state['target']) + old_modules, new_modules = _modules_at(git, state['base']), _modules_at(git, state['target']) + steps, moved = _plan_gone(git, old, new, old_modules, new_modules) + kept = _kept(git, 'HEAD', state['target']) + old_url_of = dict(old_modules.values()) + for path, pin in sorted(new.items()): + if path not in kept: + steps.append(_plan_one(path, pin, new_modules.get(path, ('', '')), old_url_of, state, moved)) + return steps + + +def _plan_gone( + git: GitRunner, + old: dict[str, str], + new: dict[str, str], + old_modules: dict[str, tuple[str, str]], + new_modules: dict[str, tuple[str, str]], +) -> tuple[list[_Step], dict[str, str]]: + """What to do for the nested repos that upstream moved or removed. Also {new path: old path}.""" + # A move keeps the submodule's name in .gitmodules + new_path_of = {name: path for path, (name, _) in new_modules.items() if path not in old} + steps = [] + moved: dict[str, str] = {} + for path in sorted(p for p in set(old) - set(new) if _is_nested(p)): + step = _plan_gone_one(git, path, new_path_of.get(old_modules.get(path, ('', ''))[0])) + if step.what == 'move': + moved[step.to] = path + steps.append(step) + return steps, moved + + +def _plan_gone_one(git: GitRunner, path: str, to: str | None) -> _Step: + """Move the nested repo at `path` to `to`, or remove it if there is no `to`.""" + if to and Path(to).exists(): + return _Step('skip', path, note=f"upstream moved it to {to}, but that path is taken. It stays") + if to: + return _Step('move', path, to=to) + note = "your changes in it go too. They stay in the history" if _has_local_changes(git, path) else '' + return _Step('remove', path, note=note) + + +def _plan_one( + path: str, + pin: str, + module: tuple[str, str], + old_url_of: dict[str, str], + state: dict[str, str], + moved: dict[str, str], +) -> _Step: + """What to do for the submodule at `path` that upstream pins at `pin`.""" + name, url = module + if not url: + return _Step('skip', path, note=".gitmodules has no url for it, so it stays as it is") + resolved = submodules.resolve(url, state['url']) + # Compare urls as .gitmodules has them. Resolved, a relative url depends on the remote. + new_url = resolved if old_url_of.get(name, url) != url else None + here = Path(moved.get(path, path)) + if not _is_nested(here): + if here.is_dir() and any(here.iterdir()): + return _Step('skip', path, note="the path is taken, so the submodule stays empty") + return _Step('nest', path, to=pin, url=resolved) + return _plan_nested(path, pin, here, new_url) + + +def _plan_nested(path: str, pin: str, here: Path, new_url: str | None) -> _Step: + """What to do for a submodule that is nested already, at `here`.""" + current = NestedConfig.from_file(here / GITNESTED_FILENAME) + if current.pin and current.commit != pin: + return _Step('skip', path, note=f"it has the pin {current.pin}, so it stays. Upstream pins {pin[:12]}") + if current.commit != pin: + return _Step('pull', path, to=pin, url=new_url) + if new_url and current.remote != new_url: + return _Step('url', path, url=new_url) + return _Step('skip', path, note='') + + +def _has_local_changes(git: GitRunner, path: str) -> bool: + """Whether path differs from the upstream commit it records. False if that can't be told.""" + commit = NestedConfig.from_file(Path(path) / GITNESTED_FILENAME).commit + try: + return not content.matches_upstream(git, Path(path), commit) + except GitNestedError: + return False + + +def _show_plan(git: GitRunner, state: dict[str, str], heading: str) -> None: + output.say(heading) + for step in _plan(git, state): + for line in _step_lines(git, step): + output.say(line) + + +def _step_lines(git: GitRunner, step: _Step) -> list[str]: + """What --dry-run shows for `step`. For a pull, also what it does deeper down.""" + lines = [] + if step.what != 'skip' or step.note: + lines.append(f"{step.path}: would {step.text()}" + (f" ({step.note})" if step.note else '')) + if step.what == 'pull': + lines += _deeper(git, step.path, step.to, step.url) + return lines + + +def _deeper(git: GitRunner, path: str, pin: str, url: str | None) -> list[str]: + """What a pull of the nested repo at `path` to `pin` would do to the nested repos inside it.""" + config = NestedConfig.from_file(Path(path) / GITNESTED_FILENAME) + if config.submodules == 'keep': + return [] + _have_commit(git, url or config.remote, pin) + lines = [] + for relative, inner_pin in sorted(submodules.gitlinks_at(git, pin).items()): + lines += _deeper_one(git, f'{path}/{relative}', inner_pin) + gone = set(submodules.gitlinks_at(git, config.commit)) - set(submodules.gitlinks_at(git, pin)) + lines += [f"{path}/{relative}: would remove" for relative in sorted(gone) if _is_nested(f'{path}/{relative}')] + return lines + + +def _deeper_one(git: GitRunner, path: str, pin: str) -> list[str]: + """What a pull would do to the inner submodule at `path`, which its parent now pins at `pin`.""" + if not _is_nested(path): + return [] if Path(path).exists() else [f"{path}: would nest at {pin[:12]}"] + if NestedConfig.from_file(Path(path) / GITNESTED_FILENAME).commit == pin: + return [] + return [f"{path}: would pull to {pin[:12]}", *_deeper(git, path, pin, None)] + + +def _have_commit(git: GitRunner, remote: str, commit: str) -> None: + """Fetch `commit` from `remote`, unless it is here already.""" + if git.run(['cat-file', '-e', f'{commit}^{{commit}}'], may_fail=True).returncode: + git.network(['fetch', '--no-tags', '--quiet', remote, commit], f"fetching {commit[:12]} from {remote}") + + +def _apply_skip(_ctx: CommandContext, step: _Step) -> None: + if step.note: + output.warn(f"{step.path}: {step.note}") + + +def _apply_move(ctx: CommandContext, step: _Step) -> None: + Path(step.to).parent.mkdir(parents=True, exist_ok=True) + ctx.git.run(['mv', '--', step.path, step.to]) + message = f"git nested submodules: move {step.path} to {step.to}" + content.commit_gitnested_update(ctx.git, ctx.flags, Path(step.to), message) + output.say(f"{step.path}: moved to {step.to}") + + +def _apply_remove(ctx: CommandContext, step: _Step) -> None: + if step.note: + output.warn(f"{step.path}: {step.note}") + ctx.git.run(['rm', '-r', '-q', '--', step.path]) + content.commit_gitnested_update(ctx.git, ctx.flags, Path(step.path), f"git nested submodules: remove {step.path}") + output.say(f"{step.path}: removed, upstream dropped it") + + +def _apply_nest(ctx: CommandContext, step: _Step) -> None: + flags = cast('Flags', replace(ctx.flags, submodules=_choice_of_the_others(ctx.git))) + clone.clone_at(ctx.git, flags, Path(step.path), step.url or '', step.to) + + +def _choice_of_the_others(git: GitRunner) -> str | None: + """What the nested repos do with their submodules: 'keep' if one keeps them, else 'nest'. + + None if no .gitnested says it. + """ + files = git.check_output(['ls-files', '-z', '--', f':(glob)**/{GITNESTED_FILENAME}']).split('\0') + choices = {(configio.read(Path(path)).get('nested') or {}).get('submodules') for path in files if path} + return next((choice for choice in ('keep', 'nest') if choice in choices), None) + + +def _apply_url(ctx: CommandContext, step: _Step) -> None: + subdir = Path(step.path) + config.write_field(ctx.git, subdir / GITNESTED_FILENAME, 'remote', step.url or '') + content.commit_gitnested_update(ctx.git, ctx.flags, subdir, f"git nested submodules: new url for {step.path}") + output.say(f"{step.path}: remote set to {step.url}") + + +def _apply_pull(ctx: CommandContext, step: _Step) -> None: + try: + pull.pull_to(ctx, Path(step.path), step.url, step.to) + except GitNestedError as e: + tree = _stopped_tree(ctx, step.path) + if tree is None: + raise + patch = re.search(r'the patch (\S+) does not apply', e.message) + why = f" The patch {patch[1]} does not apply." if patch else '' + raise GitNestedError(_stopped_message(ctx.git, step.path, tree, why)) from None + + +def _stopped_tree(ctx: CommandContext, path: str) -> Path | None: + """The worktree where the pull of `path` stopped on conflicts, if it did.""" + branch = refs.nested_branch(ctx.git, refs.sanitize_subref(path)) + # A rebase that stopped has no branch checked out + tree = worktree.find_worktree(ctx.git, branch) or ctx.tmp / branch + return tree if tree.is_dir() and worktree.stopped(ctx.git, tree) else None + + +def _stopped_message(git: GitRunner, path: str, tree: Path, why: str) -> str: + """How to go on after a conflict in a nested repo: the `submodules` way only.""" + files = _conflicts(git, tree) + return ( + f"{path}: the update stopped on conflicts in {', '.join(files) or 'its files'}.{why}\n" + "Fix them there, and 'git add' them:\n\n" + f" cd {Path(os.path.relpath(tree)).as_posix()}\n\n" + f"Then run 'git nested submodules update --continue'. It finishes {path} and goes on with the others.\n" + "Or throw the update away: 'git nested submodules update --abort'." + ) + + +# Each step makes one commit, or none for skip. +_APPLY: dict[str, Callable[[CommandContext, _Step], None]] = { + 'skip': _apply_skip, + 'move': _apply_move, + 'remove': _apply_remove, + 'nest': _apply_nest, + 'pull': _apply_pull, + 'url': _apply_url, +} diff --git a/src/git_nested/content.py b/src/git_nested/content.py index eca6f541..a986915b 100644 --- a/src/git_nested/content.py +++ b/src/git_nested/content.py @@ -135,13 +135,16 @@ def keep_inner_nested(git: GitRunner, flags: Flags, config: NestedConfig, subdir An inner nested repo comes back from `head`, with its local changes. Not with --force: that takes upstream as it is. With --recurse-submodules a new submodule is left out. It is nested next. + One that git skips, `update = none`, stays a submodule. """ drop_new = flags.recurse_submodules and not (config.filter or config.exclude) + gitmodules = subdir / '.gitmodules' for path, _ in submodules.gitlinks(git, subdir): + relative = path.removeprefix(f'{subdir.as_posix()}/') if not flags.force and _was_nested(git, head, path): git.run(['update-index', '--force-remove', '--', path]) git.run(['checkout', cast('str', head), '--', path]) - elif drop_new: + elif drop_new and not submodules.skipped_by_git(git, gitmodules, relative): _drop_gitlink(git, Path(path)) @@ -200,16 +203,22 @@ def keep_upstream_bytes(git: GitRunner, flags: Flags, subdir: Path) -> None: For a submodule, the parent's attributes do not count. For a nested repo they do. A rule like '*.bat text eol=crlf' then shows upstream's - files as changed. This adds '-text' for them to .gitattributes, in a - commit of its own. + files as changed, or checks out LF files with CRLF. This adds '-text' + for them to .gitattributes, in a commit of its own. """ changed = [path for path in git.check_output(['diff', '--name-only', '-z', '--', subdir]).split('\0') if path] - paths = [path for path in changed if _only_attributes_differ(git, path)] + converted = _converted_on_checkout(git, subdir) + paths = sorted({path for path in changed if _only_attributes_differ(git, path)} | set(converted)) if not paths: return text = _GITATTRIBUTES.read_text(encoding='utf-8') if _GITATTRIBUTES.is_file() else '' with _GITATTRIBUTES.open('w', encoding='utf-8', newline='\n') as f: f.write(_with_rules(text, paths)) + # Check them out again, now with upstream's bytes + for path in converted: + Path(path).unlink() + if converted: + git.run(['checkout', '--', *converted]) git.run(['add', '--', _GITATTRIBUTES, *paths]) output.say(f"{subdir.as_posix()}: .gitattributes keeps {len(paths)} files as upstream has them") message = f"git nested: keep {subdir.as_posix()} files as upstream has them" @@ -226,6 +235,28 @@ def _with_rules(text: str, paths: list[str]) -> str: return text + ''.join(f"{_attribute_pattern(path)} -text\n" for path in paths) +def _converted_on_checkout(git: GitRunner, subdir: Path) -> list[str]: + """The files in subdir that git checked out with other line endings than their blob has.""" + converted = [] + for entry in git.check_output(['ls-files', '--eol', '-z', '--', subdir]).split('\0'): + info, _, path = entry.partition('\t') + ends = info.split()[:2] + if path and len(ends) == 2 and ends[0][2:] and ends[1][2:] and ends[0][2:] != ends[1][2:]: + converted.append(path) + return converted + + +def split_keep_block(text: str) -> tuple[str, str]: + """`text` of .gitattributes without git-nested's block, and the block. The block is '' if there is none.""" + lines = text.splitlines(keepends=True) + if _KEEP_BYTES_HEADER not in lines: + return text, '' + start = end = lines.index(_KEEP_BYTES_HEADER) + while end + 1 < len(lines) and lines[end + 1].rstrip().endswith(' -text'): + end += 1 + return ''.join(lines[:start] + lines[end + 1 :]), ''.join(lines[start : end + 1]) + + def _only_attributes_differ(git: GitRunner, path: str) -> bool: """Whether the file has the bytes of its blob, and git shows it as changed only because of attributes.""" blob = git.check_output(['ls-files', '-s', '--', path]).split()[1:2] diff --git a/src/git_nested/models.py b/src/git_nested/models.py index bd46d7f7..f365c87f 100644 --- a/src/git_nested/models.py +++ b/src/git_nested/models.py @@ -60,6 +60,8 @@ class Flags: output_dir: str | None = None tag: str | None = None update: bool = False + resume: bool = False + abort: bool = False quiet: bool = False verbose: int = 0 @@ -175,6 +177,7 @@ class CommandContext: patch_names: list[str] | None = None verify_mode: str | None = None submodules_action: str | None = None + submodules_args: list[str] | None = None git_tmp: Path | None = None head_commit: str | None = None diff --git a/src/git_nested/submodules.py b/src/git_nested/submodules.py index cc283110..489df134 100644 --- a/src/git_nested/submodules.py +++ b/src/git_nested/submodules.py @@ -44,6 +44,13 @@ def entry(git: GitRunner, gitmodules: Path, path: str) -> tuple[str, str] | None return next(((name, f['url']) for name, f in fields.items() if f.get('path') == path and 'url' in f), None) +def skipped_by_git(git: GitRunner, gitmodules: Path, path: str) -> bool: + """Whether `gitmodules` says `update = none` for the submodule at `path`. git does not check those out.""" + found = entry(git, gitmodules, path) + key = f'submodule.{found[0]}.update' if found else '' + return bool(key) and git.check_output(['config', '-f', gitmodules, '--get', key], may_fail=True) == 'none' + + def url_of(git: GitRunner, gitmodules: Path, path: str, base: str) -> str | None: """The url `gitmodules` gives for `path`; a relative one is resolved against `base`.""" found = entry(git, gitmodules, path) diff --git a/tests/e2e/test_attributes.py b/tests/e2e/test_attributes.py index 1c6d104b..93f04632 100644 --- a/tests/e2e/test_attributes.py +++ b/tests/e2e/test_attributes.py @@ -30,10 +30,12 @@ def test_clone_keeps_crlf_files_as_upstream_has_them(foo_bar_cloned): result = cmd_git_nested(['clone', (env.upstream / 'bar').as_posix()], foo) - assert "bar: .gitattributes keeps 1 files as upstream has them" in result.output + assert "bar: .gitattributes keeps 2 files as upstream has them" in result.output assert env.run(['git', 'status', '--porcelain'], cwd=foo).output == '' assert "/bar/make.bat -text" in (foo / '.gitattributes').read_text() - assert 'plain.bat' not in (foo / '.gitattributes').read_text() + # git would check this one out with CRLF. A submodule keeps it LF. + assert "/bar/plain.bat -text" in (foo / '.gitattributes').read_text() + assert (foo / 'bar' / 'plain.bat').read_bytes() == b"echo plain\n" assert env.run(['git', 'show', 'HEAD:bar/make.bat'], cwd=foo, text=False).stdout == b"echo hi\r\n" assert cmd_git_nested('verify bar', cwd=foo).returncode == 0 diff --git a/tests/e2e/test_issue_184.py b/tests/e2e/test_issue_184.py index e075b833..59fa7cde 100644 --- a/tests/e2e/test_issue_184.py +++ b/tests/e2e/test_issue_184.py @@ -1,4 +1,4 @@ -"""The scenario of issue #184. +"""The scenario of issue #184, from start to end. Upstream has these repos: @@ -7,11 +7,12 @@ - docs: has the submodule theme. Super adds it later. - lib, tools, theme: plain repos. -The user wants the submodules as nested repos: no gitlink, no .gitmodules. +The user forks super. The fork has no submodules, only nested repos. It +pulls from the remote 'upstream' and pushes to its own 'origin'. """ import pytest -from conftest import cmd_git_nested +from conftest import assert_gitnested_field, cmd_git_nested, git_rev_parse ALLOW_FILE = ['-c', 'protocol.file.allow=always'] @@ -80,11 +81,31 @@ def upstream(env): return up +@pytest.fixture +def mine(env, upstream): + """The user's fork: flattened, with 'origin' as the fork and 'upstream' as super.""" + repo = env.workspace / 'mine' + env.run(['git', 'clone', '-q', upstream.url('super'), repo]) + cmd_git_nested('submodules init --recursive', cwd=repo) + fork = env.upstream / 'fork' + env.run(['git', 'init', '-q', '--bare', fork]) + env.run(['git', 'remote', 'rename', 'origin', 'upstream'], cwd=repo) + env.run(['git', 'remote', 'add', 'origin', fork], cwd=repo) + env.run(['git', 'push', '-q', '-u', 'origin', 'master'], cwd=repo) + return repo + + # ============================================================================ # Helpers: checks in the fork # ============================================================================ +def _update_from_upstream(env, repo, check=True): + """Fetch upstream, then `submodules update upstream/master`.""" + env.run(['git', 'fetch', '-q', 'upstream'], cwd=repo) + return cmd_git_nested('submodules update upstream/master', cwd=repo, check=check) + + def _git(env, repo, *args): return env.run(['git', *args], cwd=repo).output.strip() @@ -103,6 +124,156 @@ def _nested_repos(env, repo): return sorted(path.removesuffix('/.gitnested') for path in files if path.endswith('/.gitnested')) +def _owner(path, roots): + """The nested repo that `path` belongs to: the longest root that holds it.""" + owners = [root for root in roots if path == root or path.startswith(f'{root}/')] + return max(owners, key=len) if owners else path + + +def _submodules_per_commit(env, repo, base, roots): + """For each commit after base, the nested repos it changed.""" + result = [] + for commit in _git(env, repo, 'rev-list', '--reverse', f'{base}..HEAD').split(): + paths = _git(env, repo, 'diff', '--name-only', f'{commit}^', commit).split() + result.append({_owner(path, roots) for path in paths if path != '.gitmodules'}) + return result + + +def _last_merge(env, repo): + return _git(env, repo, 'rev-list', '--merges', '-1', 'HEAD') + + +def _is_ancestor(env, repo, commit): + return env.run(['git', 'merge-base', '--is-ancestor', commit, 'HEAD'], cwd=repo, check=False).returncode == 0 + + +def _edit(env, repo, path, text): + with (repo / path).open('a') as f: + f.write(f'{text}\n') + env.run(['git', 'commit', '-qam', f'my change in {path}'], cwd=repo) + + +# ============================================================================ +# The whole story +# ============================================================================ + + +def test_the_fork_of_issue_184(env, upstream, mine): + """Flatten the fork. Then follow upstream over two rounds of changes.""" + # --- The start: everything is flat, the history of super is there. + assert _nested_repos(env, mine) == ['modules/core', 'modules/core/third_party/lib', 'tools'] + assert _is_ancestor(env, mine, upstream.head('super')) + assert (mine / 'modules' / 'core' / 'third_party' / 'lib' / 'Lib').is_file() + _assert_flat(env, mine) + + # --- Local changes in the flattened submodules, at every depth. + _edit(env, mine, 'modules/core/Core', 'my core change') + _edit(env, mine, 'modules/core/third_party/lib/Lib', 'my lib change') + _edit(env, mine, 'tools/Tools', 'my tools change') + env.add_new_files('FORK', cwd=mine) + + # --- Upstream round 1. + upstream.write('lib', LibNew='lib 2') + lib_pin = upstream.push('lib', 'lib: new file') + upstream.follow('core', 'third_party/lib') + upstream.write('core', CoreNew='core 2') + core_pin = upstream.push('core', 'core: new file, bump lib') + upstream.write('tools', ToolsNew='tools 2') + upstream.push('tools', 'tools: new file') + mirror = env.upstream / 'tools-mirror' + env.run(['git', 'clone', '-q', '--bare', upstream.url('tools'), mirror]) + + upstream.write('super', README='more text') + upstream.follow('super', 'modules/core') + alice = upstream.push('super', 'Update readme and core', author='Alice') + upstream.add_submodule('super', 'docs', 'docs') + bob = upstream.push('super', 'Add docs', author='Bob') + upstream.git('super', 'submodule', 'set-url', 'tools', mirror.as_posix()) + upstream.follow('super', 'tools') + tools_pin = upstream.git('super', '-C', 'tools', 'rev-parse', 'HEAD') + alice_again = upstream.push('super', 'Move tools to a mirror and update it', author='Alice') + + _update_from_upstream(env, mine) + + # Each superproject commit is there, as upstream made it. + for commit, author in ((alice, 'Alice'), (bob, 'Bob'), (alice_again, 'Alice')): + assert _is_ancestor(env, mine, commit) + assert _git(env, mine, 'log', '-1', '--format=%an', commit) == author + merge = _last_merge(env, mine) + assert git_rev_parse([f'{merge}^2'], cwd=mine) == alice_again + + # Each submodule moved, with the local changes kept. + assert (mine / 'modules' / 'core' / 'CoreNew').is_file() + assert 'my core change' in (mine / 'modules' / 'core' / 'Core').read_text() + assert_gitnested_field(mine / 'modules' / 'core' / '.gitnested', commit=core_pin) + lib = mine / 'modules' / 'core' / 'third_party' / 'lib' + assert (lib / 'LibNew').is_file() + assert 'my lib change' in (lib / 'Lib').read_text() + assert_gitnested_field(lib / '.gitnested', commit=lib_pin) + assert (mine / 'tools' / 'ToolsNew').is_file() + assert 'my tools change' in (mine / 'tools' / 'Tools').read_text() + assert_gitnested_field(mine / 'tools' / '.gitnested', remote=mirror.as_posix(), commit=tools_pin) + + # The new submodule is nested, its own submodule too. + assert (mine / 'docs' / 'theme' / 'Theme').is_file() + assert _nested_repos(env, mine) == [ + 'docs', + 'docs/theme', + 'modules/core', + 'modules/core/third_party/lib', + 'tools', + ] + + # Each changed submodule got its own commit. Nothing else is in them. + changed = _submodules_per_commit(env, mine, merge, _nested_repos(env, mine)) + assert all(len(repos) == 1 for repos in changed), changed + assert sorted(repo for repos in changed for repo in repos) == sorted([ + 'docs', + 'docs/theme', + 'modules/core', + 'modules/core/third_party/lib', + 'tools', + ]) + _assert_flat(env, mine) + + # --- The fork can push. A plain clone of it has all files. + env.run(['git', 'push', '-q', 'origin', 'master'], cwd=mine) + teammate = env.workspace / 'teammate' + env.run(['git', 'clone', '-q', env.upstream / 'fork', teammate]) + assert (teammate / 'modules' / 'core' / 'third_party' / 'lib' / 'LibNew').is_file() + assert (teammate / 'docs' / 'theme' / 'Theme').is_file() + + # --- Upstream round 2: drop tools, move docs. + _edit(env, mine, 'docs/Docs', 'my docs change') + upstream.git('super', 'rm', '-q', 'tools') + dropped = upstream.push('super', 'Drop tools', author='Bob') + (upstream.work / 'super' / 'documentation').mkdir() + upstream.git('super', 'mv', 'docs', 'documentation/docs') + moved = upstream.push('super', 'Move docs', author='Alice') + + _update_from_upstream(env, mine) + + assert _is_ancestor(env, mine, dropped) + assert _is_ancestor(env, mine, moved) + assert not (mine / 'tools').exists() + assert not (mine / 'docs').exists() + docs = mine / 'documentation' / 'docs' + assert 'my docs change' in (docs / 'Docs').read_text() + assert (docs / 'theme' / 'Theme').is_file() + assert _nested_repos(env, mine) == [ + 'documentation/docs', + 'documentation/docs/theme', + 'modules/core', + 'modules/core/third_party/lib', + ] + _assert_flat(env, mine) + + # --- Nothing new upstream. + head = git_rev_parse(['HEAD'], cwd=mine) + assert 'already up to date' in _update_from_upstream(env, mine).output + assert git_rev_parse(['HEAD'], cwd=mine) == head + + # ============================================================================ # Single cases of the story # ============================================================================ @@ -119,3 +290,75 @@ def test_init_in_a_clone_the_user_already_has(env, upstream): assert (repo / 'modules' / 'core' / 'third_party' / 'lib' / 'Lib').is_file() assert not (repo / '.git' / 'modules' / 'modules' / 'core').exists() _assert_flat(env, repo) + + +def test_a_conflict_inside_a_submodule(env, upstream, mine): + """The fork and upstream change the same lines. You fix it the git way. Then you continue.""" + _edit(env, mine, 'modules/core/Core', 'my core change') + upstream.write('core', Core='upstream core change') + core_pin = upstream.push('core', 'core: change Core') + upstream.write('tools', ToolsNew='tools 2') + tools_pin = upstream.push('tools', 'tools: new file') + upstream.follow('super', 'modules/core') + upstream.follow('super', 'tools') + upstream.push('super', 'Bump core and tools') + + result = _update_from_upstream(env, mine, check=False) + + assert result.returncode == 1 + assert 'modules/core' in result.output + assert 'submodules update --continue' in result.output + worktree = next(line.split('cd ', 1)[1] for line in result.output.splitlines() if 'cd ' in line) + + worktree = mine / worktree + (worktree / 'Core').write_text('core\nboth changes\n') + env.run(['git', 'add', 'Core'], cwd=worktree) + env.run(['git', 'commit', '-q', '--no-edit'], cwd=worktree) + cmd_git_nested('commit modules/core', cwd=mine) + cmd_git_nested('submodules update --continue', cwd=mine) + + assert (mine / 'modules' / 'core' / 'Core').read_text() == 'core\nboth changes\n' + assert_gitnested_field(mine / 'modules' / 'core' / '.gitnested', commit=core_pin) + assert_gitnested_field(mine / 'tools' / '.gitnested', commit=tools_pin) + _assert_flat(env, mine) + + +def test_upstream_history_with_a_merge_commit(env, upstream, mine): + """Upstream works on a branch and merges it. The fork keeps that history as it is.""" + upstream.git('super', 'checkout', '-q', '-b', 'feature') + upstream.write('core', CoreNew='core 2') + core_pin = upstream.push('core', 'core: new file') + upstream.follow('super', 'modules/core') + upstream.git('super', 'commit', '-qam', 'Bump core on a branch') + feature = upstream.head('super') + upstream.git('super', 'checkout', '-q', 'master') + upstream.git('super', '-c', 'user.name=Alice', 'merge', '-q', '--no-ff', '-m', 'Merge feature', 'feature') + upstream.git('super', 'submodule', 'update', '-q') + upstream.git('super', 'push', '-q', 'origin', 'master') + merged = upstream.head('super') + + _update_from_upstream(env, mine) + + assert _is_ancestor(env, mine, feature) + assert _is_ancestor(env, mine, merged) + assert_gitnested_field(mine / 'modules' / 'core' / '.gitnested', commit=core_pin) + _assert_flat(env, mine) + + +def test_a_submodule_bump_inside_a_submodule(env, upstream, mine): + """Only lib moves, deep inside core. core's commit pins it, super pins core.""" + _edit(env, mine, 'modules/core/third_party/lib/Lib', 'my lib change') + upstream.write('lib', LibNew='lib 2') + lib_pin = upstream.push('lib', 'lib: new file') + upstream.follow('core', 'third_party/lib') + upstream.push('core', 'core: bump lib') + upstream.follow('super', 'modules/core') + upstream.push('super', 'Bump core') + + _update_from_upstream(env, mine) + + lib = mine / 'modules' / 'core' / 'third_party' / 'lib' + assert (lib / 'LibNew').is_file() + assert 'my lib change' in (lib / 'Lib').read_text() + assert_gitnested_field(lib / '.gitnested', commit=lib_pin) + _assert_flat(env, mine) diff --git a/tests/e2e/test_submodules_command.py b/tests/e2e/test_submodules_command.py index 64bb8b67..17c685d0 100644 --- a/tests/e2e/test_submodules_command.py +++ b/tests/e2e/test_submodules_command.py @@ -3,11 +3,23 @@ Upstream bar is the superproject. Its submodules become nested repos. """ +import pytest from conftest import assert_gitnested_field, clone_repo, cmd_git_nested, git_rev_parse ALLOW_FILE = ['-c', 'protocol.file.allow=always'] +@pytest.fixture +def fork(foo_bar_cloned): + """Fork bar and nest its submodule 'sub'.""" + env = foo_bar_cloned + _upstream_adds_submodule(env, env.upstream / 'foo', 'sub') + env.fork = env.workspace / 'fork' + clone_repo((env.upstream / 'bar').as_posix(), env.fork) + cmd_git_nested('init --from-submodule sub', cwd=env.fork) + return env + + # ============================================================================ # Helpers: changes in upstream # ============================================================================ @@ -28,6 +40,22 @@ def _upstream_adds_submodule(env, url, path): return git_rev_parse(['HEAD'], cwd=bar / path) +def _upstream_commit(env, name): + """Commit a new file in upstream bar. Return the commit.""" + env.add_new_files(name, cwd=env.workspace / 'bar') + env.run(['git', 'push', '-q'], cwd=env.workspace / 'bar') + return git_rev_parse(['HEAD'], cwd=env.workspace / 'bar') + + +def _upstream_bumps_sub(env): + """Make upstream bar pin a new foo commit. Return it.""" + env.add_new_files('FooNew', cwd=env.workspace / 'foo') + env.run(['git', 'push', '-q'], cwd=env.workspace / 'foo') + env.run(['git', 'pull', '-q'], cwd=env.workspace / 'bar' / 'sub') + _push_bar(env, 'bump sub') + return git_rev_parse(['HEAD'], cwd=env.workspace / 'bar' / 'sub') + + def _make_mid(env): """Make upstream mid. It has the submodule 'inner'. Return its url.""" url = env.upstream / 'mid' @@ -41,6 +69,16 @@ def _make_mid(env): return url +def _update(env, *args, check=True): + return cmd_git_nested(['submodules', 'update', *args], env.fork, check=check) + + +def _fetch_and_update(env, *args, remote='origin', check=True): + """Fetch `remote`, then update to its master.""" + env.run(['git', 'fetch', '-q', remote], cwd=env.fork) + return _update(env, f'{remote}/master', *args, check=check) + + # ============================================================================ # Helpers: checks # ============================================================================ @@ -55,6 +93,10 @@ def _assert_flat(env, repo): assert env.run(['git', 'status', '--porcelain'], cwd=repo).output == '' +def _is_ancestor(env, repo, commit): + return env.run(['git', 'merge-base', '--is-ancestor', commit, 'HEAD'], cwd=repo, check=False).returncode == 0 + + def _top_dirs_per_commit(env, repo, base): """The top dirs that each commit after base changed, sorted. .gitmodules does not count.""" commits = env.run(['git', 'rev-list', f'{base}..HEAD'], cwd=repo).output.split() @@ -63,6 +105,10 @@ def _top_dirs_per_commit(env, repo, base): return sorted(dirs, key=sorted) +def _last_update(env, repo): + return env.run(['git', 'rev-list', '--merges', '-1', 'HEAD'], cwd=repo).output.strip() + + # ============================================================================ # submodules init # ============================================================================ @@ -123,6 +169,52 @@ def test_init_recursive_writes_nest(foo_bar_cloned): assert 'mid: matches upstream' in cmd_git_nested('verify mid', cwd=repo).output +def _upstream_bumps_inner_of_mid(env): + """Make upstream mid pin a new foo commit, and bar pin the new mid. Return the new inner pin.""" + env.add_new_files('FooNext', cwd=env.workspace / 'foo') + env.run(['git', 'push', '-q'], cwd=env.workspace / 'foo') + mid = env.workspace / 'mid' + env.run(['git', *ALLOW_FILE, 'submodule', 'update', '-q', '--init', '--remote', 'inner'], cwd=mid) + env.run(['git', 'commit', '-qam', 'bump inner'], cwd=mid) + env.run(['git', 'push', '-q', 'origin', 'master'], cwd=mid) + bar_mid = env.workspace / 'bar' / 'mid' + env.run(['git', 'pull', '-q', 'origin', 'master'], cwd=bar_mid) + _push_bar(env, 'bump mid') + return git_rev_parse(['HEAD:inner'], cwd=mid) + + +def test_update_keeps_a_deeper_submodule_as_init_left_it(foo_bar_cloned): + env = foo_bar_cloned + _upstream_adds_submodule(env, _make_mid(env), 'mid') + env.fork = env.workspace / 'fork' + clone_repo((env.upstream / 'bar').as_posix(), env.fork) + cmd_git_nested('submodules init', cwd=env.fork) + pinned = _upstream_bumps_inner_of_mid(env) + env.run(['git', 'fetch', '-q'], cwd=env.fork) + preview = _update(env, 'origin/master', '--dry-run').output + assert 'mid: would pull to' in preview + assert 'mid/inner' not in preview + + _fetch_and_update(env) + + entry = env.run(['git', 'ls-tree', 'HEAD', 'mid/inner'], cwd=env.fork).output.split() + assert entry[:3] == ['160000', 'commit', pinned] + assert not (env.fork / 'mid' / 'inner' / '.gitnested').exists() + + +def test_a_new_submodule_follows_the_choice_of_the_others(fork): + env = fork + cmd_git_nested(['config', 'sub', 'submodules', 'keep'], cwd=env.fork) + env.run(['git', 'commit', '-qam', 'keep the submodules of sub'], cwd=env.fork) + _upstream_adds_submodule(env, _make_mid(env), 'mid') + + result = _fetch_and_update(env) + + assert_gitnested_field(env.fork / 'mid' / '.gitnested', submodules='keep') + assert _gitlinks(env, env.fork) == ['mid/inner'] + assert "mid: these submodules stay empty: inner; its .gitnested says 'submodules: keep'" in result.output + + def test_init_recursive_resolves_deeper_relative_urls_against_their_parent(foo_bar_cloned): """--resolve-relative is for the first level. Deeper, the url of the parent submodule counts.""" env = foo_bar_cloned @@ -216,6 +308,107 @@ def test_init_paths_are_relative_to_the_current_directory(foo_bar_cloned): assert _gitlinks(env, repo) == ['ext/lib'] +def _partial_fork(env, *options): + """Fork bar with sub, ext/lib and ext/more. init with `options`.""" + for path in ('sub', 'ext/lib', 'ext/more'): + _upstream_adds_submodule(env, env.upstream / 'foo', path) + env.fork = env.workspace / 'fork' + clone_repo((env.upstream / 'bar').as_posix(), env.fork) + cmd_git_nested(['submodules', 'init', *options], cwd=env.fork) + + +def _module(env, key): + return env.run(['git', 'config', '-f', '.gitmodules', key], cwd=env.fork, check=False).output.strip() + + +def test_update_keeps_a_new_submodule_that_init_ignores(foo_bar_cloned): + env = foo_bar_cloned + _partial_fork(env, '--ignore', 'ext') + pinned = _upstream_adds_submodule(env, env.upstream / 'foo', 'ext/new') + + assert 'ext/new' not in _fetch_and_update(env, '--dry-run').output + _fetch_and_update(env) + + assert _gitlinks(env, env.fork) == ['ext/lib', 'ext/more', 'ext/new'] + assert env.run(['git', 'ls-tree', 'HEAD', 'ext/new'], cwd=env.fork).output.split()[2] == pinned + assert _module(env, 'submodule.ext/new.path') == 'ext/new' + assert _module(env, 'nested.ignore') == 'ext' + + +def test_update_keeps_a_new_submodule_outside_of_only(foo_bar_cloned): + env = foo_bar_cloned + _partial_fork(env, '--only', 'sub') + _upstream_adds_submodule(env, env.upstream / 'foo', 'other') + + _fetch_and_update(env) + + assert _gitlinks(env, env.fork) == ['ext/lib', 'ext/more', 'other'] + + +def test_update_nests_a_new_submodule_that_init_selects(foo_bar_cloned): + env = foo_bar_cloned + _partial_fork(env, '--only', 'ext') + _upstream_adds_submodule(env, env.upstream / 'foo', 'ext/new') + + _fetch_and_update(env) + + assert (env.fork / 'ext' / 'new' / '.gitnested').is_file() + assert _gitlinks(env, env.fork) == ['sub'] + + +def _set_url(env, path, url): + bar = env.workspace / 'bar' + env.run(['git', 'config', '-f', '.gitmodules', f'submodule.{path}.url', url], cwd=bar) + _push_bar(env, f'new url for {path}') + + +def test_update_takes_a_new_url_of_a_kept_submodule(foo_bar_cloned): + env = foo_bar_cloned + _partial_fork(env, '--ignore', 'ext') + (env.fork / '.gitmodules').write_text("# mine\n" + (env.fork / '.gitmodules').read_text()) + env.run(['git', 'commit', '-qam', 'my note in .gitmodules'], cwd=env.fork) + _set_url(env, 'ext/lib', 'https://example.com/lib.git') + + _fetch_and_update(env) + + assert _module(env, 'submodule.ext/lib.url') == 'https://example.com/lib.git' + assert '# mine' in (env.fork / '.gitmodules').read_text() + assert _module(env, 'submodule.sub.url') == '' + + +def test_a_conflict_in_gitmodules_stops_and_continues(foo_bar_cloned): + env = foo_bar_cloned + _partial_fork(env, '--ignore', 'ext') + env.run(['git', 'config', '-f', '.gitmodules', 'submodule.ext/lib.url', 'https://mine/lib.git'], cwd=env.fork) + env.run(['git', 'commit', '-qam', 'my url'], cwd=env.fork) + _set_url(env, 'ext/lib', 'https://theirs/lib.git') + + result = _fetch_and_update(env, check=False) + + assert 'the merge has conflicts in: .gitmodules' in result.output + assert '# <<<<<<< HEAD' in (env.fork / '.gitmodules').read_text() + assert env.run(['git', 'config', '-f', '.gitmodules', '--list'], cwd=env.fork).returncode == 0 + env.run(['git', 'checkout', '--ours', '.gitmodules'], cwd=env.fork) + env.run(['git', 'add', '.gitmodules'], cwd=env.fork) + _update(env, '--continue') + assert _module(env, 'submodule.ext/lib.url') == 'https://mine/lib.git' + assert env.run(['git', 'status', '--porcelain'], cwd=env.fork).output == '' + + +def test_update_follows_a_kept_submodule_that_upstream_moved(foo_bar_cloned): + env = foo_bar_cloned + _partial_fork(env, '--ignore', 'ext') + bar = env.workspace / 'bar' + env.run(['git', 'mv', 'ext/lib', 'ext/moved'], cwd=bar) + env.run(['git', 'commit', '-qm', 'move ext/lib'], cwd=bar) + env.run(['git', 'push', '-q'], cwd=bar) + + _fetch_and_update(env) + + assert _gitlinks(env, env.fork) == ['ext/more', 'ext/moved'] + assert _module(env, 'submodule.ext/lib.path') == 'ext/moved' + + def test_init_refuses_a_path_without_a_submodule(foo_bar_cloned): env = foo_bar_cloned repo = _repo_with_three_submodules(env) @@ -228,6 +421,337 @@ def test_init_refuses_a_path_without_a_submodule(foo_bar_cloned): assert git_rev_parse(['HEAD'], cwd=repo) == head +def test_update_after_a_partial_init(foo_bar_cloned): + """The nested one follows upstream. The other stays a submodule.""" + env = foo_bar_cloned + _upstream_adds_submodule(env, env.upstream / 'foo', 'other') + _upstream_adds_submodule(env, env.upstream / 'foo', 'sub') + env.fork = env.workspace / 'fork' + clone_repo((env.upstream / 'bar').as_posix(), env.fork) + cmd_git_nested('submodules init --only sub', cwd=env.fork) + pinned = _upstream_bumps_sub(env) + + _fetch_and_update(env) + + assert_gitnested_field(env.fork / 'sub' / '.gitnested', commit=pinned) + assert _gitlinks(env, env.fork) == ['other'] + + +# ============================================================================ +# submodules update +# ============================================================================ + + +def test_update_keeps_each_upstream_commit(fork): + """Upstream commits keep their SHAs.""" + env = fork + first = _upstream_commit(env, 'one') + second = _upstream_commit(env, 'two') + env.add_new_files('local', cwd=env.fork) + + _fetch_and_update(env) + + assert _is_ancestor(env, env.fork, first) + assert git_rev_parse(['HEAD^2'], cwd=env.fork) == second + assert (env.fork / 'two').is_file() + _assert_flat(env, env.fork) + + +def _git_merge(env): + """A plain git merge of upstream. It stops on the submodule.""" + env.run(['git', 'fetch', '-q'], cwd=env.fork) + return env.run(['git', 'merge', '--no-edit', 'origin/master'], cwd=env.fork, check=False) + + +def test_update_during_a_git_merge_says_what_to_do(fork): + env = fork + _upstream_bumps_sub(env) + assert _git_merge(env).returncode != 0 + + result = _update(env, check=False) + + assert result.returncode == 1 + assert "a git merge is in progress. Run 'git merge --abort'." in result.output + assert "'git nested submodules update '" in result.output + + +def test_update_moves_a_submodule_that_upstream_bumped(fork): + env = fork + pinned = _upstream_bumps_sub(env) + + _fetch_and_update(env) + + assert (env.fork / 'sub' / 'FooNew').is_file() + assert_gitnested_field(env.fork / 'sub' / '.gitnested', commit=pinned) + _assert_flat(env, env.fork) + + +def test_update_keeps_local_changes_in_a_submodule(fork): + env = fork + env.modify_files('sub/Foo', text='local', cwd=env.fork) + _upstream_bumps_sub(env) + + _fetch_and_update(env) + + assert 'local' in (env.fork / 'sub' / 'Foo').read_text() + assert (env.fork / 'sub' / 'FooNew').is_file() + _assert_flat(env, env.fork) + + +def test_update_makes_one_commit_per_submodule(fork): + """One commit per changed submodule after the merge.""" + env = fork + _upstream_bumps_sub(env) + _upstream_adds_submodule(env, env.upstream / 'foo', 'lib') + + _fetch_and_update(env) + + assert _top_dirs_per_commit(env, env.fork, _last_update(env, env.fork)) == [{'lib'}, {'sub'}] + _assert_flat(env, env.fork) + + +def test_update_nests_a_new_submodule(fork): + env = fork + pinned = _upstream_adds_submodule(env, env.upstream / 'foo', 'lib') + + _fetch_and_update(env) + + assert (env.fork / 'lib' / 'Foo').is_file() + assert_gitnested_field(env.fork / 'lib' / '.gitnested', remote=(env.upstream / 'foo').as_posix(), commit=pinned) + _assert_flat(env, env.fork) + + +def test_update_nests_submodules_of_a_new_submodule(fork): + env = fork + _upstream_adds_submodule(env, _make_mid(env), 'mid') + + _fetch_and_update(env) + + assert (env.fork / 'mid' / 'Mid').is_file() + assert (env.fork / 'mid' / '.gitnested').is_file() + assert (env.fork / 'mid' / 'inner' / 'Foo').is_file() + assert (env.fork / 'mid' / 'inner' / '.gitnested').is_file() + _assert_flat(env, env.fork) + + +def test_update_removes_a_submodule_that_upstream_dropped(fork): + """The fork's own nested repos stay.""" + env = fork + cmd_git_nested(['clone', (env.upstream / 'foo').as_posix(), 'own'], env.fork) + env.run(['git', 'rm', '-q', 'sub'], cwd=env.workspace / 'bar') + _push_bar(env, 'drop sub') + + _fetch_and_update(env) + + assert not (env.fork / 'sub').exists() + assert (env.fork / 'own' / '.gitnested').is_file() + _assert_flat(env, env.fork) + + +def test_update_follows_a_moved_submodule(fork): + """Local changes move along.""" + env = fork + env.modify_files('sub/Foo', text='local', cwd=env.fork) + (env.workspace / 'bar' / 'ext').mkdir() + env.run(['git', 'mv', 'sub', 'ext/sub'], cwd=env.workspace / 'bar') + _push_bar(env, 'move sub') + + _fetch_and_update(env) + + assert not (env.fork / 'sub').exists() + assert 'local' in (env.fork / 'ext' / 'sub' / 'Foo').read_text() + assert (env.fork / 'ext' / 'sub' / '.gitnested').is_file() + _assert_flat(env, env.fork) + + +def test_update_keeps_a_real_submodule(foo_bar_cloned): + """A submodule that the fork did not nest stays a submodule.""" + env = foo_bar_cloned + _upstream_adds_submodule(env, env.upstream / 'foo', 'sub') + env.fork = env.workspace / 'fork' + clone_repo((env.upstream / 'bar').as_posix(), env.fork) + env.add_new_files('local', cwd=env.fork) + pinned = _upstream_bumps_sub(env) + + _fetch_and_update(env) + + entry = env.run(['git', 'ls-tree', 'HEAD', 'sub'], cwd=env.fork).output.split() + assert entry[:3] == ['160000', 'commit', pinned] + assert not (env.fork / 'sub' / '.gitnested').exists() + + +def test_update_follows_a_new_url(fork): + env = fork + mirror = env.upstream / 'foo-mirror' + env.run(['git', 'clone', '-q', '--bare', env.upstream / 'foo', mirror]) + env.run(['git', 'submodule', 'set-url', 'sub', mirror.as_posix()], cwd=env.workspace / 'bar') + _push_bar(env, 'move sub to a mirror') + + _fetch_and_update(env) + + assert_gitnested_field(env.fork / 'sub' / '.gitnested', remote=mirror.as_posix()) + _assert_flat(env, env.fork) + + +def test_update_stops_on_a_conflict_and_continues(fork): + """You fix other conflicts. Then --continue finishes.""" + env = fork + env.modify_files('Bar', text='fork', cwd=env.fork) + env.modify_files('Bar', text='upstream', cwd=env.workspace / 'bar') + env.run(['git', 'push', '-q'], cwd=env.workspace / 'bar') + pinned = _upstream_bumps_sub(env) + + result = _fetch_and_update(env, check=False) + + assert result.returncode == 1 + assert 'Bar' in result.output + assert 'submodules update --continue' in result.output + + (env.fork / 'Bar').write_text('both\n') + env.run(['git', 'add', 'Bar'], cwd=env.fork) + _update(env, '--continue') + + assert _is_ancestor(env, env.fork, git_rev_parse(['origin/master'], cwd=env.fork)) + assert (env.fork / 'Bar').read_text() == 'both\n' + assert_gitnested_field(env.fork / 'sub' / '.gitnested', commit=pinned) + _assert_flat(env, env.fork) + + +def test_update_twice_changes_nothing(fork): + env = fork + _upstream_bumps_sub(env) + _fetch_and_update(env) + head = git_rev_parse(['HEAD'], cwd=env.fork) + + result = _fetch_and_update(env) + + assert 'already up to date' in result.output + assert git_rev_parse(['HEAD'], cwd=env.fork) == head + + +# ============================================================================ +# submodules update: stopping, aborting, errors +# ============================================================================ + + +def _conflict_in_bar(env): + """The fork and upstream change Bar. Upstream also bumps sub. Return the new pin.""" + env.modify_files('Bar', text='fork', cwd=env.fork) + env.modify_files('Bar', text='upstream', cwd=env.workspace / 'bar') + env.run(['git', 'push', '-q'], cwd=env.workspace / 'bar') + return _upstream_bumps_sub(env) + + +def test_update_abort(fork): + env = fork + _conflict_in_bar(env) + head = git_rev_parse(['HEAD'], cwd=env.fork) + assert _fetch_and_update(env, check=False).returncode == 1 + + result = _update(env, '--abort') + + assert 'aborted' in result.output + assert git_rev_parse(['HEAD'], cwd=env.fork) == head + assert env.run(['git', 'status', '--porcelain'], cwd=env.fork).output == '' + assert 'stopped before' not in _fetch_and_update(env, check=False).output + + +def test_update_continue_after_the_merge_was_aborted(fork): + """--continue does not update submodules for a merge that is gone.""" + env = fork + _conflict_in_bar(env) + _fetch_and_update(env, check=False) + env.run(['git', 'merge', '--abort'], cwd=env.fork) + before = (env.fork / 'sub' / '.gitnested').read_text() + + result = _update(env, '--continue', check=False) + + assert result.returncode == 1 + assert 'submodules update --abort' in result.output + assert (env.fork / 'sub' / '.gitnested').read_text() == before + + +def test_update_from_a_repo_without_common_history(fork): + env = fork + + env.run(['git', 'fetch', '-q', env.upstream / 'foo', 'master'], cwd=env.fork) + + result = _update(env, 'FETCH_HEAD', check=False) + + assert result.returncode == 1 + assert 'has no history in common with this branch' in result.output + + +def test_a_failed_merge_leaves_nothing_to_continue(fork): + """An untracked file is where upstream adds one. git does not merge.""" + env = fork + _upstream_commit(env, 'one') + (env.fork / 'one').write_text('mine\n') + result = _fetch_and_update(env, check=False) + assert result.returncode == 1 + assert 'the merge of origin/master failed' in result.output + (env.fork / 'one').unlink() + + tip = _upstream_commit(env, 'two') + _fetch_and_update(env) + + assert _is_ancestor(env, env.fork, tip) + + +def test_continue_or_abort_without_a_pull(fork): + env = fork + assert _update(env, '--continue', check=False).returncode == 1 + assert _update(env, '--abort', check=False).returncode == 1 + + +def test_update_on_a_detached_head(fork): + env = fork + env.run(['git', 'checkout', '-q', '--detach'], cwd=env.fork) + + result = _fetch_and_update(env, check=False) + + assert result.returncode == 1 + assert 'detached' in result.output + + +def test_update_signoff(fork): + env = fork + _upstream_commit(env, 'one') + + _fetch_and_update(env, '--signoff') + + message = env.run(['git', 'log', '-1', '--format=%B', _last_update(env, env.fork)], cwd=env.fork).output + assert 'Signed-off-by:' in message + + +def test_update_warns_about_changes_in_a_dropped_submodule(fork): + env = fork + env.modify_files('sub/Foo', text='local', cwd=env.fork) + env.run(['git', 'rm', '-q', 'sub'], cwd=env.workspace / 'bar') + _push_bar(env, 'drop sub') + + result = _fetch_and_update(env) + + assert 'sub: your changes in it go too' in result.output + assert not (env.fork / 'sub').exists() + + +def test_update_keeps_a_moved_submodule_if_the_path_is_taken(fork): + """The fork has its own ext/sub. sub stays where it is.""" + env = fork + env.add_new_files('ext/sub/mine', cwd=env.fork) + (env.workspace / 'bar' / 'ext').mkdir() + env.run(['git', 'mv', 'sub', 'ext/sub'], cwd=env.workspace / 'bar') + _push_bar(env, 'move sub') + + result = _fetch_and_update(env) + + assert 'that path is taken' in result.output + assert 'ext/sub: the path is taken' in result.output + assert (env.fork / 'sub' / '.gitnested').is_file() + assert (env.fork / 'ext' / 'sub' / 'mine').is_file() + + # ============================================================================ # submodules init: errors and relative urls # ============================================================================ @@ -245,11 +769,342 @@ def test_init_needs_a_clean_tree(foo_bar_cloned): assert 'changes' in result.output +def test_relative_urls_in_a_fork(foo_bar_cloned): + """The fork lives somewhere else. Relative urls point to the project it forked.""" + env = foo_bar_cloned + bar = env.workspace / 'bar' + env.run(['git', *ALLOW_FILE, 'submodule', 'add', '-q', '../foo', 'sub'], cwd=bar) + _push_bar(env, 'add sub') + forks = env.tmp / 'forks' + env.run(['git', 'clone', '-q', '--bare', env.upstream / 'bar', forks / 'bar']) + env.fork = env.workspace / 'fork' + clone_repo((forks / 'bar').as_posix(), env.fork) + env.run(['git', 'remote', 'add', 'upstream', env.upstream / 'bar'], cwd=env.fork) + cmd_git_nested(['submodules', 'init', '--resolve-relative', (env.upstream / 'bar').as_posix()], cwd=env.fork) + pinned = _upstream_bumps_sub(env) + _fetch_and_update(env, remote='upstream') + + remote = (env.upstream / 'foo').as_posix() + assert_gitnested_field(env.fork / 'sub' / '.gitnested', remote=remote, commit=pinned) + _assert_flat(env, env.fork) + + # ============================================================================ -# Nested repos with patches +# submodules update # ============================================================================ -PATCHES = '.gitnested.patches' + +def test_update_a_fetched_branch(fork): + env = fork + pinned = _upstream_bumps_sub(env) + tip = git_rev_parse(['HEAD'], cwd=env.workspace / 'bar') + env.run(['git', 'fetch', '-q'], cwd=env.fork) + + _update(env, 'origin/master') + + assert _is_ancestor(env, env.fork, tip) + assert_gitnested_field(env.fork / 'sub' / '.gitnested', commit=pinned) + _assert_flat(env, env.fork) + + +def test_update_a_tag(fork): + env = fork + pinned = _upstream_bumps_sub(env) + env.run(['git', 'tag', 'v2'], cwd=env.workspace / 'bar') + env.run(['git', 'push', '-q', 'origin', 'v2'], cwd=env.workspace / 'bar') + _upstream_commit(env, 'after-v2') + env.run(['git', 'fetch', '-q', '--tags'], cwd=env.fork) + + _update(env, 'v2') + + assert_gitnested_field(env.fork / 'sub' / '.gitnested', commit=pinned) + assert not (env.fork / 'after-v2').exists() + _assert_flat(env, env.fork) + + +def test_update_does_not_fetch(fork): + env = fork + _upstream_commit(env, 'one') + + result = _update(env, 'origin/master') + + assert 'already up to date' in result.output + + +def test_update_needs_a_known_commit(fork): + env = fork + + result = _update(env, 'nope', check=False) + + assert result.returncode == 1 + assert 'nope: no such commit. Fetch it first.' in result.output + + +def test_update_resolves_relative_urls_against_the_remote_of_the_ref(foo_bar_cloned): + env = foo_bar_cloned + bar = env.workspace / 'bar' + env.run(['git', *ALLOW_FILE, 'submodule', 'add', '-q', '../foo', 'sub'], cwd=bar) + _push_bar(env, 'add sub') + forks = env.tmp / 'forks' + env.run(['git', 'clone', '-q', '--bare', env.upstream / 'bar', forks / 'bar']) + env.fork = env.workspace / 'fork' + clone_repo((forks / 'bar').as_posix(), env.fork) + env.run(['git', 'remote', 'add', 'upstream', env.upstream / 'bar'], cwd=env.fork) + cmd_git_nested(['submodules', 'init', '--resolve-relative', (env.upstream / 'bar').as_posix()], cwd=env.fork) + _upstream_adds_submodule(env, env.upstream / 'foo', 'lib') + env.run(['git', 'submodule', 'set-url', 'lib', '../foo'], cwd=bar) + _push_bar(env, 'lib is relative') + env.run(['git', 'fetch', '-q', 'upstream'], cwd=env.fork) + + _update(env, 'upstream/master') + + assert_gitnested_field(env.fork / 'lib' / '.gitnested', remote=(env.upstream / 'foo').as_posix()) + _assert_flat(env, env.fork) + + +# ============================================================================ +# --dry-run and --single-commit +# ============================================================================ + + +def test_update_dry_run_changes_nothing(fork): + env = fork + env.modify_files('sub/Foo', text='local', cwd=env.fork) + _upstream_adds_submodule(env, env.upstream / 'foo', 'lib') + env.run(['git', 'rm', '-q', 'sub'], cwd=env.workspace / 'bar') + _push_bar(env, 'drop sub') + head = git_rev_parse(['HEAD'], cwd=env.fork) + + result = _fetch_and_update(env, '--dry-run') + + assert 'would merge 2 commits of origin/master' in result.output + assert 'lib: would nest at' in result.output + assert 'sub: would remove (your changes in it go too' in result.output + assert git_rev_parse(['HEAD'], cwd=env.fork) == head + assert env.run(['git', 'status', '--porcelain'], cwd=env.fork).output == '' + assert (env.fork / 'sub' / '.gitnested').is_file() + assert _fetch_and_update(env).returncode == 0 + + +def test_update_dry_run_shows_a_pull(fork): + env = fork + pinned = _upstream_bumps_sub(env) + env.run(['git', 'fetch', '-q'], cwd=env.fork) + + result = _update(env, '--dry-run', 'origin/master') + + assert f'sub: would pull to {pinned[:12]}' in result.output + + +def test_single_commit(fork): + """One merge commit holds the upstream commits and the submodule changes.""" + env = fork + pinned = _upstream_bumps_sub(env) + _upstream_adds_submodule(env, env.upstream / 'foo', 'lib') + + _fetch_and_update(env, '--single-commit') + + assert git_rev_parse(['HEAD'], cwd=env.fork) == _last_update(env, env.fork) + message = env.run(['git', 'log', '-1', '--format=%B'], cwd=env.fork).output + assert message.startswith('Merge') + assert 'git nested' in message + assert_gitnested_field(env.fork / 'sub' / '.gitnested', commit=pinned) + assert (env.fork / 'lib' / '.gitnested').is_file() + _assert_flat(env, env.fork) + + +def test_options_that_do_not_go_together(fork): + env = fork + for args in (['--continue', '--dry-run'], ['--abort', 'origin', 'master'], ['--continue', '--abort']): + assert _fetch_and_update(env, *args, check=False).returncode == 1 + result = cmd_git_nested('submodules init --dry-run', cwd=env.fork, check=False) + assert result.returncode == 1 + + +# ============================================================================ +# A shorter way through a conflict, and keeping a submodule back +# ============================================================================ + + +def test_continue_finishes_a_conflict_inside_a_submodule(fork): + """Fix the files where the message says, 'git add' them, then --continue. No 'git nested commit'.""" + env = fork + env.modify_files('sub/Foo', text='local', cwd=env.fork) + other = env.workspace / 'foo' + env.modify_files('Foo', text='upstream', cwd=other) + env.run(['git', 'push', '-q'], cwd=other) + env.run(['git', 'pull', '-q'], cwd=env.workspace / 'bar' / 'sub') + pinned = git_rev_parse(['HEAD'], cwd=other) + _upstream_adds_submodule(env, env.upstream / 'foo', 'zlib') + + result = _fetch_and_update(env, check=False) + + assert result.returncode == 1 + assert 'submodules update --continue' in result.output + worktree = env.fork / '.git' / 'tmp' / 'nested' / 'sub' + (worktree / 'Foo').write_text('both\n') + env.run(['git', 'add', 'Foo'], cwd=worktree) + + _update(env, '--continue') + + assert (env.fork / 'sub' / 'Foo').read_text() == 'both\n' + assert_gitnested_field(env.fork / 'sub' / '.gitnested', commit=pinned) + assert (env.fork / 'zlib' / '.gitnested').is_file() + _assert_flat(env, env.fork) + + +def test_continue_says_which_conflicts_are_left(fork): + env = fork + env.modify_files('sub/Foo', text='local', cwd=env.fork) + other = env.workspace / 'foo' + env.modify_files('Foo', text='upstream', cwd=other) + env.run(['git', 'push', '-q'], cwd=other) + env.run(['git', 'pull', '-q'], cwd=env.workspace / 'bar' / 'sub') + _push_bar(env, 'bump sub') + _fetch_and_update(env, check=False) + + result = _update(env, '--continue', check=False) + + assert result.returncode == 1 + assert 'sub: fix the conflicts in' in result.output + assert 'Foo' in result.output + + +def test_a_pinned_submodule_stays(fork): + env = fork + before = git_rev_parse(['HEAD'], cwd=env.workspace / 'bar' / 'sub') + cmd_git_nested(['config', 'sub', 'pin', before], env.fork) + env.run(['git', 'commit', '-qm', 'keep sub'], cwd=env.fork) + _upstream_bumps_sub(env) + + result = _fetch_and_update(env) + + assert f'sub: it has the pin {before}, so it stays' in result.output + assert_gitnested_field(env.fork / 'sub' / '.gitnested', commit=before) + assert not (env.fork / 'sub' / 'FooNew').exists() + + +# ============================================================================ +# Less common paths +# ============================================================================ + + +def test_wrong_arguments(fork): + env = fork + assert 'too many arguments' in cmd_git_nested('submodules init upstream', cwd=env.fork, check=False).output + assert "--only only works with 'submodules init'" in _update(env, 'x', '--only', 'sub', check=False).output + assert "--ignore only works with 'submodules init'" in _update(env, 'x', '--ignore', 'sub', check=False).output + assert "--recursive only works with 'submodules init'" in _update(env, 'x', '--recursive', check=False).output + result = _update(env, 'x', '--resolve-relative', 'y', check=False) + assert "--resolve-relative only works with 'submodules init'" in result.output + result = cmd_git_nested('submodules init --only sub --ignore sub', cwd=env.fork, check=False) + assert "--only and --ignore can't be used together" in result.output + assert "'submodules update' needs a commit" in _update(env, check=False).output + assert 'too many arguments' in _update(env, 'a', 'b', check=False).output + + +def test_update_after_a_stop_needs_continue_or_abort(fork): + env = fork + _conflict_in_bar(env) + _fetch_and_update(env, check=False) + env.run(['git', 'merge', '--abort'], cwd=env.fork) + + result = _fetch_and_update(env, check=False) + + assert "A 'submodules update' stopped before." in result.output + + +def test_single_commit_without_submodule_changes(fork): + env = fork + _upstream_commit(env, 'one') + + _fetch_and_update(env, '--single-commit') + + assert git_rev_parse(['HEAD'], cwd=env.fork) == _last_update(env, env.fork) + + +def test_a_gitlink_without_url_stays_as_it_is(fork): + env = fork + bar = env.workspace / 'bar' + commit = git_rev_parse(['HEAD'], cwd=bar) + env.run(['git', 'update-index', '--add', '--cacheinfo', f'160000,{commit},nourl'], cwd=bar) + env.run(['git', 'commit', '-qm', 'gitlink only'], cwd=bar) + env.run(['git', 'push', '-q'], cwd=bar) + + result = _fetch_and_update(env) + + assert 'nourl: .gitmodules has no url for it, so it stays as it is' in result.output + _assert_flat(env, env.fork) + + +def test_a_teammates_clone_of_the_fork(fork): + """A plain clone of the fork has no upstream objects of the nested repos. A pull works there too.""" + env = fork + shared = env.tmp / 'shared-fork' + # --no-local: copy only what a clone over the network gets + env.run(['git', 'clone', '-q', '--bare', '--no-local', env.fork, shared]) + teammate = env.workspace / 'teammate' + clone_repo(shared.as_posix(), teammate) + env.run(['git', 'remote', 'add', 'upstream', env.upstream / 'bar'], cwd=teammate) + env.modify_files('sub/Foo', text='mine', cwd=teammate) + env.run(['git', 'rm', '-q', 'sub'], cwd=env.workspace / 'bar') + _push_bar(env, 'drop sub') + env.fork = teammate + + _fetch_and_update(env, remote='upstream') + + assert not (teammate / 'sub').exists() + _assert_flat(env, teammate) + + +# ============================================================================ +# The state file of a merge that stopped +# ============================================================================ + + +def _state_file(env): + return env.fork / '.git' / 'nested-submodules-update.json' + + +@pytest.mark.parametrize('text', ['not json', '{}', '{"base": "x", "target": "y", "url": ""}']) +def test_continue_with_a_broken_state_file(fork, text): + env = fork + _conflict_in_bar(env) + _fetch_and_update(env, check=False) + _state_file(env).write_text(text) + + result = _update(env, '--continue', check=False) + + assert result.returncode == 1 + assert 'is broken. Start over with --abort.' in result.output + + +def test_single_commit_after_a_conflict(fork): + env = fork + pinned = _conflict_in_bar(env) + _fetch_and_update(env, '--single-commit', check=False) + (env.fork / 'Bar').write_text('both\n') + env.run(['git', 'add', 'Bar'], cwd=env.fork) + + _update(env, '--continue') + + assert git_rev_parse(['HEAD'], cwd=env.fork) == _last_update(env, env.fork) + assert_gitnested_field(env.fork / 'sub' / '.gitnested', commit=pinned) + _assert_flat(env, env.fork) + + +def test_single_commit_when_the_merge_commit_is_gone(fork): + """You threw the merge away and moved to upstream itself. There is nothing to fold into.""" + env = fork + _conflict_in_bar(env) + _fetch_and_update(env, '--single-commit', check=False) + env.run(['git', 'merge', '--abort'], cwd=env.fork) + env.run(['git', 'reset', '-q', '--hard', 'origin/master'], cwd=env.fork) + + result = _update(env, '--continue', check=False) + + assert result.returncode == 1 + assert 'is gone' in result.output def test_init_refuses_only_with_ignore(foo_bar_cloned): @@ -279,3 +1134,350 @@ def test_init_resolves_relative_urls_only_when_asked(foo_bar_cloned): cmd_git_nested(['submodules', 'init', '--resolve-relative', (env.upstream / 'bar').as_posix()], cwd=repo) assert_gitnested_field(repo / 'sub' / '.gitnested', remote=(env.upstream / 'foo').as_posix()) + + +# ============================================================================ +# Nested repos with patches +# ============================================================================ + +PATCHES = '.gitnested.patches' + + +def _commit(env, message): + env.run(['git', 'add', '-A'], cwd=env.fork) + env.run(['git', 'commit', '-q', '-m', message], cwd=env.fork) + + +def _with_patch(env, path='sub'): + """Turn the patches on for `path`, change its Foo, and keep that change as a patch.""" + cmd_git_nested(['config', path, 'patches', PATCHES], cwd=env.fork) + # bytes: write_text would make it CRLF on Windows + (env.fork / path / 'Foo').write_bytes(b"local\n") + _commit(env, 'local change') + cmd_git_nested(['patches', 'create', path], cwd=env.fork) + _commit(env, 'the patch') + + +def _verified(env, path='sub'): + return cmd_git_nested(['patches', 'verify', path], cwd=env.fork).output.strip().endswith(" with 1 patch") + + +def _upstream_changes_foo(env, text): + """Upstream foo changes Foo, and bar pins it. Return the pin.""" + foo = env.workspace / 'foo' + (foo / 'Foo').write_bytes(text) + env.run(['git', 'commit', '-qam', 'upstream Foo'], cwd=foo) + env.run(['git', 'push', '-q'], cwd=foo) + env.run(['git', 'pull', '-q'], cwd=env.workspace / 'bar' / 'sub') + _push_bar(env, 'bump sub') + return git_rev_parse(['HEAD'], cwd=foo) + + +def test_update_applies_the_patches_on_the_new_pin(fork): + env = fork + _with_patch(env) + pinned = _upstream_bumps_sub(env) + + _fetch_and_update(env) + + assert (env.fork / 'sub' / 'Foo').read_text() == "local\n" + assert (env.fork / 'sub' / 'FooNew').is_file() + assert_gitnested_field(env.fork / 'sub' / '.gitnested', commit=pinned) + assert _verified(env) + _assert_flat(env, env.fork) + + +def test_update_drops_a_patch_upstream_has(fork): + env = fork + _with_patch(env) + _upstream_changes_foo(env, b"local\n") + + _fetch_and_update(env) + + assert (env.fork / 'sub' / 'Foo').read_text() == "local\n" + assert not list((env.fork / 'sub' / PATCHES).glob('*.patch')) + _assert_flat(env, env.fork) + + +def test_continue_finishes_a_patch_that_does_not_apply(fork): + env = fork + _with_patch(env) + pinned = _upstream_changes_foo(env, b"upstream\n") + + result = _fetch_and_update(env, check=False) + + assert result.returncode == 1 + assert "The patch 0001-TODO-title.patch does not apply." in result.output + assert 'submodules update --continue' in result.output + worktree = env.fork / '.git' / 'tmp' / 'nested' / 'sub' + (worktree / 'Foo').write_bytes(b"upstream and local\n") + env.run(['git', 'add', 'Foo'], cwd=worktree) + + _update(env, '--continue') + + assert (env.fork / 'sub' / 'Foo').read_text() == "upstream and local\n" + assert_gitnested_field(env.fork / 'sub' / '.gitnested', commit=pinned) + assert "+upstream and local" in (env.fork / 'sub' / PATCHES / '0001-TODO-title.patch').read_text() + assert _verified(env) + _assert_flat(env, env.fork) + + +def test_update_moves_the_patches_with_a_moved_submodule(fork): + env = fork + _with_patch(env) + (env.workspace / 'bar' / 'ext').mkdir() + env.run(['git', 'mv', 'sub', 'ext/sub'], cwd=env.workspace / 'bar') + _push_bar(env, 'move sub') + + _fetch_and_update(env) + + assert (env.fork / 'ext' / 'sub' / 'Foo').read_text() == "local\n" + assert _verified(env, 'ext/sub') + _assert_flat(env, env.fork) + + +def test_update_warns_when_it_removes_a_submodule_with_patches(fork): + env = fork + _with_patch(env) + env.run(['git', 'rm', '-q', 'sub'], cwd=env.workspace / 'bar') + _push_bar(env, 'drop sub') + + result = _fetch_and_update(env) + + assert not (env.fork / 'sub').exists() + assert 'sub' in result.output + _assert_flat(env, env.fork) + + +def test_continue_finishes_a_rebase_that_stopped(fork): + """Without patches, but with method rebase: the pull stops in a rebase.""" + env = fork + cmd_git_nested(['config', 'sub', 'method', 'rebase'], cwd=env.fork) + (env.fork / 'sub' / 'Foo').write_bytes(b"local\n") + _commit(env, 'local change') + pinned = _upstream_changes_foo(env, b"upstream\n") + + assert _fetch_and_update(env, check=False).returncode == 1 + worktree = env.fork / '.git' / 'tmp' / 'nested' / 'sub' + (worktree / 'Foo').write_bytes(b"upstream and local\n") + env.run(['git', 'add', 'Foo'], cwd=worktree) + + _update(env, '--continue') + + assert (env.fork / 'sub' / 'Foo').read_text() == "upstream and local\n" + assert_gitnested_field(env.fork / 'sub' / '.gitnested', commit=pinned) + _assert_flat(env, env.fork) + + +def test_continue_after_you_finished_the_patch_yourself(fork): + env = fork + _with_patch(env) + _upstream_changes_foo(env, b"upstream\n") + assert _fetch_and_update(env, check=False).returncode == 1 + worktree = env.fork / '.git' / 'tmp' / 'nested' / 'sub' + (worktree / 'Foo').write_bytes(b"upstream and local\n") + env.run(['git', 'add', 'Foo'], cwd=worktree) + env.run(['git', '-c', 'core.editor=true', 'am', '--continue'], cwd=worktree) + + _update(env, '--continue') + + assert (env.fork / 'sub' / 'Foo').read_text() == "upstream and local\n" + assert _verified(env) + _assert_flat(env, env.fork) + + +def test_update_keeps_what_git_skips(fork): + """git does not check out a submodule with `update = none`. update leaves it a submodule too.""" + env = fork + _upstream_adds_submodule(env, env.upstream / 'foo', 'lib') + _upstream_adds_submodule(env, env.upstream / 'foo', 'big') + env.run(['git', 'config', '-f', '.gitmodules', 'submodule.big.update', 'none'], cwd=env.workspace / 'bar') + _push_bar(env, 'skip big') + + preview = _fetch_and_update(env, '--dry-run').output + _fetch_and_update(env) + + assert 'lib: would nest at' in preview + assert 'big' not in preview + assert (env.fork / 'lib' / '.gitnested').is_file() + assert _gitlinks(env, env.fork) == ['big'] + assert _module(env, 'submodule.big.update') == 'none' + + +def test_init_keeps_what_git_skips(foo_bar_cloned): + env = foo_bar_cloned + _upstream_adds_submodule(env, _make_mid(env), 'mid') + _upstream_adds_submodule(env, env.upstream / 'foo', 'big') + env.run(['git', 'config', '-f', '.gitmodules', 'submodule.big.update', 'none'], cwd=env.workspace / 'bar') + _push_bar(env, 'skip big') + mid = env.workspace / 'mid' + env.run(['git', 'config', '-f', '.gitmodules', 'submodule.inner.update', 'none'], cwd=mid) + env.run(['git', 'commit', '-qam', 'skip inner'], cwd=mid) + env.run(['git', 'push', '-q', 'origin', 'master'], cwd=mid) + env.run(['git', 'pull', '-q', 'origin', 'master'], cwd=env.workspace / 'bar' / 'mid') + _push_bar(env, 'bump mid') + repo = env.workspace / 'repo' + clone_repo((env.upstream / 'bar').as_posix(), repo) + + result = cmd_git_nested('submodules init --recursive', cwd=repo) + + assert "mid/inner: stays a submodule, .gitmodules says 'update = none'" in result.output + assert _gitlinks(env, repo) == ['big', 'mid/inner'] + + +def test_update_after_abort_updates_the_rest(fork): + """You finished a nested repo by hand and threw the rest away. update does the rest.""" + env = fork + env.modify_files('sub/Foo', text='local', cwd=env.fork) + other = env.workspace / 'foo' + env.modify_files('Foo', text='upstream', cwd=other) + env.run(['git', 'push', '-q'], cwd=other) + env.run(['git', 'pull', '-q'], cwd=env.workspace / 'bar' / 'sub') + pinned = git_rev_parse(['HEAD'], cwd=other) + _upstream_adds_submodule(env, env.upstream / 'foo', 'zlib') + assert _fetch_and_update(env, check=False).returncode == 1 + worktree = env.fork / '.git' / 'tmp' / 'nested' / 'sub' + (worktree / 'Foo').write_text('both\n') + env.run(['git', 'add', 'Foo'], cwd=worktree) + env.run(['git', 'commit', '-q', '--no-edit'], cwd=worktree) + cmd_git_nested('commit sub', cwd=env.fork) + assert 'aborted' in _update(env, '--abort').output + assert not (env.fork / 'zlib').exists() + + assert 'zlib: would nest at' in _update(env, 'origin/master', '--dry-run').output + _update(env, 'origin/master') + + assert (env.fork / 'zlib' / '.gitnested').is_file() + assert_gitnested_field(env.fork / 'sub' / '.gitnested', commit=pinned) + _assert_flat(env, env.fork) + assert 'already up to date with origin/master' in _update(env, 'origin/master').output + + +def test_catching_up_has_no_single_commit(fork): + """origin/master is merged already. There is no merge to fold the commits into.""" + result = _update(fork, 'origin/master', '--single-commit', check=False) + + assert result.returncode == 1 + assert "--single-commit only works when 'submodules update' merges" in result.output + + +# ============================================================================ +# What a tester found on a big project (issue #184) +# ============================================================================ + + +def _conflict_in_sub(env): + """A local change in sub, and upstream changes the same line. Return the new pin.""" + env.modify_files('sub/Foo', text='local', cwd=env.fork) + other = env.workspace / 'foo' + env.modify_files('Foo', text='upstream', cwd=other) + env.run(['git', 'push', '-q'], cwd=other) + env.run(['git', 'pull', '-q'], cwd=env.workspace / 'bar' / 'sub') + _push_bar(env, 'bump sub') + return git_rev_parse(['HEAD'], cwd=other) + + +def test_abort_removes_the_worktree_of_a_stopped_nested_repo(fork): + env = fork + _conflict_in_sub(env) + assert _fetch_and_update(env, check=False).returncode == 1 + worktree = env.fork / '.git' / 'tmp' / 'nested' / 'sub' + assert worktree.is_dir() + + _update(env, '--abort') + + assert not worktree.exists() + assert 'nested/sub' not in env.run(['git', 'worktree', 'list'], cwd=env.fork).output + + +def test_a_conflict_in_a_nested_repo_shows_the_submodules_way_only(fork): + env = fork + _conflict_in_sub(env) + + result = _fetch_and_update(env, check=False) + + assert result.returncode == 1 + assert 'git nested commit' not in result.output + assert 'sub: the update stopped on conflicts in Foo' in result.output + assert 'cd .git/tmp/nested/sub' in result.output + assert "'git nested submodules update --continue'" in result.output + assert "'git nested submodules update --abort'" in result.output + + +def test_dry_run_shows_the_deeper_submodules(foo_bar_cloned): + env = foo_bar_cloned + _upstream_adds_submodule(env, _make_mid(env), 'mid') + env.fork = env.workspace / 'fork' + clone_repo((env.upstream / 'bar').as_posix(), env.fork) + cmd_git_nested('submodules init --recursive', cwd=env.fork) + pinned = _upstream_bumps_inner_of_mid(env) + env.run(['git', 'fetch', '-q'], cwd=env.fork) + + result = _update(env, 'origin/master', '--dry-run') + + assert 'mid: would pull to' in result.output + assert f'mid/inner: would pull to {pinned[:12]}' in result.output + _update(env, 'origin/master') + assert_gitnested_field(env.fork / 'mid' / 'inner' / '.gitnested', commit=pinned) + + +def test_the_gitattributes_block_does_not_conflict(foo_bar_cloned): + """git-nested owns its block at the end of .gitattributes. Upstream's new rules come in above it.""" + env = foo_bar_cloned + foo = env.workspace / 'foo' + (foo / 'make.bat').write_bytes(b"echo hi\r\n") + env.run(['git', '-c', 'core.autocrlf=false', 'add', 'make.bat'], cwd=foo) + env.run(['git', 'commit', '-qm', 'make.bat'], cwd=foo) + env.run(['git', 'push', '-q'], cwd=foo) + bar = env.workspace / 'bar' + (bar / '.gitattributes').write_text("*.bat text eol=crlf\n") + env.run(['git', 'add', '.gitattributes'], cwd=bar) + _upstream_adds_submodule(env, env.upstream / 'foo', 'sub') + env.fork = env.workspace / 'fork' + clone_repo((env.upstream / 'bar').as_posix(), env.fork) + cmd_git_nested('submodules init', cwd=env.fork) + assert '/sub/make.bat -text' in (env.fork / '.gitattributes').read_text() + with (bar / '.gitattributes').open('a') as f: + f.write("*.bin binary\n") + _push_bar(env, 'bin files') + + _fetch_and_update(env) + + rules = (env.fork / '.gitattributes').read_text() + assert rules.index('*.bin binary') < rules.index('# git-nested') + assert rules.rstrip().endswith('/sub/make.bat -text') + assert env.run(['git', 'status', '--porcelain'], cwd=env.fork).output == '' + + +def test_dry_run_shows_a_new_deeper_submodule_and_skips_an_unchanged_one(foo_bar_cloned): + env = foo_bar_cloned + _upstream_adds_submodule(env, _make_mid(env), 'mid') + env.fork = env.workspace / 'fork' + clone_repo((env.upstream / 'bar').as_posix(), env.fork) + cmd_git_nested('submodules init --recursive', cwd=env.fork) + mid = env.workspace / 'mid' + env.run(['git', *ALLOW_FILE, 'submodule', 'add', '-q', (env.upstream / 'foo').as_posix(), 'extra'], cwd=mid) + env.run(['git', 'commit', '-qm', 'add extra'], cwd=mid) + env.run(['git', 'push', '-q', 'origin', 'master'], cwd=mid) + env.run(['git', 'pull', '-q', 'origin', 'master'], cwd=env.workspace / 'bar' / 'mid') + _push_bar(env, 'bump mid') + env.run(['git', 'fetch', '-q'], cwd=env.fork) + + preview = _update(env, 'origin/master', '--dry-run').output + + assert 'mid/extra: would nest at' in preview + assert 'mid/inner' not in preview + + +def test_a_failed_pull_of_a_nested_repo_shows_its_own_error(fork): + """No conflict, so there is no worktree to fix things in.""" + env = fork + cmd_git_nested(['config', 'sub', 'remote', (env.tmp / 'gone').as_posix()], cwd=env.fork) + env.run(['git', 'commit', '-qam', 'sub from somewhere else'], cwd=env.fork) + _upstream_bumps_sub(env) + + result = _fetch_and_update(env, check=False) + + assert result.returncode == 1 + assert 'gone' in result.output + assert 'the update stopped on conflicts' not in result.output diff --git a/tests/unit/test_command.py b/tests/unit/test_command.py index df424bb4..bd216950 100644 --- a/tests/unit/test_command.py +++ b/tests/unit/test_command.py @@ -157,6 +157,22 @@ def test_parse_args_method_shorthand_sets_the_method_flag(cmd, args, method): assert ctx.flags.method == method +@pytest.mark.parametrize( + 'args', + [ + ['submodules', 'update', '--dry-run', 'upstream/main'], + ['submodules', 'update', 'upstream/main', '--dry-run'], + ['submodules', '--dry-run', 'update', 'upstream/main'], + ], +) +def test_parse_args_submodules_options_go_anywhere(cmd, args): + """Before Python 3.12, argparse failed when an option stood between the action and its arguments.""" + _, ctx = cmd.parse_args(args) + assert ctx.flags.dry_run + assert ctx.submodules_action == 'update' + assert ctx.submodules_args == ['upstream/main'] + + @pytest.mark.parametrize( 'args', [ @@ -168,5 +184,6 @@ def test_parse_args_method_shorthand_sets_the_method_flag(cmd, args, method): def test_parse_args_submodules_options_keep_their_value(cmd, args): _, ctx = cmd.parse_args(args) assert ctx.submodules_action == 'init' + assert ctx.submodules_args == [] assert ctx.flags.only == ['lib/core', 'third_party'] assert ctx.flags.resolve_relative == 'u'