From eda7678343afb72dec322e6326f4145a25e2392d Mon Sep 17 00:00:00 2001 From: Thorsten Klein Date: Sat, 14 Nov 2026 09:27:42 +0100 Subject: [PATCH] feat: add `submodules init` to nest all submodules at once `git nested submodules init` turns each submodule into a nested repo, at the commit it pins. Each one gets a commit. - `--recursive` nests their submodules too, at any depth. - `--only` and `--ignore` pick some of them. `init` keeps the choice in .gitmodules. - `--resolve-relative ` resolves relative submodule urls. Without it, `init` stops at one. - The choice goes into .gitnested as `submodules: nest` or `keep`. - A branch or push of a nested repo inside a nested repo leaves the outer one alone. Refs #184 --- README.md | 7 + doc/architecture/diagrams.md | 4 + doc/cli/config.md | 5 +- doc/cli/index.md | 1 + doc/cli/init.md | 1 + doc/cli/submodules.md | 89 ++++++++ doc/concepts/comparison.md | 3 +- doc/configuration/gitnested-file.md | 10 + doc/guides/submodules.md | 15 ++ doc/index.md | 2 + man/man1/git-nested.1 | 3 + src/git_nested/cli/location.py | 13 +- src/git_nested/cli/parser.py | 4 + src/git_nested/cli/spec.py | 39 +++- src/git_nested/commands/__init__.py | 2 + src/git_nested/commands/clone.py | 18 +- src/git_nested/commands/config.py | 1 + src/git_nested/commands/init.py | 11 +- src/git_nested/commands/pull.py | 10 +- src/git_nested/commands/submodules.py | 116 ++++++++++ src/git_nested/commands/verify.py | 6 +- src/git_nested/gitfile.py | 5 +- src/git_nested/history.py | 8 +- src/git_nested/models.py | 10 + src/git_nested/submodules.py | 16 +- tests/conftest.py | 2 + tests/e2e/test_issue_184.py | 121 ++++++++++ tests/e2e/test_submodules_command.py | 281 ++++++++++++++++++++++++ tests/e2e/test_submodules_contribute.py | 196 +++++++++++++++++ tests/unit/test_command.py | 15 ++ 30 files changed, 985 insertions(+), 29 deletions(-) create mode 100644 doc/cli/submodules.md create mode 100644 src/git_nested/commands/submodules.py create mode 100644 tests/e2e/test_issue_184.py create mode 100644 tests/e2e/test_submodules_command.py create mode 100644 tests/e2e/test_submodules_contribute.py diff --git a/README.md b/README.md index b22cdcc6..bfd405d6 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,13 @@ git nested status # see where everything st A small `.gitnested` file in the folder remembers the URL, the branch and the upstream commit. So the commands stay short. +Your repository has submodules already? Turn them all into nested +repositories: + +```bash +git nested submodules init --recursive +``` + ## Why git-nested? - **Self-contained.** Your repository has all files. A plain `git clone` diff --git a/doc/architecture/diagrams.md b/doc/architecture/diagrams.md index b3c1749c..3a738e45 100644 --- a/doc/architecture/diagrams.md +++ b/doc/architecture/diagrams.md @@ -781,6 +781,10 @@ sequenceDiagram --- +## submodules + +`init` runs `init --from-submodule` for each submodule. + ## diff ```mermaid diff --git a/doc/cli/config.md b/doc/cli/config.md index 15d90b6e..d1d18716 100644 --- a/doc/cli/config.md +++ b/doc/cli/config.md @@ -46,6 +46,7 @@ instead of editing the file by hand. | `patches` | the folder with the local changes as patches, relative to the subdir | yes | | `patches-ignore` | paths in the subdir that stay local, so no patch needs their changes | no | | `patches-local` | patches a push leaves out, by file name | no | +| `submodules` | `nest` or `keep`: whether a pull nests the submodules in it. Unset: nest | yes | The other keys are written by the commands themselves. Setting one by hand would describe a state the repository is not in. @@ -68,5 +69,5 @@ git commit -m "lib: pull with rebase, never push" - A relative path as a `remote` value is taken relative to the folder you are in, like git does. It is stored relative to the top of your repository. -- An unknown key, or a bad value for `method`, `squash` or `readonly`, is - refused. +- An unknown key, or a bad value for `method`, `squash`, `readonly` or + `submodules`, is refused. diff --git a/doc/cli/index.md b/doc/cli/index.md index 7c12a6eb..ff1b7967 100644 --- a/doc/cli/index.md +++ b/doc/cli/index.md @@ -16,6 +16,7 @@ Each command has its own page. This page lists what they share. | [`push`](push.md) | push a nested repository's local changes upstream | | [`fetch`](fetch.md) | fetch upstream content for a nested repository | | [`status`](status.md) | report on the nested repositories of this repository | +| [`submodules`](submodules.md) | turn all submodules into nested repositories | | [`diff`](diff.md) | show a nested repository's local changes against upstream | | [`patches`](patches.md) | write, check, drop or cherry-pick the patches of a nested repository | | [`verify`](verify.md) | check that a nested repository is its upstream commit plus patches or diffs | diff --git a/doc/cli/init.md b/doc/cli/init.md index a70c8de3..a617e0fd 100644 --- a/doc/cli/init.md +++ b/doc/cli/init.md @@ -91,6 +91,7 @@ git nested init --from-submodule ext/lib This removes the submodule and clones the same URL at the same commit into its place. A relative submodule URL is resolved against the `origin` remote of your repository. See [Upstream submodules](../guides/submodules.md). +To nest all submodules at once, use [`submodules init`](submodules.md). ## Notes diff --git a/doc/cli/submodules.md b/doc/cli/submodules.md new file mode 100644 index 00000000..386ea737 --- /dev/null +++ b/doc/cli/submodules.md @@ -0,0 +1,89 @@ +# `git nested submodules` + +Turn all submodules into nested repositories. + +```text +git nested submodules init [--recursive] [--resolve-relative ] [--only | --ignore ]... +``` + +## What it does + +Use it for a repository whose submodules should be plain files. You get one +repository with all the code in it. + +The repository itself stays a normal git clone of its upstream. Each +submodule becomes a nested repository. + +`init` nests every submodule at the commit it pins. Each submodule gets one +commit. With `--recursive`, it nests their submodules too. With `--only` or +`--ignore`, it nests only some of them. + +No gitlink and no `.gitmodules` entry stays for a nested submodule. + +## Options + +| Option | Meaning | +|---|---| +| `--only ` | nest only the submodules at or below this path. The others stay submodules. Give it more than once for more paths | +| `--ignore ` | nest all submodules but the ones at or below this path. Give it more than once for more paths. Not with `--only` | +| `--recursive` | nest the submodules of the submodules too, at any depth. Without it, they stay submodules. The choice goes into each `.gitnested` as [`submodules`](../configuration/gitnested-file.md) | +| `--resolve-relative ` | resolve relative submodule urls against this url. Without it, `init` stops at a relative url | +| `-S`, `--gpg-sign` | GPG-sign the commits | +| `--signoff` | add a `Signed-off-by:` trailer to the commits | + +## Examples + +```bash +git clone https://github.com/user/project +cd project +git nested submodules init --recursive +``` + +### Relative urls + +A url like `../lib.git` is relative to the url of the superproject. In a +fork, that is not `origin`. So `init` stops and asks for the url: + +```bash +git nested submodules init --resolve-relative https://github.com/org/project +``` + +Now `../lib.git` is `https://github.com/org/lib.git`. This is only for +the first level. With `--recursive`, a deeper relative url is relative to +the url of its parent submodule, as in git. + +### Only some submodules + +Name the paths to nest: + +```bash +git nested submodules init --only lib/core --only third_party +``` + +This nests `lib/core` and every submodule below `third_party`. The others +stay submodules. + +Or name the ones to keep as submodules: + +```bash +git nested submodules init --ignore vendor/big +``` + +`init` keeps `--only` and `--ignore` in `.gitmodules`, in a `[nested]` +section. + +## Notes + +- `init` needs a clean working tree. +- If your `.gitattributes` would change files of a nested repository, for + example `*.bat text eol=crlf`, git-nested adds `-text` for these files to + `.gitattributes`. So they stay as upstream has them, as in a submodule. +- Push your changes to each submodule's upstream with + [`push`](push.md), as for any nested repository. + +## See also + +- [Upstream submodules](../guides/submodules.md) +- [`init --from-submodule`](init.md): nest one submodule. +- [`clone --recurse-submodules`](clone.md): nest a repository with + submodules in a subdirectory. diff --git a/doc/concepts/comparison.md b/doc/concepts/comparison.md index a942ec0d..fb76736d 100644 --- a/doc/concepts/comparison.md +++ b/doc/concepts/comparison.md @@ -31,7 +31,8 @@ your repository. With git-nested the files are part of your repository. A plain `git clone` gets them, and every commit is complete. -To switch, see [`git nested init --from-submodule`](../cli/init.md). +To switch, see [`git nested init --from-submodule`](../cli/init.md) for one +submodule, or [`git nested submodules`](../cli/submodules.md) for all of them. ## git subtree diff --git a/doc/configuration/gitnested-file.md b/doc/configuration/gitnested-file.md index 884a5146..6333a512 100644 --- a/doc/configuration/gitnested-file.md +++ b/doc/configuration/gitnested-file.md @@ -194,6 +194,16 @@ A name with no patch file is an error. So when a patch is renumbered or renamed, fix the list. A patch whose title starts with `git-nested local:` stays local too, whatever its number. +### `submodules` + +What happens to the submodules inside the nested repository: + +- `nest`: they become nested repositories too. +- `keep`: they stay submodules. + +`submodules init` writes it: `keep`, or `nest` with `--recursive`. Unset +means `nest`. + ### `cmdver` The git-nested version that last wrote the file. For information only. diff --git a/doc/guides/submodules.md b/doc/guides/submodules.md index 85cc8d69..6a3ded4f 100644 --- a/doc/guides/submodules.md +++ b/doc/guides/submodules.md @@ -53,3 +53,18 @@ This: A relative submodule URL is resolved against the `origin` remote of your repository. + +## Turn a whole repository into one + +Your repository has many submodules, maybe submodules in submodules. You +want all of them as plain files: + +```bash +git nested submodules init --recursive +``` + +`init --recursive` nests all of them, at any depth. Without +`--recursive`, it nests only the first level. `--only ` nests only +some. + +See [`submodules`](../cli/submodules.md). diff --git a/doc/index.md b/doc/index.md index 604dbc41..97b9f055 100644 --- a/doc/index.md +++ b/doc/index.md @@ -74,6 +74,7 @@ The full reference of the command line. There is one page per command. | [`push`](cli/push.md) | Send your changes upstream | | [`fetch`](cli/fetch.md) | Download upstream changes, change no file | | [`status`](cli/status.md) | Show all nested repositories | +| [`submodules`](cli/submodules.md) | Turn all submodules into nested repositories | | [`diff`](cli/diff.md) | Compare a nested repository with upstream | | [`patches`](cli/patches.md) | Keep local changes as patch files | | [`verify`](cli/verify.md) | Check local changes against patch files | @@ -100,6 +101,7 @@ cli/pull cli/push cli/fetch cli/status +cli/submodules cli/diff cli/patches cli/verify diff --git a/man/man1/git-nested.1 b/man/man1/git-nested.1 index ffba6755..bbb0610e 100644 --- a/man/man1/git-nested.1 +++ b/man/man1/git-nested.1 @@ -99,6 +99,9 @@ run a git command in the worktree of a nested repository .B status report on the nested repositories of this repository .TP +.B submodules +turn all submodules into nested repositories +.TP .B verify check that a nested repository is its upstream commit plus patches or diffs .TP diff --git a/src/git_nested/cli/location.py b/src/git_nested/cli/location.py index 9de2708f..480e8d9d 100644 --- a/src/git_nested/cli/location.py +++ b/src/git_nested/cli/location.py @@ -29,7 +29,7 @@ # Commands whose missing is not filled in from the current directory: # status reports on all of them, clone guesses one from the upstream, and init # is what makes a directory nested in the first place. -_NO_IMPLIED_SUBDIR = ('clone', 'completion', 'init', 'status', 'version') +_NO_IMPLIED_SUBDIR = ('clone', 'completion', 'init', 'status', 'submodules', 'version') def at_top_level(git: GitRunner, command: str, ctx: CommandContext) -> AbstractContextManager[object]: @@ -56,11 +56,13 @@ def at_top_level(git: GitRunner, command: str, ctx: CommandContext) -> AbstractC # relative one keeps working in every clone of the parent repository. ctx.upstream = _remote_from(prefix, ctx.upstream) ctx.flags.remote = _remote_from(prefix, ctx.flags.remote) + ctx.flags.resolve_relative = _remote_from(prefix, ctx.flags.resolve_relative) if ctx.config_key == 'remote': ctx.config_value = _remote_from(prefix, ctx.config_value) - # --ignore paths are given like . - ctx.flags.ignore = [_joined(prefix, Path(path)) for path in ctx.flags.ignore or []] + # --ignore and --only paths are given like . + ctx.flags.ignore = _all_joined(prefix, ctx.flags.ignore) + ctx.flags.only = _all_joined(prefix, ctx.flags.only) if ctx.subdir is not None: ctx.subdir = _joined(prefix, Path(ctx.subdir)) elif command not in _NO_IMPLIED_SUBDIR and not ctx.flags.all: @@ -99,6 +101,11 @@ def _remote_from(prefix: str, remote: str | None) -> str | None: return joined if joined.startswith('.') else f'./{joined}' +def _all_joined(prefix: str, paths: list[str] | None) -> list[str] | None: + """Each of `paths` joined like a . None stays None.""" + return [_joined(prefix, Path(path)) for path in paths] if paths else paths + + def _joined(prefix: str, subdir: Path) -> str: """`subdir`, typed below the top level at `prefix`, as a path from the top level.""" if subdir.anchor: diff --git a/src/git_nested/cli/parser.py b/src/git_nested/cli/parser.py index 53c1a2ac..e9364293 100644 --- a/src/git_nested/cli/parser.py +++ b/src/git_nested/cli/parser.py @@ -70,6 +70,9 @@ def _flags_from_args(args: argparse.Namespace) -> Flags: filter=getattr(args, 'filter', []), force=getattr(args, 'force', False), ignore=getattr(args, 'ignore', None), + only=getattr(args, 'only', None), + resolve_relative=getattr(args, 'resolve_relative', None), + recursive=getattr(args, 'recursive', False), from_submodule=getattr(args, 'from_submodule', False), from_patches=getattr(args, 'from_patches', False), local=getattr(args, 'local', False), @@ -210,5 +213,6 @@ def parse_args(git: GitRunner, args_list: list[str]) -> tuple[str, CommandContex patch_names=getattr(args, 'patch_names', None), patches=patches, verify_mode=verify_mode, + submodules_action=getattr(args, 'submodules_action', None), ) return args.command, context diff --git a/src/git_nested/cli/spec.py b/src/git_nested/cli/spec.py index 9b724154..d7d7db36 100644 --- a/src/git_nested/cli/spec.py +++ b/src/git_nested/cli/spec.py @@ -79,6 +79,7 @@ 'update', ], 'status': ['ALL', 'all', 'fetch', 'short'], + 'submodules': ['gpg_sign', 'ignore', 'only', 'recursive', 'resolve_relative', 'signoff'], 'verify': ['diffs', 'ignore', 'keep', 'output_dir', 'patches'], 'version': [], } @@ -101,6 +102,7 @@ 'repair': "point a nested repository's sync point back into the current history", 'run': "run a git command in the worktree of a nested repository", 'status': "report on the nested repositories of this repository", + 'submodules': "turn all submodules into nested repositories", 'verify': "check that a nested repository is its upstream commit plus patches or diffs", 'version': "print version information", } @@ -179,9 +181,28 @@ { 'action': 'append', 'metavar': 'PATH', - 'help': "skip this path of , just this time", + 'help': "skip this path of , just this time; submodules init: do not nest the submodules there", }, ), + ( + 'only', + ('--only',), + { + 'action': 'append', + 'metavar': 'PATH', + 'help': "init: nest only the submodules at or below this path; give it more than once for more", + }, + ), + ( + 'recursive', + ('--recursive',), + {'action': 'store_true', 'help': "init: nest the submodules of the submodules too, at any depth"}, + ), + ( + 'resolve_relative', + ('--resolve-relative',), + {'metavar': 'URL', 'help': "init: resolve relative submodule urls against this url"}, + ), ('exclude', ('--exclude',), {'action': 'append', 'help': "leave this path of the upstream repository out"}), ( 'gpg_sign', @@ -334,6 +355,7 @@ # everything after goes to git untouched, options included. ('git_args', {'nargs': '...', 'help': "the git command to run, with its arguments"}), ], + 'submodules': [('submodules_action', {'metavar': 'action', 'choices': ['init'], 'help': "init"})], 'verify': [('subdir', {'help': _SUBDIR_HELP})], } @@ -424,6 +446,15 @@ "Report every nested repository in this repository: where it comes from, and whether it has local changes. " "With -q, just the paths, one per line." ), + 'submodules': ( + "Turn the submodules of a repository into plain files. " + "Each submodule becomes a nested repository. " + "The repository itself stays a normal git clone of its upstream. " + "'init' nests all submodules at the commits they pin, one commit each. " + "With --recursive, their submodules too. " + "--only and --ignore pick some of them. " + "A relative submodule url needs --resolve-relative." + ), 'verify': ( "Apply the files to the upstream commit in .gitnested, in a temporary worktree, and compare " "the result with . Patches go in with git am; plain diffs need --diffs. " @@ -519,6 +550,12 @@ "git nested status", "git nested status -q", ], + 'submodules': [ + "git nested submodules init", + "git nested submodules init --recursive", + "git nested submodules init --only lib/core", + "git nested submodules init --resolve-relative https://github.com/org/project", + ], 'verify': [ "git nested verify ext/lib --patches patches/lib/*.patch", "git nested verify -o patches/lib ext/lib", diff --git a/src/git_nested/commands/__init__.py b/src/git_nested/commands/__init__.py index 632a5de3..349a1153 100644 --- a/src/git_nested/commands/__init__.py +++ b/src/git_nested/commands/__init__.py @@ -32,6 +32,7 @@ repair, run, status, + submodules, verify, version, ) @@ -52,6 +53,7 @@ 'repair': repair.cmd_repair, 'run': run.cmd_run, 'status': status.cmd_status, + 'submodules': submodules.cmd_submodules, 'verify': verify.cmd_verify, 'version': version.cmd_version, } diff --git a/src/git_nested/commands/clone.py b/src/git_nested/commands/clone.py index 6d9c02a2..0b6ac237 100644 --- a/src/git_nested/commands/clone.py +++ b/src/git_nested/commands/clone.py @@ -181,6 +181,14 @@ def _clone_into( content.keep_upstream_bytes(git, flags, subdir) if flags.recurse_submodules: nest_submodules(git, flags, config, subdir) + else: + _warn_about_submodules(git, flags, subdir) + + +def _warn_about_submodules(git: GitRunner, flags: Flags, subdir: Path) -> None: + """Say which submodules stay empty, and what nests them.""" + if flags.hint: + submodules.warn_if_any(git, subdir, f"use {flags.hint} to nest them") else: submodules.warn_if_any(git, subdir) @@ -206,10 +214,16 @@ def nest_submodules(git: GitRunner, flags: Flags, config: NestedConfig, subdir: def clone_at(git: GitRunner, flags: Flags, subdir: Path, url: str, commit: str) -> None: - """Clone `url` at `commit` into subdir, and its submodules too.""" + """Clone `url` at `commit` into subdir, and its submodules too. + + Not its submodules if flags.submodules is 'keep'. flags.submodules goes + into .gitnested, and to the submodules nested here. + """ inner = Flags( branch=commit, - recurse_submodules=True, + recurse_submodules=flags.submodules != 'keep', + submodules=flags.submodules, + hint=flags.hint, gpg_sign=flags.gpg_sign, signoff=flags.signoff, quiet=flags.quiet, diff --git a/src/git_nested/commands/config.py b/src/git_nested/commands/config.py index a57413c3..23ec6975 100644 --- a/src/git_nested/commands/config.py +++ b/src/git_nested/commands/config.py @@ -15,6 +15,7 @@ 'method': ('merge', 'rebase'), 'squash': ('true', 'false'), 'readonly': ('true', 'false'), + 'submodules': ('nest', 'keep'), } diff --git a/src/git_nested/commands/init.py b/src/git_nested/commands/init.py index 55034784..00ef6a26 100644 --- a/src/git_nested/commands/init.py +++ b/src/git_nested/commands/init.py @@ -16,8 +16,11 @@ _GITMODULES = Path('.gitmodules') -def _init_from_submodule(git: GitRunner, flags: Flags, subdir: Path) -> None: - """Replace the submodule at subdir by a nested repository, at the same commit and url.""" +def init_from_submodule(git: GitRunner, flags: Flags, subdir: Path, base: str | None = None) -> None: + """Replace the submodule at subdir by a nested repository, at the same commit and url. + + A relative url is relative to `base`, or else to the url of origin. + """ path = subdir.as_posix() found = submodules.entry(git, _GITMODULES, path) link = git.check_output(['ls-tree', 'HEAD', '--', path]).split() @@ -26,7 +29,7 @@ def _init_from_submodule(git: GitRunner, flags: Flags, subdir: Path) -> None: name, url = found pinned = link[2] # A relative url is relative to this repository's own remote. - url = submodules.resolve(url, git.check_output(['remote', 'get-url', 'origin'], may_fail=True)) + url = submodules.resolve(url, base or git.check_output(['remote', 'get-url', 'origin'], may_fail=True)) output.verbose(f"removing the submodule {name}") git.run(['submodule', 'deinit', '-q', '-f', '--', path], may_fail=True) @@ -74,7 +77,7 @@ def cmd_init(ctx: CommandContext) -> None: """Initialize a subdirectory as a nested repo.""" git, flags, head_commit = ctx.git, ctx.flags, ctx.head if flags.from_submodule: - _init_from_submodule(git, flags, setup.resolve_subdir(ctx.subdir)) + init_from_submodule(git, flags, setup.resolve_subdir(ctx.subdir)) return subdir, gitnested, subref, config = setup.setup_command(git, 'init', flags, ctx.subdir, ctx.upstream) _validate_init_target(subdir, gitnested, git) diff --git a/src/git_nested/commands/pull.py b/src/git_nested/commands/pull.py index 6fedf04f..225a789b 100644 --- a/src/git_nested/commands/pull.py +++ b/src/git_nested/commands/pull.py @@ -325,7 +325,8 @@ def cmd_pull(ctx: CommandContext) -> None: follow_inner(ctx, config, subdir) if flags.recurse_submodules: clone.nest_submodules(git, flags, config, subdir) - submodules.warn_if_any(git, subdir) + elif config.submodules != 'keep': + submodules.warn_if_any(git, subdir) output.say(f"{subdir.as_posix()}: pulled from {config.remote} ({config.branch})") @@ -348,7 +349,10 @@ def follow_inner(ctx: CommandContext, config: NestedConfig, subdir: Path) -> Non def pull_to(ctx: CommandContext, subdir: Path, url: str | None, pin: str) -> None: - """Pull subdir to `pin`, and from `url` if that is new. Local changes stay.""" + """Pull subdir to `pin`, and from `url` if that is new. Local changes stay. + + New submodules in it are nested, unless its .gitnested says 'keep'. + """ current = NestedConfig.from_file(subdir / GITNESTED_FILENAME) if current.commit == pin: return @@ -356,7 +360,7 @@ def pull_to(ctx: CommandContext, subdir: Path, url: str | None, pin: str) -> Non branch=pin, remote=url if url and url != current.remote else None, update=True, - recurse_submodules=True, + recurse_submodules=current.submodules != 'keep', gpg_sign=ctx.flags.gpg_sign, signoff=ctx.flags.signoff, quiet=ctx.flags.quiet, diff --git a/src/git_nested/commands/submodules.py b/src/git_nested/commands/submodules.py new file mode 100644 index 00000000..35e96996 --- /dev/null +++ b/src/git_nested/commands/submodules.py @@ -0,0 +1,116 @@ +"""Turn all submodules into nested repos. + +The root repo stays a normal clone of the upstream superproject. Its +submodules are nested repos. + +- init: nest every submodule, or the ones named, one commit each. +""" + +from __future__ import annotations + +import posixpath +from dataclasses import replace +from pathlib import Path +from typing import cast + +from .. import checks, output, submodules +from ..errors import GitNestedError +from ..git import GitRunner +from ..models import CommandContext, Flags +from . import init + +_GITMODULES = '.gitmodules' +# --only and --ignore, kept in .gitmodules +_ONLY, _IGNORE = 'nested.only', 'nested.ignore' + + +def cmd_submodules(ctx: CommandContext) -> None: + """Run `submodules init`.""" + if ctx.flags.only and ctx.flags.ignore: + output.usage_error("--only and --ignore can't be used together") + checks.check_worktree_clean(ctx.git, 'init') + _nest_all(ctx.git, ctx.flags) + + +# ============================================================================ +# init +# ============================================================================ + + +def _nest_all(git: GitRunner, flags: Flags) -> None: + """Nest each submodule at the commit it pins. With --recursive, their submodules too. + + With --only, only the submodules at or below its paths. With --ignore, + all others. The rest stay submodules. Relative submodule urls need + --resolve-relative. + """ + links = _selected(git, flags) + if not links: + output.say("no submodules to nest") + return + if not flags.resolve_relative: + _refuse_relative_urls(git, links) + _keep_selection(git, flags) + # .gitnested keeps the choice + choice = cast('Flags', replace(flags, submodules='nest' if flags.recursive else 'keep', hint='--recursive')) + for path in links: + init.init_from_submodule(git, choice, Path(path), flags.resolve_relative) + + +def _selected(git: GitRunner, flags: Flags) -> list[str]: + """The submodules that --only and --ignore leave in.""" + links = [path for path, _ in submodules.gitlinks(git, Path())] + if flags.only: + return _below_any(links, flags.only) + skipped = set(_below_any(links, flags.ignore)) if flags.ignore else set() + return [link for link in links if link not in skipped] + + +def _keep_selection(git: GitRunner, flags: Flags) -> None: + """Write --only and --ignore into .gitmodules. It is part of the first commit.""" + if not (flags.only or flags.ignore): + return + only = [posixpath.normpath(Path(path).as_posix()) for path in flags.only or []] + ignore = [posixpath.normpath(Path(path).as_posix()) for path in flags.ignore or []] + _add_selection(git, Path(_GITMODULES), (only, ignore)) + git.run(['add', '--', _GITMODULES]) + + +def _add_selection(git: GitRunner, file: Path, selection: tuple[list[str], list[str]]) -> None: + """Add the --only and --ignore paths of `selection` to `file`.""" + for key, paths in zip((_ONLY, _IGNORE), selection, strict=True): + for path in filter(None, paths): + git.run(['config', '-f', file, '--add', key, path]) + + +def _refuse_relative_urls(git: GitRunner, links: list[str]) -> None: + """Fail if a submodule in `links` has a relative url. + + It is relative to the superproject's url. In a fork that is not origin. + So git-nested does not guess. + """ + relative = [ + path + for path in links + if (found := submodules.entry(git, Path(_GITMODULES), path)) and submodules.is_relative(found[1]) + ] + if relative: + raise GitNestedError( + f"these submodules have a relative url: {', '.join(relative)}\n" + "Use --resolve-relative to say what they are relative to. " + "That is the url of the project they come from." + ) + + +def _below_any(links: list[str], paths: list[str]) -> list[str]: + """The submodules in `links` at or below one of `paths`. Each path must hit one.""" + wanted = [posixpath.normpath(Path(path).as_posix()) for path in paths] + for path, given in zip(wanted, paths, strict=True): + if not any(_below(link, path) for link in links): + raise GitNestedError(f"{given}: no submodule there") + return [link for link in links if any(_below(link, path) for path in wanted)] + + +def _below(link: str, path: str) -> bool: + """Whether `link` is `path` or below it.""" + return path in ('.', link) or link.startswith(f'{path}/') diff --git a/src/git_nested/commands/verify.py b/src/git_nested/commands/verify.py index b8a60ca0..6ca6c588 100644 --- a/src/git_nested/commands/verify.py +++ b/src/git_nested/commands/verify.py @@ -4,7 +4,7 @@ from pathlib import Path -from .. import filters, output, patchbuild, patches +from .. import filters, history, output, patchbuild, patches from ..cli import setup from ..errors import GitNestedError from ..git import GitRunner @@ -138,7 +138,9 @@ def cmd_verify(ctx: CommandContext) -> None: local_tree = git.check_output(['rev-parse', f'HEAD:{subdir.as_posix()}']) base = upstream_base(git, config, subref, subdir) - pathspec = patches.pathspec(patches.local_paths(git, config, subdir), filters.ignored_paths(ctx.flags, subdir)) + # Nested repos inside are not upstream content of this one + ignored = [*filters.ignored_paths(ctx.flags, subdir), *history.gitlinked_inner(git, config, subdir)] + pathspec = patches.pathspec(patches.local_paths(git, config, subdir), ignored) path = ctx.tmp / 'verify' / subref with patchbuild.worktree_at(git, path, base, keep=ctx.flags.keep): diff --git a/src/git_nested/gitfile.py b/src/git_nested/gitfile.py index 48985c84..02141108 100644 --- a/src/git_nested/gitfile.py +++ b/src/git_nested/gitfile.py @@ -33,11 +33,12 @@ 'patches': "the folder with the local changes as patches, relative to the subdir", 'patches-ignore': "paths in the subdir that stay local, so no patch needs their changes", 'patches-local': "patches a push leaves out, by file name", + 'submodules': "'nest' or 'keep': whether a pull nests the submodules in it. Unset: nest", } # The rest are written by the nested operations themselves; setting one by # hand would describe a state the repository is not in. -WRITABLE_CONFIG_FIELDS = ('remote', 'branch', 'method', 'squash', 'parent', 'pin', 'readonly', 'patches') +WRITABLE_CONFIG_FIELDS = ('remote', 'branch', 'method', 'squash', 'parent', 'pin', 'readonly', 'patches', 'submodules') def create_level_gitnested_files(git: GitRunner, subdir: Path, level: int | None = None) -> None: @@ -110,7 +111,7 @@ def update_gitrepo_file( nested['method'] = flags.method or config.method or 'merge' nested['cmdver'] = VERSION # These are written only when given - for field in ('filter', 'exclude', 'path', 'pin'): + for field in ('filter', 'exclude', 'path', 'pin', 'submodules'): if getattr(flags, field): nested[field] = getattr(flags, field) diff --git a/src/git_nested/history.py b/src/git_nested/history.py index 2db66e2e..0ed86624 100644 --- a/src/git_nested/history.py +++ b/src/git_nested/history.py @@ -217,13 +217,17 @@ def _as_gitlink(self, commit: str, path: str, prefix: str) -> None: """Put the inner nested repository at `path` back as the submodule it came from. Its gitlink pins the commit its .gitnested records in `commit`. In - the index it sits under `prefix`. + the index it sits under `prefix`. If `commit` does not have it yet, + it keeps upstream's gitlink: a recursive clone nests it one commit + later, and that is no change. """ + entry = f'{prefix}{path}' pinned = _recorded_commit(self.git, commit, self.subdir / path / GITNESTED_FILENAME) if pinned: - entry = f'{prefix}{path}' self.git.run(['rm', '--cached', '-r', '-q', '-f', '--ignore-unmatch', '--', entry], env=self.index_env) self.git.run(['update-index', '--add', '--cacheinfo', f'160000,{pinned},{entry}'], env=self.index_env) + elif self.git.run(['cat-file', '-e', f'{commit}:{(self.subdir / path).as_posix()}'], may_fail=True).returncode: + self._keep_upstream(commit, [entry]) def _keep_upstream(self, commit: str, paths: list[str]) -> None: """Set `paths` to what the recorded upstream commit has. diff --git a/src/git_nested/models.py b/src/git_nested/models.py index 1102ccce..bd46d7f7 100644 --- a/src/git_nested/models.py +++ b/src/git_nested/models.py @@ -28,6 +28,13 @@ class Flags: filter: list[str] | None = None force: bool = False ignore: list[str] | None = None + only: list[str] | None = None + resolve_relative: str | None = None + recursive: bool = False + # 'nest' or 'keep': what a new nested repo does with its submodules + submodules: str | None = None + # The option to name when submodules stay empty + hint: str = '' from_submodule: bool = False from_patches: bool = False local: bool = False @@ -90,6 +97,7 @@ class NestedConfig: patches_ignore: tuple[str, ...] = () # Patches a push leaves out, by file name: they stay local. patches_local: tuple[str, ...] = () + submodules: str = '' @classmethod def from_file(cls, filepath: str | Path) -> NestedConfig: @@ -113,6 +121,7 @@ def from_file(cls, filepath: str | Path) -> NestedConfig: patches=_patches(nested.get('patches')), patches_ignore=_paths(nested.get('patches-ignore')), patches_local=_paths(nested.get('patches-local')), + submodules=str(nested.get('submodules') or ''), ) if not config.remote: raise GitNestedError(f"{path.as_posix()} has no 'remote'") @@ -165,6 +174,7 @@ class CommandContext: patches_action: str | None = None patch_names: list[str] | None = None verify_mode: str | None = None + submodules_action: str | None = None git_tmp: Path | None = None head_commit: str | None = None diff --git a/src/git_nested/submodules.py b/src/git_nested/submodules.py index 67d0c1a7..cc283110 100644 --- a/src/git_nested/submodules.py +++ b/src/git_nested/submodules.py @@ -50,9 +50,14 @@ def url_of(git: GitRunner, gitmodules: Path, path: str, base: str) -> str | None return resolve(found[1], base) if found else None +def is_relative(url: str) -> bool: + """Whether `url` is relative ('./x', '../x').""" + return url.startswith(('./', '../')) + + def resolve(url: str, base: str) -> str: """`url`, when relative ('./x', '../x'), resolved against `base`, as git does it.""" - if not url.startswith(('./', '../')): + if not is_relative(url): return url scheme, sep, rest = base.rpartition('://') if not sep: @@ -61,11 +66,8 @@ def resolve(url: str, base: str) -> str: return f'{scheme}{sep}{posixpath.normpath(posixpath.join(rest, url))}' -def warn_if_any(git: GitRunner, subdir: Path) -> None: - """Say which submodules under subdir stay empty.""" +def warn_if_any(git: GitRunner, subdir: Path, advice: str = "use --recurse-submodules to nest them") -> None: + """Say which submodules under subdir stay empty, and what to do about it.""" paths = [path.removeprefix(f'{subdir.as_posix()}/') for path, _ in gitlinks(git, subdir)] if paths: - output.warn( - f"{subdir.as_posix()}: these submodules stay empty: {', '.join(paths)}; " - "use --recurse-submodules to nest them" - ) + output.warn(f"{subdir.as_posix()}: these submodules stay empty: {', '.join(paths)}; {advice}") diff --git a/tests/conftest.py b/tests/conftest.py index e9a736f4..0248478b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -551,6 +551,7 @@ def assert_gitnested_field( method: str | None = None, squash: str | None = None, version: str | None = None, + submodules: str | None = None, ): """Assert that according fields in .gitnested YAML file have the expected value""" assert Path(gitnested_path).exists() @@ -569,6 +570,7 @@ def assert_gitnested_field( _assert_yaml_field(nested_data, 'method', method) _assert_yaml_field(nested_data, 'squash', squash) _assert_yaml_field(nested_data, 'cmdver', version) + _assert_yaml_field(nested_data, 'submodules', submodules) def _get_commit_info(cwd, ref: str, format_str: str) -> str: diff --git a/tests/e2e/test_issue_184.py b/tests/e2e/test_issue_184.py new file mode 100644 index 00000000..e075b833 --- /dev/null +++ b/tests/e2e/test_issue_184.py @@ -0,0 +1,121 @@ +"""The scenario of issue #184. + +Upstream has these repos: + +- super: the superproject. Submodules: modules/core and tools. +- core: has the submodule third_party/lib. +- docs: has the submodule theme. Super adds it later. +- lib, tools, theme: plain repos. + +The user wants the submodules as nested repos: no gitlink, no .gitmodules. +""" + +import pytest +from conftest import cmd_git_nested + +ALLOW_FILE = ['-c', 'protocol.file.allow=always'] + + +class Upstream: + """The upstream repos, each with a work clone to make commits in.""" + + def __init__(self, env): + self.env = env + self.work = env.workspace / 'upstream-work' + + def url(self, name): + return (self.env.upstream / name).as_posix() + + def git(self, name, *args): + return self.env.run(['git', *ALLOW_FILE, *args], cwd=self.work / name).output.strip() + + def create(self, name, **files): + self.env.run(['git', 'init', '-q', '--bare', self.url(name)]) + self.env.run(['git', 'clone', '-q', self.url(name), self.work / name]) + self.write(name, **files) + self.push(name, f'{name}: first commit') + + def write(self, name, **files): + """Append a line to each file.""" + for file, text in files.items(): + path = self.work / name / file + path.parent.mkdir(parents=True, exist_ok=True) + with path.open('a') as f: + f.write(f'{text}\n') + + def add_submodule(self, name, sub, path): + self.git(name, 'submodule', 'add', '-q', self.url(sub), path) + + def follow(self, name, path): + """Let the submodule at `path` pin the newest commit of its upstream.""" + self.git(name, '-C', path, 'fetch', '-q') + self.git(name, '-C', path, 'checkout', '-q', 'origin/master') + + def push(self, name, message, author='Upstream Dev'): + """Commit all changes as `author` and push. Return the commit.""" + self.git(name, 'add', '-A') + self.git(name, '-c', f'user.name={author}', '-c', 'user.email=dev@upstream', 'commit', '-qm', message) + self.git(name, 'push', '-q', 'origin', 'HEAD:master') + return self.git(name, 'rev-parse', 'HEAD') + + def head(self, name): + return self.git(name, 'rev-parse', 'HEAD') + + +@pytest.fixture +def upstream(env): + up = Upstream(env) + for name in ('lib', 'tools', 'theme'): + up.create(name, **{name.capitalize(): name}) + up.create('core', Core='core') + up.add_submodule('core', 'lib', 'third_party/lib') + up.push('core', 'core: add lib') + up.create('docs', Docs='docs') + up.add_submodule('docs', 'theme', 'theme') + up.push('docs', 'docs: add theme') + up.create('super', README='super') + up.add_submodule('super', 'core', 'modules/core') + up.add_submodule('super', 'tools', 'tools') + up.push('super', 'add core and tools') + return up + + +# ============================================================================ +# Helpers: checks in the fork +# ============================================================================ + + +def _git(env, repo, *args): + return env.run(['git', *args], cwd=repo).output.strip() + + +def _assert_flat(env, repo): + """No gitlink, no .gitmodules, nothing left over.""" + modes = {line.split()[0] for line in _git(env, repo, 'ls-files', '-s').splitlines()} + assert '160000' not in modes + assert not (repo / '.gitmodules').exists() + assert _git(env, repo, 'status', '--porcelain') == '' + + +def _nested_repos(env, repo): + """The paths of all nested repos in HEAD.""" + files = _git(env, repo, 'ls-tree', '-r', '--name-only', 'HEAD').splitlines() + return sorted(path.removesuffix('/.gitnested') for path in files if path.endswith('/.gitnested')) + + +# ============================================================================ +# Single cases of the story +# ============================================================================ + + +def test_init_in_a_clone_the_user_already_has(env, upstream): + """The user has a normal clone with submodules. init flattens it in place.""" + repo = env.workspace / 'clone' + env.run(['git', *ALLOW_FILE, 'clone', '-q', '--recurse-submodules', upstream.url('super'), repo]) + + cmd_git_nested('submodules init --recursive', cwd=repo) + + assert _nested_repos(env, repo) == ['modules/core', 'modules/core/third_party/lib', 'tools'] + assert (repo / 'modules' / 'core' / 'third_party' / 'lib' / 'Lib').is_file() + assert not (repo / '.git' / 'modules' / 'modules' / 'core').exists() + _assert_flat(env, repo) diff --git a/tests/e2e/test_submodules_command.py b/tests/e2e/test_submodules_command.py new file mode 100644 index 00000000..64bb8b67 --- /dev/null +++ b/tests/e2e/test_submodules_command.py @@ -0,0 +1,281 @@ +"""Tests for `git nested submodules` (#184). + +Upstream bar is the superproject. Its submodules become nested repos. +""" + +from conftest import assert_gitnested_field, clone_repo, cmd_git_nested, git_rev_parse + +ALLOW_FILE = ['-c', 'protocol.file.allow=always'] + + +# ============================================================================ +# Helpers: changes in upstream +# ============================================================================ + + +def _push_bar(env, message): + bar = env.workspace / 'bar' + env.run(['git', 'commit', '-qam', message], cwd=bar) + env.run(['git', 'push', '-q'], cwd=bar) + return git_rev_parse(['HEAD'], cwd=bar) + + +def _upstream_adds_submodule(env, url, path): + """Add a submodule to upstream bar. Return its pin.""" + bar = env.workspace / 'bar' + env.run(['git', *ALLOW_FILE, 'submodule', 'add', '-q', url.as_posix(), path], cwd=bar) + _push_bar(env, f'add {path}') + return git_rev_parse(['HEAD'], cwd=bar / path) + + +def _make_mid(env): + """Make upstream mid. It has the submodule 'inner'. Return its url.""" + url = env.upstream / 'mid' + env.run(['git', 'init', '-q', '--bare', url]) + mid = env.workspace / 'mid' + env.run(['git', 'clone', '-q', url, mid]) + env.add_new_files('Mid', cwd=mid) + env.run(['git', *ALLOW_FILE, 'submodule', 'add', '-q', (env.upstream / 'foo').as_posix(), 'inner'], cwd=mid) + env.run(['git', 'commit', '-qm', 'add inner'], cwd=mid) + env.run(['git', 'push', '-q', 'origin', 'master'], cwd=mid) + return url + + +# ============================================================================ +# Helpers: checks +# ============================================================================ + + +def _assert_flat(env, repo): + """No submodules and no leftovers.""" + modes = [line.split()[0] for line in env.run(['git', 'ls-files', '-s'], cwd=repo).output.splitlines()] + assert '160000' not in modes + assert not (repo / '.gitmodules').exists() + assert not list(repo.glob('*~*')) + assert env.run(['git', 'status', '--porcelain'], cwd=repo).output == '' + + +def _top_dirs_per_commit(env, repo, base): + """The top dirs that each commit after base changed, sorted. .gitmodules does not count.""" + commits = env.run(['git', 'rev-list', f'{base}..HEAD'], cwd=repo).output.split() + changed = [env.run(['git', 'diff', '--name-only', f'{c}^', c], cwd=repo).output.split() for c in commits] + dirs = [{path.split('/')[0] for path in paths if path != '.gitmodules'} for paths in changed] + return sorted(dirs, key=sorted) + + +# ============================================================================ +# submodules init +# ============================================================================ + + +def test_init_nests_all_submodules(foo_bar_cloned): + env = foo_bar_cloned + sub = _upstream_adds_submodule(env, env.upstream / 'foo', 'sub') + lib = _upstream_adds_submodule(env, env.upstream / 'foo', 'ext/lib') + tip = git_rev_parse(['HEAD'], cwd=env.workspace / 'bar') + repo = env.workspace / 'repo' + clone_repo((env.upstream / 'bar').as_posix(), repo) + + cmd_git_nested('submodules init', cwd=repo) + + assert_gitnested_field(repo / 'sub' / '.gitnested', commit=sub) + assert_gitnested_field(repo / 'ext' / 'lib' / '.gitnested', commit=lib) + assert _top_dirs_per_commit(env, repo, tip) == [{'ext'}, {'sub'}] + _assert_flat(env, repo) + + +def test_init_nests_submodules_of_submodules(foo_bar_cloned): + env = foo_bar_cloned + _upstream_adds_submodule(env, _make_mid(env), 'mid') + repo = env.workspace / 'repo' + clone_repo((env.upstream / 'bar').as_posix(), repo) + + cmd_git_nested('submodules init --recursive', cwd=repo) + + assert (repo / 'mid' / 'inner' / 'Foo').is_file() + assert (repo / 'mid' / 'inner' / '.gitnested').is_file() + _assert_flat(env, repo) + + +def test_init_without_recursive_leaves_deeper_submodules(foo_bar_cloned): + env = foo_bar_cloned + _upstream_adds_submodule(env, _make_mid(env), 'mid') + repo = env.workspace / 'repo' + clone_repo((env.upstream / 'bar').as_posix(), repo) + + result = cmd_git_nested('submodules init', cwd=repo) + + assert_gitnested_field(repo / 'mid' / '.gitnested', submodules='keep') + assert _gitlinks(env, repo) == ['mid/inner'] + assert 'mid: these submodules stay empty: inner; use --recursive to nest them' in result.output + + +def test_init_recursive_writes_nest(foo_bar_cloned): + env = foo_bar_cloned + _upstream_adds_submodule(env, _make_mid(env), 'mid') + repo = env.workspace / 'repo' + clone_repo((env.upstream / 'bar').as_posix(), repo) + + cmd_git_nested('submodules init --recursive', cwd=repo) + + assert_gitnested_field(repo / 'mid' / '.gitnested', submodules='nest') + assert_gitnested_field(repo / 'mid' / 'inner' / '.gitnested', submodules='nest') + assert 'mid: matches upstream' in cmd_git_nested('verify mid', cwd=repo).output + + +def test_init_recursive_resolves_deeper_relative_urls_against_their_parent(foo_bar_cloned): + """--resolve-relative is for the first level. Deeper, the url of the parent submodule counts.""" + env = foo_bar_cloned + mid = env.workspace / 'mid' + _make_mid(env) + env.run(['git', 'submodule', 'set-url', 'inner', '../foo'], cwd=mid) + env.run(['git', 'commit', '-qam', 'inner is relative'], cwd=mid) + env.run(['git', 'push', '-q', 'origin', 'master'], cwd=mid) + bar = env.workspace / 'bar' + env.run(['git', *ALLOW_FILE, 'submodule', 'add', '-q', '../mid', 'mid'], cwd=bar) + _push_bar(env, 'add mid') + repo = env.workspace / 'repo' + clone_repo((env.upstream / 'bar').as_posix(), repo) + + base = (env.upstream / 'bar').as_posix() + cmd_git_nested(['submodules', 'init', '--recursive', '--resolve-relative', base], cwd=repo) + + assert_gitnested_field(repo / 'mid' / '.gitnested', remote=(env.upstream / 'mid').as_posix()) + assert_gitnested_field(repo / 'mid' / 'inner' / '.gitnested', remote=(env.upstream / 'foo').as_posix()) + _assert_flat(env, repo) + + +def test_init_without_submodules_changes_nothing(foo_bar_cloned): + env = foo_bar_cloned + repo = env.workspace / 'bar' + head = git_rev_parse(['HEAD'], cwd=repo) + + result = cmd_git_nested('submodules init', cwd=repo) + + assert 'no submodules' in result.output + assert git_rev_parse(['HEAD'], cwd=repo) == head + + +def _repo_with_three_submodules(env): + """A clone of bar with the submodules sub, ext/lib and ext/more.""" + for path in ('sub', 'ext/lib', 'ext/more'): + _upstream_adds_submodule(env, env.upstream / 'foo', path) + repo = env.workspace / 'repo' + clone_repo((env.upstream / 'bar').as_posix(), repo) + return repo + + +def _gitlinks(env, repo): + entries = env.run(['git', 'ls-files', '-s'], cwd=repo).output.splitlines() + return sorted(entry.split('\t')[1] for entry in entries if entry.startswith('160000 ')) + + +def test_init_nests_only_the_named_submodules(foo_bar_cloned): + env = foo_bar_cloned + repo = _repo_with_three_submodules(env) + + cmd_git_nested('submodules init --only ext/lib/', cwd=repo) + + assert (repo / 'ext' / 'lib' / '.gitnested').is_file() + assert _gitlinks(env, repo) == ['ext/more', 'sub'] + paths = env.run(['git', 'config', '-f', '.gitmodules', '--get-regexp', 'path$'], cwd=repo).output.split() + assert 'ext/more' in paths + assert 'ext/lib' not in paths + only = env.run(['git', 'config', '-f', '.gitmodules', 'nested.only'], cwd=repo).output.strip() + assert only == 'ext/lib' + + +def test_init_nests_the_submodules_below_a_path(foo_bar_cloned): + env = foo_bar_cloned + repo = _repo_with_three_submodules(env) + + cmd_git_nested('submodules init --only ext', cwd=repo) + + assert (repo / 'ext' / 'lib' / '.gitnested').is_file() + assert (repo / 'ext' / 'more' / '.gitnested').is_file() + assert _gitlinks(env, repo) == ['sub'] + + +def test_init_ignore_skips_the_submodules_below_a_path(foo_bar_cloned): + env = foo_bar_cloned + repo = _repo_with_three_submodules(env) + + cmd_git_nested('submodules init --ignore ext', cwd=repo) + + assert (repo / 'sub' / '.gitnested').is_file() + assert _gitlinks(env, repo) == ['ext/lib', 'ext/more'] + + +def test_init_paths_are_relative_to_the_current_directory(foo_bar_cloned): + env = foo_bar_cloned + repo = _repo_with_three_submodules(env) + + cmd_git_nested('submodules init --only more --only ../sub', cwd=repo / 'ext') + + assert (repo / 'ext' / 'more' / '.gitnested').is_file() + assert _gitlinks(env, repo) == ['ext/lib'] + + +def test_init_refuses_a_path_without_a_submodule(foo_bar_cloned): + env = foo_bar_cloned + repo = _repo_with_three_submodules(env) + head = git_rev_parse(['HEAD'], cwd=repo) + + result = cmd_git_nested('submodules init --only sub --only nope', cwd=repo, check=False) + + assert result.returncode == 1 + assert 'nope: no submodule there' in result.output + assert git_rev_parse(['HEAD'], cwd=repo) == head + + +# ============================================================================ +# submodules init: errors and relative urls +# ============================================================================ + + +def test_init_needs_a_clean_tree(foo_bar_cloned): + env = foo_bar_cloned + _upstream_adds_submodule(env, env.upstream / 'foo', 'sub') + repo = env.workspace / 'bar' + (repo / 'Bar').write_text('changed\n') + + result = cmd_git_nested('submodules init', cwd=repo, check=False) + + assert result.returncode == 1 + assert 'changes' in result.output + + +# ============================================================================ +# Nested repos with patches +# ============================================================================ + +PATCHES = '.gitnested.patches' + + +def test_init_refuses_only_with_ignore(foo_bar_cloned): + env = foo_bar_cloned + repo = _repo_with_three_submodules(env) + + result = cmd_git_nested('submodules init --only sub --ignore ext', cwd=repo, check=False) + + assert result.returncode == 1 + assert "--only and --ignore can't be used together" in result.output + + +def test_init_resolves_relative_urls_only_when_asked(foo_bar_cloned): + """A fork lives somewhere else. Its relative urls point to the project it forked.""" + env = foo_bar_cloned + env.run(['git', *ALLOW_FILE, 'submodule', 'add', '-q', '../foo', 'sub'], cwd=env.workspace / 'bar') + _push_bar(env, 'add sub') + forks = env.tmp / 'forks' + env.run(['git', 'clone', '-q', '--bare', env.upstream / 'bar', forks / 'bar']) + repo = env.workspace / 'fork' + clone_repo((forks / 'bar').as_posix(), repo) + + result = cmd_git_nested('submodules init', cwd=repo, check=False) + assert 'these submodules have a relative url: sub' in result.output + assert 'Use --resolve-relative ' in result.output + assert not (repo / 'sub' / '.gitnested').exists() + + cmd_git_nested(['submodules', 'init', '--resolve-relative', (env.upstream / 'bar').as_posix()], cwd=repo) + assert_gitnested_field(repo / 'sub' / '.gitnested', remote=(env.upstream / 'foo').as_posix()) diff --git a/tests/e2e/test_submodules_contribute.py b/tests/e2e/test_submodules_contribute.py new file mode 100644 index 00000000..2488dd69 --- /dev/null +++ b/tests/e2e/test_submodules_contribute.py @@ -0,0 +1,196 @@ +"""Tests for contributing back from nested submodules (#184). + +Upstream bar has the submodule mid. mid has the submodule inner. +`submodules init --recursive` nests both. A branch or push of one of them +must carry its own changes only: no files of the other nested repos, no +.gitnested files, and no changes of the parent. +""" + +import pytest +from conftest import clone_repo, cmd_git_nested, git_rev_parse + +ALLOW_FILE = ['-c', 'protocol.file.allow=always'] + + +def _make_mid(env): + """Make upstream mid with the submodule inner, which is upstream foo.""" + url = env.upstream / 'mid' + env.run(['git', 'init', '-q', '--bare', url]) + mid = env.workspace / 'mid' + env.run(['git', 'clone', '-q', url, mid]) + (mid / 'Mid').write_text("mid\n") + env.run(['git', 'add', 'Mid'], cwd=mid) + env.run(['git', *ALLOW_FILE, 'submodule', 'add', '-q', (env.upstream / 'foo').as_posix(), 'inner'], cwd=mid) + env.run(['git', 'commit', '-qm', 'mid with inner'], cwd=mid) + env.run(['git', 'push', '-q', 'origin', 'master'], cwd=mid) + return url + + +@pytest.fixture +def fork(foo_bar_cloned): + """A fork of bar, with mid and mid/inner nested, and one commit that changes all three.""" + env = foo_bar_cloned + mid_url = _make_mid(env) + bar = env.workspace / 'bar' + env.run(['git', *ALLOW_FILE, 'submodule', 'add', '-q', mid_url.as_posix(), 'mid'], cwd=bar) + env.run(['git', 'commit', '-qm', 'add mid'], cwd=bar) + env.run(['git', 'push', '-q'], cwd=bar) + env.mid_tip = git_rev_parse(['HEAD'], cwd=env.workspace / 'mid') + env.inner_pin = git_rev_parse(['HEAD:inner'], cwd=env.workspace / 'mid') + env.fork = env.workspace / 'fork' + clone_repo((env.upstream / 'bar').as_posix(), env.fork) + cmd_git_nested('submodules init --recursive', cwd=env.fork) + + fork = env.fork + (fork / 'Bar').write_text("changed in the parent\n") + (fork / 'mid' / 'Mid').write_text("changed in mid\n") + (fork / 'mid' / 'new-in-mid').write_text("new in mid\n") + (fork / 'mid' / 'inner' / 'Foo').write_text("changed in inner\n") + (fork / 'mid' / 'inner' / 'new-in-inner').write_text("new in inner\n") + env.run(['git', 'add', '-A'], cwd=fork) + env.run(['git', 'commit', '-qm', 'change the parent, mid and inner at once'], cwd=fork) + return env + + +def _tree(env, repo, rev): + """{path: (mode, object)} of `rev` in `repo`.""" + entries = env.run(['git', 'ls-tree', '-r', '-z', rev], cwd=repo).output.split('\0') + return {path: tuple(meta.split()[::2]) for meta, _, path in (e.partition('\t') for e in entries if e)} + + +def _changed(env, repo, old, new): + return sorted(env.run(['git', 'diff', '--name-only', old, new], cwd=repo).output.split()) + + +def _branch_name(output): + """The branch that `git nested branch` says it created.""" + return output.split('branch ', 1)[1].split(' created', 1)[0] + + +def _assert_only_own_content(tree): + """No files of other nested repos, and no .gitnested files.""" + assert not [path for path in tree if path.startswith('inner/')] + assert not [path for path in tree if path.rsplit('/', 1)[-1].startswith('.gitnested')] + assert 'Bar' not in tree + + +# ============================================================================ +# git nested branch +# ============================================================================ + + +def test_branch_of_mid_has_mid_changes_only(fork): + env = fork + result = cmd_git_nested('branch mid', cwd=env.fork) + branch = _branch_name(result.output) + + tree = _tree(env, env.fork, branch) + assert tree['inner'] == ('160000', env.inner_pin) + _assert_only_own_content(tree) + assert _changed(env, env.fork, env.mid_tip, branch) == ['Mid', 'new-in-mid'] + assert env.run(['git', 'merge-base', '--is-ancestor', env.mid_tip, branch], cwd=env.fork).returncode == 0 + + +def test_branch_of_inner_has_inner_changes_only(fork): + env = fork + result = cmd_git_nested('branch mid/inner', cwd=env.fork) + branch = _branch_name(result.output) + + tree = _tree(env, env.fork, branch) + assert not [path for path in tree if path.rsplit('/', 1)[-1].startswith('.gitnested')] + assert 'Mid' not in tree + assert 'Bar' not in tree + assert _changed(env, env.fork, env.inner_pin, branch) == ['Foo', 'new-in-inner'] + + +def test_each_commit_of_the_branch_stays_inside_mid(fork): + """A commit of the parent that changes nothing in mid is not on the branch.""" + env = fork + (env.fork / 'Bar').write_text("parent only\n") + env.run(['git', 'commit', '-qam', 'parent only'], cwd=env.fork) + (env.fork / 'mid' / 'inner' / 'Foo').write_text("inner only\n") + env.run(['git', 'commit', '-qam', 'inner only'], cwd=env.fork) + + branch = _branch_name(cmd_git_nested('branch mid', cwd=env.fork).output) + + commits = env.run(['git', 'rev-list', f'{env.mid_tip}..{branch}'], cwd=env.fork).output.split() + assert commits + for commit in commits: + changed = env.run(['git', 'diff-tree', '--no-commit-id', '--name-only', '-r', commit], cwd=env.fork).output + assert set(changed.split()) <= {'Mid', 'new-in-mid'} + + +# ============================================================================ +# git nested push +# ============================================================================ + + +def test_push_of_mid_sends_mid_changes_only(fork): + env = fork + cmd_git_nested('push -b master mid', cwd=env.fork) + + upstream = env.upstream / 'mid' + tree = _tree(env, upstream, 'master') + assert tree['inner'] == ('160000', env.inner_pin) + _assert_only_own_content(tree) + assert _changed(env, upstream, env.mid_tip, 'master') == ['Mid', 'new-in-mid'] + + +def test_push_of_inner_sends_inner_changes_only(fork): + env = fork + cmd_git_nested('push -b master mid/inner', cwd=env.fork) + + upstream = env.upstream / 'foo' + tree = _tree(env, upstream, 'master') + assert not [path for path in tree if path.rsplit('/', 1)[-1].startswith('.gitnested')] + assert 'Mid' not in tree + assert _changed(env, upstream, env.inner_pin, 'master') == ['Foo', 'new-in-inner'] + + +def test_push_of_mid_keeps_the_pin_of_inner(fork): + """Pushing inner does not bump mid's gitlink. Only a pull of inner moves what mid pins.""" + env = fork + cmd_git_nested('push -b master mid/inner', cwd=env.fork) + + cmd_git_nested('push -b master mid', cwd=env.fork) + + assert _tree(env, env.upstream / 'mid', 'master')['inner'] == ('160000', env.inner_pin) + + +def test_push_of_mid_after_a_pull_of_inner_bumps_the_pin(fork): + """Push inner, pull it, then push mid: mid's gitlink moves to what inner pushed.""" + env = fork + cmd_git_nested('push -b master mid/inner', cwd=env.fork) + pushed = git_rev_parse(['master'], cwd=env.upstream / 'foo') + cmd_git_nested('pull -b master mid/inner', cwd=env.fork) + + cmd_git_nested('push -b master mid', cwd=env.fork) + + upstream = env.upstream / 'mid' + assert _tree(env, upstream, 'master')['inner'] == ('160000', pushed) + assert _changed(env, upstream, env.mid_tip, 'master') == ['Mid', 'inner', 'new-in-mid'] + + +def test_push_of_mid_with_a_kept_inner(foo_bar_cloned): + """Without --recursive inner stays a submodule. A push of mid keeps its gitlink as it is.""" + env = foo_bar_cloned + mid_url = _make_mid(env) + bar = env.workspace / 'bar' + env.run(['git', *ALLOW_FILE, 'submodule', 'add', '-q', mid_url.as_posix(), 'mid'], cwd=bar) + env.run(['git', 'commit', '-qm', 'add mid'], cwd=bar) + env.run(['git', 'push', '-q'], cwd=bar) + mid_tip = git_rev_parse(['HEAD'], cwd=env.workspace / 'mid') + inner_pin = git_rev_parse(['HEAD:inner'], cwd=env.workspace / 'mid') + fork = env.workspace / 'fork' + clone_repo((env.upstream / 'bar').as_posix(), fork) + cmd_git_nested('submodules init', cwd=fork) + (fork / 'mid' / 'Mid').write_text("changed in mid\n") + env.run(['git', 'commit', '-qam', 'change mid'], cwd=fork) + + cmd_git_nested('push -b master mid', cwd=fork) + + upstream = env.upstream / 'mid' + tree = _tree(env, upstream, 'master') + assert tree['inner'] == ('160000', inner_pin) + _assert_only_own_content(tree) + assert _changed(env, upstream, mid_tip, 'master') == ['Mid'] diff --git a/tests/unit/test_command.py b/tests/unit/test_command.py index 3bd33da0..df424bb4 100644 --- a/tests/unit/test_command.py +++ b/tests/unit/test_command.py @@ -155,3 +155,18 @@ def test_check_existing_worktree_errors_without_gitnested(cmd, monkeypatch, tmp_ def test_parse_args_method_shorthand_sets_the_method_flag(cmd, args, method): _command, ctx = cmd.parse_args(args) assert ctx.flags.method == method + + +@pytest.mark.parametrize( + 'args', + [ + ['submodules', 'init', '--only', 'lib/core', '--only', 'third_party', '--resolve-relative', 'u'], + ['submodules', '--resolve-relative', 'u', 'init', '--only', 'lib/core', '--only', 'third_party'], + ['submodules', '--only', 'lib/core', 'init', '--only=third_party', '--resolve-relative=u'], + ], +) +def test_parse_args_submodules_options_keep_their_value(cmd, args): + _, ctx = cmd.parse_args(args) + assert ctx.submodules_action == 'init' + assert ctx.flags.only == ['lib/core', 'third_party'] + assert ctx.flags.resolve_relative == 'u'