diff --git a/src/kiro_crew/cloud/ec2.py b/src/kiro_crew/cloud/ec2.py index d90202f5c38..efd7392c4e0 100644 --- a/src/kiro_crew/cloud/ec2.py +++ b/src/kiro_crew/cloud/ec2.py @@ -211,6 +211,104 @@ def azs_offering_instance_type(instance_type: str, profile: str, region: str) -> return {o.get("Location", "") for o in offerings if o.get("Location")} +# Hosts the bootstrap MUST resolve to build the box. Keep in sync with the +# UserData in templates/kirocrew-ec2.yaml — the kiro-cli URL is pinned to +# us-east-1 there regardless of the launch region, so it is literal here too. +_BOOTSTRAP_DOWNLOAD_HOSTS = ( + "desktop-release.q.us-east-1.amazonaws.com", # kiro-cli musl build + "nodejs.org", # Node >= NODE_MAJOR_MIN tarball +) + + +def _zone_shadows_host(zone: str, host: str) -> bool: + """True when a hosted zone named ``zone`` is authoritative for ``host``. + + A private hosted zone owns its apex **and every subdomain**, so + ``q.us-east-1.amazonaws.com`` shadows ``desktop-release.q.us-east-1.amazonaws.com``. + Matching is done on label boundaries so ``xq.us-east-1.amazonaws.com`` does + not match — a plain ``endswith`` would produce false positives. + """ + zone = zone.rstrip(".").lower() + host = host.rstrip(".").lower() + if not zone or not host: + return False + return host == zone or host.endswith("." + zone) + + +def shadowed_download_hosts( + vpc_id: str, profile: str, region: str +) -> list[tuple[str, str]]: + """``(host, zone)`` pairs where a private hosted zone hides a download host. + + An interface VPC endpoint with private DNS enabled creates a private hosted + zone that is authoritative for its whole domain. Amazon Q's + ``com.amazonaws..q`` endpoint creates one for + ``q..amazonaws.com`` — and kiro-cli is downloaded from + ``desktop-release.q.us-east-1.amazonaws.com``, which sits inside it. In such + a VPC the lookup is answered by the private zone, finds no matching record, + and returns NXDOMAIN **without falling through to public DNS**, so the + bootstrap dies ~4 minutes in on a name that resolves fine everywhere else. + + The failure is deterministic — retries do not help — and it surfaces as + "kiro-cli did not install", which names the wrong layer. One read-only call + here turns it into a pre-launch error. + + Returns an empty list when the check cannot be performed (for example the + launch role predates ``route53:ListHostedZonesByVPC``): a missing optional + permission must never block a launch that would otherwise succeed. + """ + try: + data = aws.checked_json( + [ + "route53", + "list-hosted-zones-by-vpc", + "--vpc-id", + vpc_id, + "--vpc-region", + region, + ], + profile, + region, + action="route53:ListHostedZonesByVPC", + ) + except aws.AWSError: + # Non-fatal by design — see the docstring. + logger.info("could not list private hosted zones for %s; skipping DNS preflight", vpc_id) + return [] + + summaries = data.get("HostedZoneSummaries", []) if isinstance(data, dict) else [] + hits: list[tuple[str, str]] = [] + for host in _BOOTSTRAP_DOWNLOAD_HOSTS: + for zone in summaries: + name = zone.get("Name", "") if isinstance(zone, dict) else "" + if _zone_shadows_host(name, host): + hits.append((host, name.rstrip("."))) + break + return hits + + +def assert_download_hosts_resolvable(vpc_id: str, profile: str, region: str) -> None: + """Fail fast when a private hosted zone shadows a bootstrap download host. + + Raises :class:`aws.AWSError` naming the zone, the host, and the ``--subnet`` + remedy. See :func:`shadowed_download_hosts` for why this is worth a check. + """ + hits = shadowed_download_hosts(vpc_id, profile, region) + if not hits: + return + detail = "; ".join(f"{host} is inside private zone {zone}" for host, zone in hits) + raise aws.AWSError( + f"VPC {vpc_id} has a private hosted zone that shadows a host the bootstrap " + f"must download from ({detail}). Inside this VPC that name resolves to " + "NXDOMAIN instead of falling through to public DNS, so the install would " + "fail several minutes from now with a misleading error. This is usually an " + "interface VPC endpoint with private DNS enabled (e.g. Amazon Q's " + "`com.amazonaws..q`). Launch into a VPC without that endpoint via " + "`--subnet `, or disable private DNS on the endpoint, then retry.", + action="route53:ListHostedZonesByVPC", + ) + + def discover_network( profile: str, region: str, instance_type: str = "" ) -> tuple[str, str, str]: @@ -594,6 +692,10 @@ def _cleanup_uploaded_source() -> None: vpc_id, subnet_id, egress_kind = discover_network( profile, region, tier.instance_type ) + # Both paths above settle on a VPC; check the resolver BEFORE provisioning + # anything. A private hosted zone that shadows a download host makes the + # bootstrap fail deterministically minutes later, blaming the wrong layer. + assert_download_hosts_resolvable(vpc_id, profile, region) except Exception: _cleanup_uploaded_source() raise diff --git a/src/kiro_crew/cloud/iam.py b/src/kiro_crew/cloud/iam.py index 24221b6393c..d72ec3c379e 100644 --- a/src/kiro_crew/cloud/iam.py +++ b/src/kiro_crew/cloud/iam.py @@ -213,6 +213,20 @@ def policy_document() -> dict[str, Any]: ], "Resource": "*", }, + { + # DNS preflight: a private hosted zone bound to the target VPC can be + # authoritative for a host the bootstrap downloads from (e.g. Amazon + # Q's `q..amazonaws.com` endpoint zone shadows + # desktop-release.q.us-east-1.amazonaws.com), which makes the install + # fail on NXDOMAIN with no fallthrough to public DNS. The launch + # degrades gracefully without this action — it just loses the early + # warning — so it is safe to omit on an older policy. + # ListHostedZonesByVPC does not support resource-level permissions. + "Sid": "Route53DnsPreflight", + "Effect": "Allow", + "Action": ["route53:ListHostedZonesByVPC"], + "Resource": "*", + }, { # ec2:RunInstances — request-tag-gated on the NEW instance only. # diff --git a/test/test_cloud_ec2.py b/test/test_cloud_ec2.py index afed4a78465..ac9567dee2f 100644 --- a/test/test_cloud_ec2.py +++ b/test/test_cloud_ec2.py @@ -802,6 +802,84 @@ def _nat_route_table(subnet_ids): } +class TestDnsPreflight: + """A private hosted zone on the target VPC can hide a bootstrap download host. + + The zone is authoritative for its whole subtree, so the lookup returns + NXDOMAIN instead of falling through to public DNS and the bootstrap fails + minutes later blaming the wrong layer. These cover the detection only. + """ + + def test_zone_shadows_subdomain_and_apex(self): + assert ec2._zone_shadows_host( + "q.us-east-1.amazonaws.com.", "desktop-release.q.us-east-1.amazonaws.com" + ) + assert ec2._zone_shadows_host("nodejs.org", "nodejs.org") + + def test_match_is_on_label_boundaries(self): + # A plain endswith would wrongly match these. + assert not ec2._zone_shadows_host( + "xq.us-east-1.amazonaws.com", "desktop-release.q.us-east-1.amazonaws.com" + ) + assert not ec2._zone_shadows_host("notnodejs.org", "nodejs.org") + # A narrower zone does not shadow a shorter name. + assert not ec2._zone_shadows_host("a.b.nodejs.org", "nodejs.org") + + def test_empty_zone_never_shadows(self): + assert not ec2._zone_shadows_host("", "nodejs.org") + assert not ec2._zone_shadows_host(".", "nodejs.org") + + def test_detects_q_endpoint_zone(self, monkeypatch): + def fake_json(args, profile="", region="", *, action, timeout=aws.DEFAULT_TIMEOUT): + if "list-hosted-zones-by-vpc" in args: + return { + "HostedZoneSummaries": [ + {"Name": "q.us-east-1.amazonaws.com.", "HostedZoneId": "Z1"}, + {"Name": "efs.us-east-1.amazonaws.com.", "HostedZoneId": "Z2"}, + ] + } + return {} + + monkeypatch.setattr(aws, "checked_json", fake_json) + hits = ec2.shadowed_download_hosts("vpc-1", "dev", "us-east-1") + assert hits == [ + ("desktop-release.q.us-east-1.amazonaws.com", "q.us-east-1.amazonaws.com") + ] + + def test_clean_vpc_has_no_hits(self, monkeypatch): + def fake_json(args, profile="", region="", *, action, timeout=aws.DEFAULT_TIMEOUT): + if "list-hosted-zones-by-vpc" in args: + return {"HostedZoneSummaries": [{"Name": "internal.example.com."}]} + return {} + + monkeypatch.setattr(aws, "checked_json", fake_json) + assert ec2.shadowed_download_hosts("vpc-1", "dev", "us-east-1") == [] + + def test_missing_permission_is_not_fatal(self, monkeypatch): + # An older launch policy has no route53:ListHostedZonesByVPC. Losing the + # early warning is acceptable; blocking an otherwise-fine launch is not. + def fake_json(args, profile="", region="", *, action, timeout=aws.DEFAULT_TIMEOUT): + raise aws.AWSError("AccessDenied", action="route53:ListHostedZonesByVPC") + + monkeypatch.setattr(aws, "checked_json", fake_json) + assert ec2.shadowed_download_hosts("vpc-1", "dev", "us-east-1") == [] + # ...and the assert wrapper stays quiet too. + ec2.assert_download_hosts_resolvable("vpc-1", "dev", "us-east-1") + + def test_assert_raises_with_actionable_text(self, monkeypatch): + def fake_json(args, profile="", region="", *, action, timeout=aws.DEFAULT_TIMEOUT): + if "list-hosted-zones-by-vpc" in args: + return {"HostedZoneSummaries": [{"Name": "q.us-east-1.amazonaws.com."}]} + return {} + + monkeypatch.setattr(aws, "checked_json", fake_json) + with pytest.raises(aws.AWSError) as err: + ec2.assert_download_hosts_resolvable("vpc-1", "dev", "us-east-1") + msg = str(err.value) + assert "q.us-east-1.amazonaws.com" in msg + assert "--subnet" in msg # the user needs a way forward, not just a diagnosis + + class TestDiscoverNetwork: def test_prefers_default_vpc_and_public_subnet(self, monkeypatch): def fake_json(args, profile="", region="", *, action, timeout=aws.DEFAULT_TIMEOUT): diff --git a/test/test_cloud_iam.py b/test/test_cloud_iam.py index 5a00b71bf52..ba3af3b1ff5 100644 --- a/test/test_cloud_iam.py +++ b/test/test_cloud_iam.py @@ -30,6 +30,9 @@ def test_covers_core_launch_actions(self): "ec2:DescribeInstanceTypeOfferings", # discover_network verifies subnet egress via route tables "ec2:DescribeRouteTables", + # DNS preflight: detect a private hosted zone that shadows a host the + # bootstrap downloads from (NXDOMAIN with no public fallthrough). + "route53:ListHostedZonesByVPC", "s3:CreateBucket", "s3:PutObject", # `aws cloudformation deploy` always goes through a change set.