From 63c3044939e450d0da39f0daf5fadcb516cb748c Mon Sep 17 00:00:00 2001 From: Tim WU <8848995+timwukp@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:52:15 +0800 Subject: [PATCH] feat(cloud): fail launch preflight when a private hosted zone shadows a bootstrap download host An interface VPC endpoint with private DNS enabled creates a private hosted zone authoritative for its whole domain. Amazon Q's com.amazonaws..q endpoint creates one for q..amazonaws.com, and kiro-cli is downloaded from desktop-release.q.us-east-1.amazonaws.com -- inside that subtree. The lookup is answered by the private zone, finds no record, and returns NXDOMAIN without falling through to public DNS, so the bootstrap fails minutes later with "kiro-cli did not install", naming the wrong layer. Check it before provisioning: one route53:ListHostedZonesByVPC call at the single point where the --subnet and auto-discovery paths converge. A missing permission is non-fatal so an older launch policy keeps working. Refs #7522 --- src/kiro_crew/cloud/ec2.py | 102 +++++++++++++++++++++++++++++++++++++ src/kiro_crew/cloud/iam.py | 14 +++++ test/test_cloud_ec2.py | 78 ++++++++++++++++++++++++++++ test/test_cloud_iam.py | 3 ++ 4 files changed, 197 insertions(+) diff --git a/src/kiro_crew/cloud/ec2.py b/src/kiro_crew/cloud/ec2.py index d90202f5c38..efd7392c4e0 100644 --- a/src/kiro_crew/cloud/ec2.py +++ b/src/kiro_crew/cloud/ec2.py @@ -211,6 +211,104 @@ def azs_offering_instance_type(instance_type: str, profile: str, region: str) -> return {o.get("Location", "") for o in offerings if o.get("Location")} +# Hosts the bootstrap MUST resolve to build the box. Keep in sync with the +# UserData in templates/kirocrew-ec2.yaml — the kiro-cli URL is pinned to +# us-east-1 there regardless of the launch region, so it is literal here too. +_BOOTSTRAP_DOWNLOAD_HOSTS = ( + "desktop-release.q.us-east-1.amazonaws.com", # kiro-cli musl build + "nodejs.org", # Node >= NODE_MAJOR_MIN tarball +) + + +def _zone_shadows_host(zone: str, host: str) -> bool: + """True when a hosted zone named ``zone`` is authoritative for ``host``. + + A private hosted zone owns its apex **and every subdomain**, so + ``q.us-east-1.amazonaws.com`` shadows ``desktop-release.q.us-east-1.amazonaws.com``. + Matching is done on label boundaries so ``xq.us-east-1.amazonaws.com`` does + not match — a plain ``endswith`` would produce false positives. + """ + zone = zone.rstrip(".").lower() + host = host.rstrip(".").lower() + if not zone or not host: + return False + return host == zone or host.endswith("." + zone) + + +def shadowed_download_hosts( + vpc_id: str, profile: str, region: str +) -> list[tuple[str, str]]: + """``(host, zone)`` pairs where a private hosted zone hides a download host. + + An interface VPC endpoint with private DNS enabled creates a private hosted + zone that is authoritative for its whole domain. Amazon Q's + ``com.amazonaws..q`` endpoint creates one for + ``q..amazonaws.com`` — and kiro-cli is downloaded from + ``desktop-release.q.us-east-1.amazonaws.com``, which sits inside it. In such + a VPC the lookup is answered by the private zone, finds no matching record, + and returns NXDOMAIN **without falling through to public DNS**, so the + bootstrap dies ~4 minutes in on a name that resolves fine everywhere else. + + The failure is deterministic — retries do not help — and it surfaces as + "kiro-cli did not install", which names the wrong layer. One read-only call + here turns it into a pre-launch error. + + Returns an empty list when the check cannot be performed (for example the + launch role predates ``route53:ListHostedZonesByVPC``): a missing optional + permission must never block a launch that would otherwise succeed. + """ + try: + data = aws.checked_json( + [ + "route53", + "list-hosted-zones-by-vpc", + "--vpc-id", + vpc_id, + "--vpc-region", + region, + ], + profile, + region, + action="route53:ListHostedZonesByVPC", + ) + except aws.AWSError: + # Non-fatal by design — see the docstring. + logger.info("could not list private hosted zones for %s; skipping DNS preflight", vpc_id) + return [] + + summaries = data.get("HostedZoneSummaries", []) if isinstance(data, dict) else [] + hits: list[tuple[str, str]] = [] + for host in _BOOTSTRAP_DOWNLOAD_HOSTS: + for zone in summaries: + name = zone.get("Name", "") if isinstance(zone, dict) else "" + if _zone_shadows_host(name, host): + hits.append((host, name.rstrip("."))) + break + return hits + + +def assert_download_hosts_resolvable(vpc_id: str, profile: str, region: str) -> None: + """Fail fast when a private hosted zone shadows a bootstrap download host. + + Raises :class:`aws.AWSError` naming the zone, the host, and the ``--subnet`` + remedy. See :func:`shadowed_download_hosts` for why this is worth a check. + """ + hits = shadowed_download_hosts(vpc_id, profile, region) + if not hits: + return + detail = "; ".join(f"{host} is inside private zone {zone}" for host, zone in hits) + raise aws.AWSError( + f"VPC {vpc_id} has a private hosted zone that shadows a host the bootstrap " + f"must download from ({detail}). Inside this VPC that name resolves to " + "NXDOMAIN instead of falling through to public DNS, so the install would " + "fail several minutes from now with a misleading error. This is usually an " + "interface VPC endpoint with private DNS enabled (e.g. Amazon Q's " + "`com.amazonaws..q`). Launch into a VPC without that endpoint via " + "`--subnet `, or disable private DNS on the endpoint, then retry.", + action="route53:ListHostedZonesByVPC", + ) + + def discover_network( profile: str, region: str, instance_type: str = "" ) -> tuple[str, str, str]: @@ -594,6 +692,10 @@ def _cleanup_uploaded_source() -> None: vpc_id, subnet_id, egress_kind = discover_network( profile, region, tier.instance_type ) + # Both paths above settle on a VPC; check the resolver BEFORE provisioning + # anything. A private hosted zone that shadows a download host makes the + # bootstrap fail deterministically minutes later, blaming the wrong layer. + assert_download_hosts_resolvable(vpc_id, profile, region) except Exception: _cleanup_uploaded_source() raise diff --git a/src/kiro_crew/cloud/iam.py b/src/kiro_crew/cloud/iam.py index 24221b6393c..d72ec3c379e 100644 --- a/src/kiro_crew/cloud/iam.py +++ b/src/kiro_crew/cloud/iam.py @@ -213,6 +213,20 @@ def policy_document() -> dict[str, Any]: ], "Resource": "*", }, + { + # DNS preflight: a private hosted zone bound to the target VPC can be + # authoritative for a host the bootstrap downloads from (e.g. Amazon + # Q's `q..amazonaws.com` endpoint zone shadows + # desktop-release.q.us-east-1.amazonaws.com), which makes the install + # fail on NXDOMAIN with no fallthrough to public DNS. The launch + # degrades gracefully without this action — it just loses the early + # warning — so it is safe to omit on an older policy. + # ListHostedZonesByVPC does not support resource-level permissions. + "Sid": "Route53DnsPreflight", + "Effect": "Allow", + "Action": ["route53:ListHostedZonesByVPC"], + "Resource": "*", + }, { # ec2:RunInstances — request-tag-gated on the NEW instance only. # diff --git a/test/test_cloud_ec2.py b/test/test_cloud_ec2.py index afed4a78465..ac9567dee2f 100644 --- a/test/test_cloud_ec2.py +++ b/test/test_cloud_ec2.py @@ -802,6 +802,84 @@ def _nat_route_table(subnet_ids): } +class TestDnsPreflight: + """A private hosted zone on the target VPC can hide a bootstrap download host. + + The zone is authoritative for its whole subtree, so the lookup returns + NXDOMAIN instead of falling through to public DNS and the bootstrap fails + minutes later blaming the wrong layer. These cover the detection only. + """ + + def test_zone_shadows_subdomain_and_apex(self): + assert ec2._zone_shadows_host( + "q.us-east-1.amazonaws.com.", "desktop-release.q.us-east-1.amazonaws.com" + ) + assert ec2._zone_shadows_host("nodejs.org", "nodejs.org") + + def test_match_is_on_label_boundaries(self): + # A plain endswith would wrongly match these. + assert not ec2._zone_shadows_host( + "xq.us-east-1.amazonaws.com", "desktop-release.q.us-east-1.amazonaws.com" + ) + assert not ec2._zone_shadows_host("notnodejs.org", "nodejs.org") + # A narrower zone does not shadow a shorter name. + assert not ec2._zone_shadows_host("a.b.nodejs.org", "nodejs.org") + + def test_empty_zone_never_shadows(self): + assert not ec2._zone_shadows_host("", "nodejs.org") + assert not ec2._zone_shadows_host(".", "nodejs.org") + + def test_detects_q_endpoint_zone(self, monkeypatch): + def fake_json(args, profile="", region="", *, action, timeout=aws.DEFAULT_TIMEOUT): + if "list-hosted-zones-by-vpc" in args: + return { + "HostedZoneSummaries": [ + {"Name": "q.us-east-1.amazonaws.com.", "HostedZoneId": "Z1"}, + {"Name": "efs.us-east-1.amazonaws.com.", "HostedZoneId": "Z2"}, + ] + } + return {} + + monkeypatch.setattr(aws, "checked_json", fake_json) + hits = ec2.shadowed_download_hosts("vpc-1", "dev", "us-east-1") + assert hits == [ + ("desktop-release.q.us-east-1.amazonaws.com", "q.us-east-1.amazonaws.com") + ] + + def test_clean_vpc_has_no_hits(self, monkeypatch): + def fake_json(args, profile="", region="", *, action, timeout=aws.DEFAULT_TIMEOUT): + if "list-hosted-zones-by-vpc" in args: + return {"HostedZoneSummaries": [{"Name": "internal.example.com."}]} + return {} + + monkeypatch.setattr(aws, "checked_json", fake_json) + assert ec2.shadowed_download_hosts("vpc-1", "dev", "us-east-1") == [] + + def test_missing_permission_is_not_fatal(self, monkeypatch): + # An older launch policy has no route53:ListHostedZonesByVPC. Losing the + # early warning is acceptable; blocking an otherwise-fine launch is not. + def fake_json(args, profile="", region="", *, action, timeout=aws.DEFAULT_TIMEOUT): + raise aws.AWSError("AccessDenied", action="route53:ListHostedZonesByVPC") + + monkeypatch.setattr(aws, "checked_json", fake_json) + assert ec2.shadowed_download_hosts("vpc-1", "dev", "us-east-1") == [] + # ...and the assert wrapper stays quiet too. + ec2.assert_download_hosts_resolvable("vpc-1", "dev", "us-east-1") + + def test_assert_raises_with_actionable_text(self, monkeypatch): + def fake_json(args, profile="", region="", *, action, timeout=aws.DEFAULT_TIMEOUT): + if "list-hosted-zones-by-vpc" in args: + return {"HostedZoneSummaries": [{"Name": "q.us-east-1.amazonaws.com."}]} + return {} + + monkeypatch.setattr(aws, "checked_json", fake_json) + with pytest.raises(aws.AWSError) as err: + ec2.assert_download_hosts_resolvable("vpc-1", "dev", "us-east-1") + msg = str(err.value) + assert "q.us-east-1.amazonaws.com" in msg + assert "--subnet" in msg # the user needs a way forward, not just a diagnosis + + class TestDiscoverNetwork: def test_prefers_default_vpc_and_public_subnet(self, monkeypatch): def fake_json(args, profile="", region="", *, action, timeout=aws.DEFAULT_TIMEOUT): diff --git a/test/test_cloud_iam.py b/test/test_cloud_iam.py index 5a00b71bf52..ba3af3b1ff5 100644 --- a/test/test_cloud_iam.py +++ b/test/test_cloud_iam.py @@ -30,6 +30,9 @@ def test_covers_core_launch_actions(self): "ec2:DescribeInstanceTypeOfferings", # discover_network verifies subnet egress via route tables "ec2:DescribeRouteTables", + # DNS preflight: detect a private hosted zone that shadows a host the + # bootstrap downloads from (NXDOMAIN with no public fallthrough). + "route53:ListHostedZonesByVPC", "s3:CreateBucket", "s3:PutObject", # `aws cloudformation deploy` always goes through a change set.