feat(cloud): fail launch preflight when a private hosted zone shadows a bootstrap download host

src/kiro_crew/cloud/ec2.py

@@ -211,6 +211,104 @@ def azs_offering_instance_type(instance_type: str, profile: str, region: str) ->
     return {o.get("Location", "") for o in offerings if o.get("Location")}
 
 
+# Hosts the bootstrap MUST resolve to build the box. Keep in sync with the
+# UserData in templates/kirocrew-ec2.yaml — the kiro-cli URL is pinned to
+# us-east-1 there regardless of the launch region, so it is literal here too.
+_BOOTSTRAP_DOWNLOAD_HOSTS = (
+    "desktop-release.q.us-east-1.amazonaws.com",  # kiro-cli musl build
+    "nodejs.org",  # Node >= NODE_MAJOR_MIN tarball
+)
+
+
+def _zone_shadows_host(zone: str, host: str) -> bool:
+    """True when a hosted zone named ``zone`` is authoritative for ``host``.
+
+    A private hosted zone owns its apex **and every subdomain**, so
+    ``q.us-east-1.amazonaws.com`` shadows ``desktop-release.q.us-east-1.amazonaws.com``.
+    Matching is done on label boundaries so ``xq.us-east-1.amazonaws.com`` does
+    not match — a plain ``endswith`` would produce false positives.
+    """
+    zone = zone.rstrip(".").lower()
+    host = host.rstrip(".").lower()
+    if not zone or not host:
+        return False
+    return host == zone or host.endswith("." + zone)
+
+
+def shadowed_download_hosts(
+    vpc_id: str, profile: str, region: str
+) -> list[tuple[str, str]]:
+    """``(host, zone)`` pairs where a private hosted zone hides a download host.
+
+    An interface VPC endpoint with private DNS enabled creates a private hosted
+    zone that is authoritative for its whole domain. Amazon Q's
+    ``com.amazonaws.<region>.q`` endpoint creates one for
+    ``q.<region>.amazonaws.com`` — and kiro-cli is downloaded from
+    ``desktop-release.q.us-east-1.amazonaws.com``, which sits inside it. In such
+    a VPC the lookup is answered by the private zone, finds no matching record,
+    and returns NXDOMAIN **without falling through to public DNS**, so the
+    bootstrap dies ~4 minutes in on a name that resolves fine everywhere else.
+
+    The failure is deterministic — retries do not help — and it surfaces as
+    "kiro-cli did not install", which names the wrong layer. One read-only call
+    here turns it into a pre-launch error.
+
+    Returns an empty list when the check cannot be performed (for example the
+    launch role predates ``route53:ListHostedZonesByVPC``): a missing optional
+    permission must never block a launch that would otherwise succeed.
+    """
+    try:
+        data = aws.checked_json(
+            [
+                "route53",
+                "list-hosted-zones-by-vpc",
+                "--vpc-id",
+                vpc_id,
+                "--vpc-region",
+                region,
+            ],
+            profile,
+            region,
+            action="route53:ListHostedZonesByVPC",
+        )
+    except aws.AWSError:
+        # Non-fatal by design — see the docstring.
+        logger.info("could not list private hosted zones for %s; skipping DNS preflight", vpc_id)
+        return []
+
+    summaries = data.get("HostedZoneSummaries", []) if isinstance(data, dict) else []
+    hits: list[tuple[str, str]] = []
+    for host in _BOOTSTRAP_DOWNLOAD_HOSTS:
+        for zone in summaries:
+            name = zone.get("Name", "") if isinstance(zone, dict) else ""
+            if _zone_shadows_host(name, host):
+                hits.append((host, name.rstrip(".")))
+                break
+    return hits
+
+
+def assert_download_hosts_resolvable(vpc_id: str, profile: str, region: str) -> None:
+    """Fail fast when a private hosted zone shadows a bootstrap download host.
+
+    Raises :class:`aws.AWSError` naming the zone, the host, and the ``--subnet``
+    remedy. See :func:`shadowed_download_hosts` for why this is worth a check.
+    """
+    hits = shadowed_download_hosts(vpc_id, profile, region)
+    if not hits:
+        return
+    detail = "; ".join(f"{host} is inside private zone {zone}" for host, zone in hits)
+    raise aws.AWSError(
+        f"VPC {vpc_id} has a private hosted zone that shadows a host the bootstrap "
+        f"must download from ({detail}). Inside this VPC that name resolves to "
+        "NXDOMAIN instead of falling through to public DNS, so the install would "
+        "fail several minutes from now with a misleading error. This is usually an "
+        "interface VPC endpoint with private DNS enabled (e.g. Amazon Q's "
+        "`com.amazonaws.<region>.q`). Launch into a VPC without that endpoint via "
+        "`--subnet <subnet-id>`, or disable private DNS on the endpoint, then retry.",
+        action="route53:ListHostedZonesByVPC",
+    )
+
+
 def discover_network(
     profile: str, region: str, instance_type: str = ""
 ) -> tuple[str, str, str]:
@@ -594,6 +692,10 @@ def _cleanup_uploaded_source() -> None:
             vpc_id, subnet_id, egress_kind = discover_network(
                 profile, region, tier.instance_type
             )
+        # Both paths above settle on a VPC; check the resolver BEFORE provisioning
+        # anything. A private hosted zone that shadows a download host makes the
+        # bootstrap fail deterministically minutes later, blaming the wrong layer.
+        assert_download_hosts_resolvable(vpc_id, profile, region)
     except Exception:
         _cleanup_uploaded_source()
         raise

src/kiro_crew/cloud/iam.py

@@ -213,6 +213,20 @@ def policy_document() -> dict[str, Any]:
             ],
             "Resource": "*",
         },
+        {
+            # DNS preflight: a private hosted zone bound to the target VPC can be
+            # authoritative for a host the bootstrap downloads from (e.g. Amazon
+            # Q's `q.<region>.amazonaws.com` endpoint zone shadows
+            # desktop-release.q.us-east-1.amazonaws.com), which makes the install
+            # fail on NXDOMAIN with no fallthrough to public DNS. The launch
+            # degrades gracefully without this action — it just loses the early
+            # warning — so it is safe to omit on an older policy.
+            # ListHostedZonesByVPC does not support resource-level permissions.
+            "Sid": "Route53DnsPreflight",
+            "Effect": "Allow",
+            "Action": ["route53:ListHostedZonesByVPC"],
+            "Resource": "*",
+        },
         {
             # ec2:RunInstances — request-tag-gated on the NEW instance only.
             #

test/test_cloud_ec2.py

@@ -802,6 +802,84 @@ def _nat_route_table(subnet_ids):
     }
 
 
+class TestDnsPreflight:
+    """A private hosted zone on the target VPC can hide a bootstrap download host.
+
+    The zone is authoritative for its whole subtree, so the lookup returns
+    NXDOMAIN instead of falling through to public DNS and the bootstrap fails
+    minutes later blaming the wrong layer. These cover the detection only.
+    """
+
+    def test_zone_shadows_subdomain_and_apex(self):
+        assert ec2._zone_shadows_host(
+            "q.us-east-1.amazonaws.com.", "desktop-release.q.us-east-1.amazonaws.com"
+        )
+        assert ec2._zone_shadows_host("nodejs.org", "nodejs.org")
+
+    def test_match_is_on_label_boundaries(self):
+        # A plain endswith would wrongly match these.
+        assert not ec2._zone_shadows_host(
+            "xq.us-east-1.amazonaws.com", "desktop-release.q.us-east-1.amazonaws.com"
+        )
+        assert not ec2._zone_shadows_host("notnodejs.org", "nodejs.org")
+        # A narrower zone does not shadow a shorter name.
+        assert not ec2._zone_shadows_host("a.b.nodejs.org", "nodejs.org")
+
+    def test_empty_zone_never_shadows(self):
+        assert not ec2._zone_shadows_host("", "nodejs.org")
+        assert not ec2._zone_shadows_host(".", "nodejs.org")
+
+    def test_detects_q_endpoint_zone(self, monkeypatch):
+        def fake_json(args, profile="", region="", *, action, timeout=aws.DEFAULT_TIMEOUT):
+            if "list-hosted-zones-by-vpc" in args:
+                return {
+                    "HostedZoneSummaries": [
+                        {"Name": "q.us-east-1.amazonaws.com.", "HostedZoneId": "Z1"},
+                        {"Name": "efs.us-east-1.amazonaws.com.", "HostedZoneId": "Z2"},
+                    ]
+                }
+            return {}
+
+        monkeypatch.setattr(aws, "checked_json", fake_json)
+        hits = ec2.shadowed_download_hosts("vpc-1", "dev", "us-east-1")
+        assert hits == [
+            ("desktop-release.q.us-east-1.amazonaws.com", "q.us-east-1.amazonaws.com")
+        ]
+
+    def test_clean_vpc_has_no_hits(self, monkeypatch):
+        def fake_json(args, profile="", region="", *, action, timeout=aws.DEFAULT_TIMEOUT):
+            if "list-hosted-zones-by-vpc" in args:
+                return {"HostedZoneSummaries": [{"Name": "internal.example.com."}]}
+            return {}
+
+        monkeypatch.setattr(aws, "checked_json", fake_json)
+        assert ec2.shadowed_download_hosts("vpc-1", "dev", "us-east-1") == []
+
+    def test_missing_permission_is_not_fatal(self, monkeypatch):
+        # An older launch policy has no route53:ListHostedZonesByVPC. Losing the
+        # early warning is acceptable; blocking an otherwise-fine launch is not.
+        def fake_json(args, profile="", region="", *, action, timeout=aws.DEFAULT_TIMEOUT):
+            raise aws.AWSError("AccessDenied", action="route53:ListHostedZonesByVPC")
+
+        monkeypatch.setattr(aws, "checked_json", fake_json)
+        assert ec2.shadowed_download_hosts("vpc-1", "dev", "us-east-1") == []
+        # ...and the assert wrapper stays quiet too.
+        ec2.assert_download_hosts_resolvable("vpc-1", "dev", "us-east-1")
+
+    def test_assert_raises_with_actionable_text(self, monkeypatch):
+        def fake_json(args, profile="", region="", *, action, timeout=aws.DEFAULT_TIMEOUT):
+            if "list-hosted-zones-by-vpc" in args:
+                return {"HostedZoneSummaries": [{"Name": "q.us-east-1.amazonaws.com."}]}
+            return {}
+
+        monkeypatch.setattr(aws, "checked_json", fake_json)
+        with pytest.raises(aws.AWSError) as err:
+            ec2.assert_download_hosts_resolvable("vpc-1", "dev", "us-east-1")
+        msg = str(err.value)
+        assert "q.us-east-1.amazonaws.com" in msg
+        assert "--subnet" in msg  # the user needs a way forward, not just a diagnosis
+
+
 class TestDiscoverNetwork:
     def test_prefers_default_vpc_and_public_subnet(self, monkeypatch):
         def fake_json(args, profile="", region="", *, action, timeout=aws.DEFAULT_TIMEOUT):

test/test_cloud_iam.py

@@ -30,6 +30,9 @@ def test_covers_core_launch_actions(self):
             "ec2:DescribeInstanceTypeOfferings",
             # discover_network verifies subnet egress via route tables
             "ec2:DescribeRouteTables",
+            # DNS preflight: detect a private hosted zone that shadows a host the
+            # bootstrap downloads from (NXDOMAIN with no public fallthrough).
+            "route53:ListHostedZonesByVPC",
             "s3:CreateBucket",
             "s3:PutObject",
             # `aws cloudformation deploy` always goes through a change set.