From e26760d19e2cdee89d63761d6c20878f2fbdad49 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 13:28:02 -0700 Subject: [PATCH 01/20] fix(egress): let self-hosted deployments reach named private destinations (#7200) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #7200 reports "HTTP connection not working in Docker". The screenshot shows the real failure: `url must use https:// protocol` on `http://host.docker.internal:7274/...`. Sim refused the URL before opening a socket, and the message pointed at the wrong cause — switching to https would have failed too, on the private-address check one step later. Plain http was permitted only for a literal `localhost` or loopback IP. In a container loopback is the one address guaranteed useless, so the carve-out existed exactly where it could not help. Behind it, the DNS filter dropped every private address with no operator opt-out, which is why self-hosted Sim could not reach a LAN vLLM, a Jupyter server, GitHub Enterprise, or a sibling container by service name either. The cause was structural: there was no policy object, just a hardcoded default plus four incompatible escape hatches grown one per use case — `allowHttp` (protocol only), a hardcoded `isLocalhost && !isHosted`, a hand-threaded `allowRedirectToIp` for MCP, and `ALLOW_PRIVATE_DATABASE_HOSTS` for one validator. They disagreed. `allowHttp: true` relaxed the protocol gate while the address gate refused the host anyway, so vLLM and Jupyter on a LAN were broken despite passing it. Replaces all four with one policy: - `@sim/security/egress` — a pure policy value and two decision functions. No DNS, no env, no deployment-posture global, so both postures are testable in one file with no module mocking. The hosted branch of the guard had no coverage before, because `isHosted` is a module const pinned to `false`. - `lib/core/security/egress` — profiles keyed on where the URL came from: `configuredEndpoint`, `requestTarget`, `databaseHost`, `contentFetch`, `proxy`. Provenance is what determines trust, and `contentFetch` — the class where SSRF is actually exploited — never consults the allowlist, even on a deployment that allowlisted its whole internal range. - The profile is required at all ~190 call sites and travels on `SecureFetchOptions`, so every redirect hop is judged by the policy the request started under. `allowHttp` and `allowRedirectToIp` are gone. Operators name destinations with `EGRESS_ALLOWED_HOSTS` and `EGRESS_ALLOWED_IP_RANGES`. Naming one permits plain http to it and lifts the blocked-port list for it, because those are one decision about one host. Cloud metadata endpoints stay blocked however broad the allowlist is, and both variables are ignored on the hosted platform. `ALLOW_PRIVATE_DATABASE_HOSTS` keeps working as a deprecated alias that expands to the private space it always stood for, so existing deployments are unaffected; it is dropped from the docs in favour of naming specific destinations. Also removes what the consolidation made redundant: nine dead exports, three copies of "validate then host-suffix allowlist", a duplicate of `validateJiraCloudId`, and the second `validateUrlWithDNS` in the API block handler, which discarded its pinned address and checked a pre-templating string that was not the URL dialled. Adds `check:egress-boundary` so a raw HTTP transport outside the guard fails CI. Compose files gain `host.docker.internal:host-gateway`; without it the hostname does not resolve on Linux at all, which was a third distinct error for one cause. --- .../self-hosting/environment-variables.mdx | 3 +- .../docs/platform/self-hosting/security.mdx | 28 +- apps/sim/.env.example | 3 +- apps/sim/app/api/auth/sso/register/route.ts | 19 +- apps/sim/app/api/link-preview/route.ts | 1 + .../app/api/tools/zoho_desk/agents/route.ts | 1 + .../api/tools/zoho_desk/departments/route.ts | 1 + .../tools/zoho_desk/organizations/route.ts | 1 + apps/sim/connectors/gitlab/gitlab.ts | 15 +- apps/sim/connectors/mintlify/mintlify.ts | 3 +- apps/sim/connectors/obsidian/obsidian.ts | 5 +- apps/sim/connectors/s3/s3.ts | 3 +- apps/sim/connectors/sentry/sentry.ts | 13 +- apps/sim/connectors/zendesk/zendesk.ts | 1 + .../executor/handlers/api/api-handler.test.ts | 34 +- apps/sim/executor/handlers/api/api-handler.ts | 30 +- apps/sim/lib/a2a/client.ts | 3 +- apps/sim/lib/api/contracts/data-drains.ts | 4 +- .../files/download-to-workspace-file.ts | 1 + apps/sim/lib/core/config/env-flags.ts | 67 +- apps/sim/lib/core/config/env.ts | 4 +- apps/sim/lib/core/security/egress/profiles.ts | 180 ++++ apps/sim/lib/core/security/egress/validate.ts | 154 ++++ .../security/input-validation.server.test.ts | 38 +- .../core/security/input-validation.server.ts | 166 ++-- .../core/security/input-validation.test.ts | 841 ++++-------------- .../sim/lib/core/security/input-validation.ts | 703 +++------------ .../pinned-redirect-replay.server.test.ts | 23 +- .../secure-fetch-response-cap.server.test.ts | 3 +- apps/sim/lib/data-drains/destinations/s3.ts | 4 +- .../lib/data-drains/destinations/webhook.ts | 6 +- apps/sim/lib/execution/isolated-vm.ts | 7 +- .../execute-request.test.ts | 24 +- apps/sim/lib/internal/agiloft/client.test.ts | 3 +- apps/sim/lib/internal/agiloft/client.ts | 7 +- .../lib/internal/agiloft/operations.test.ts | 1 + apps/sim/lib/internal/agiloft/operations.ts | 8 +- .../internal/azure-data-explorer/client.ts | 2 + apps/sim/lib/internal/brex/client.test.ts | 1 + apps/sim/lib/internal/brex/client.ts | 3 +- apps/sim/lib/internal/buffer/operations.ts | 3 +- .../lib/internal/clickhouse/client.test.ts | 4 +- apps/sim/lib/internal/clickhouse/client.ts | 2 +- .../lib/internal/cursor/operations.test.ts | 2 +- apps/sim/lib/internal/cursor/operations.ts | 3 +- apps/sim/lib/internal/extend/client.ts | 7 +- apps/sim/lib/internal/github/operations.ts | 6 +- .../lib/internal/google-drive/client.test.ts | 3 +- apps/sim/lib/internal/google-drive/client.ts | 3 +- .../lib/internal/google-slides/operations.ts | 7 +- .../internal/google-vault/operations.test.ts | 1 + .../lib/internal/google-vault/operations.ts | 3 +- apps/sim/lib/internal/grafana/client.test.ts | 3 +- apps/sim/lib/internal/grafana/client.ts | 3 +- apps/sim/lib/internal/image/fetch.ts | 3 +- apps/sim/lib/internal/image/operations.ts | 3 +- apps/sim/lib/internal/jsm/client.ts | 7 +- apps/sim/lib/internal/jupyter/client.test.ts | 4 +- apps/sim/lib/internal/jupyter/client.ts | 4 +- apps/sim/lib/internal/linq/client.ts | 3 +- .../internal/microsoft-dataverse/client.ts | 1 + .../sim/lib/internal/microsoft-word/client.ts | 9 +- apps/sim/lib/internal/mistral/client.ts | 7 +- apps/sim/lib/internal/mistral/operations.ts | 2 +- apps/sim/lib/internal/onedrive/operations.ts | 7 +- .../lib/internal/onepassword/client.test.ts | 2 +- apps/sim/lib/internal/onepassword/client.ts | 2 +- apps/sim/lib/internal/pipedrive/client.ts | 6 +- apps/sim/lib/internal/pulse/client.ts | 3 +- apps/sim/lib/internal/reducto/client.ts | 7 +- apps/sim/lib/internal/sap-concur/client.ts | 3 + apps/sim/lib/internal/sap-s4hana/client.ts | 3 + apps/sim/lib/internal/servicenow/client.ts | 1 + .../lib/internal/sharepoint/client.test.ts | 2 + apps/sim/lib/internal/sharepoint/client.ts | 7 +- .../sim/lib/internal/slack/operations.test.ts | 1 + apps/sim/lib/internal/slack/operations.ts | 4 +- apps/sim/lib/internal/stagehand/operations.ts | 4 +- apps/sim/lib/internal/stt/operations.ts | 3 +- .../lib/internal/textract/document-input.ts | 5 +- .../lib/internal/twilio-voice/operations.ts | 3 +- apps/sim/lib/internal/typeform/operations.ts | 3 +- apps/sim/lib/internal/vision/client.test.ts | 1 + apps/sim/lib/internal/vision/client.ts | 1 + .../lib/internal/vision/operations.test.ts | 3 +- apps/sim/lib/internal/vision/operations.ts | 2 +- apps/sim/lib/internal/whatsapp/operations.ts | 3 +- apps/sim/lib/internal/windchill/client.ts | 5 + apps/sim/lib/internal/zoho-desk/operations.ts | 1 + apps/sim/lib/internal/zoom/operations.ts | 10 +- apps/sim/lib/internal/zoominfo/client.ts | 2 + .../documents/secure-fetch.server.ts | 6 +- .../sim/lib/knowledge/documents/utils.test.ts | 12 +- apps/sim/lib/media/falai.ts | 3 +- .../contexts/workspace/fetch-external-url.ts | 3 +- .../lib/uploads/utils/file-utils.server.ts | 5 +- apps/sim/lib/webhooks/polling/rss.ts | 3 +- apps/sim/lib/webhooks/providers/emailbison.ts | 6 +- apps/sim/lib/webhooks/providers/gitlab.ts | 8 +- .../lib/webhooks/providers/microsoft-teams.ts | 7 +- apps/sim/lib/webhooks/providers/slack.ts | 3 +- .../providers/azure-anthropic/index.test.ts | 6 +- apps/sim/providers/azure-anthropic/index.ts | 6 +- apps/sim/providers/azure-openai/index.test.ts | 6 +- apps/sim/providers/azure-openai/index.ts | 6 +- apps/sim/providers/vllm/index.test.ts | 4 +- apps/sim/providers/vllm/index.ts | 17 +- apps/sim/tools/bitbucket/utils.server.ts | 9 +- apps/sim/tools/convex/utils.ts | 2 +- apps/sim/tools/github/utils.server.test.ts | 3 +- apps/sim/tools/github/utils.server.ts | 3 +- apps/sim/tools/index.test.ts | 6 +- apps/sim/tools/index.ts | 3 +- apps/sim/tools/posthog/utils.ts | 2 +- docker-compose.local.yml | 5 + docker-compose.ollama.yml | 5 + docker-compose.prod.yml | 5 + package.json | 1 + packages/security/package.json | 4 + packages/security/src/egress.test.ts | 262 ++++++ packages/security/src/egress.ts | 377 ++++++++ packages/sim-setup/src/steps.ts | 13 +- packages/testing/src/mocks/env-flags.mock.ts | 10 +- scripts/check-egress-boundary.ts | 102 +++ 124 files changed, 1929 insertions(+), 1585 deletions(-) create mode 100644 apps/sim/lib/core/security/egress/profiles.ts create mode 100644 apps/sim/lib/core/security/egress/validate.ts create mode 100644 packages/security/src/egress.test.ts create mode 100644 packages/security/src/egress.ts create mode 100644 scripts/check-egress-boundary.ts diff --git a/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx b/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx index 378d6080f8d..2fd60ac5faf 100644 --- a/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx +++ b/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx @@ -147,7 +147,8 @@ Without a remote provider, user code runs in an in-process V8 isolate inside the | `WEBHOOK_EXECUTION_CONCURRENCY_LIMIT` | `75` | Webhook-triggered executions in parallel | | `SCHEDULE_EXECUTION_CONCURRENCY_LIMIT` | `30` | Scheduled executions in parallel | | `RESUME_EXECUTION_CONCURRENCY_LIMIT` | `50` | Resumed executions in parallel | -| `ALLOW_PRIVATE_DATABASE_HOSTS` | unset | Let database/connector tools reach private, reserved, and loopback hosts. Loosens the SSRF boundary | +| `EGRESS_ALLOWED_HOSTS` | unset | Comma-separated hostnames outbound requests may reach on a private network. Leading wildcard allowed, e.g. `host.docker.internal,*.svc.cluster.local` | +| `EGRESS_ALLOWED_IP_RANGES` | unset | Comma-separated CIDRs or IPs outbound requests may reach on a private network, e.g. `10.0.0.0/8` | Your reverse proxy's body-size limit must be at least as large as the app limits above. See [Networking](/platform/self-hosting/networking). diff --git a/apps/docs/content/docs/platform/self-hosting/security.mdx b/apps/docs/content/docs/platform/self-hosting/security.mdx index 0b67762d142..61768cf0adb 100644 --- a/apps/docs/content/docs/platform/self-hosting/security.mdx +++ b/apps/docs/content/docs/platform/self-hosting/security.mdx @@ -134,16 +134,34 @@ Resource ceilings for the in-process path: ## The SSRF boundary -Sim blocks outbound requests from database and connector tools to private, reserved, and loopback addresses. This stops a workflow from being used to scan your internal network. +Sim blocks outbound requests to private, reserved, and loopback addresses. This stops a workflow from being used to scan your internal network. Every outbound request is classified by where its URL came from: -Self-hosted deployments often legitimately need to reach an internal database by service name. That is opt-in: +| Provenance | Examples | Reaches allowlisted private destinations | +|---|---|---| +| Configured endpoint | A self-hosted vLLM, Jupyter, GitHub Enterprise, Grafana, ClickHouse, an MCP server, a connector's host | Yes | +| Request target | The HTTP block's URL, an A2A agent, an RSS feed, a Function block's `fetch` | Yes | +| Database host | A database, cache, or mail connector's host | Yes | +| Content fetch | An image URL, a file imported by URL, a link from a third-party API response | **No** | + +Content fetches never reach a private destination, allowlist or not — that is the class where SSRF is actually exploited. + +Deployments frequently need to reach an internal service by name or address. Name the destinations: + +```bash +EGRESS_ALLOWED_HOSTS=host.docker.internal,*.svc.cluster.local +EGRESS_ALLOWED_IP_RANGES=10.0.0.0/8,192.168.65.254/32 +``` + +Naming a destination permits plain HTTP to it and lifts the blocked-port list for it, since those are the same decision about the same host. Cloud metadata endpoints (`169.254.169.254` and equivalents) stay blocked no matter how broad the allowlist is, and both variables are ignored entirely on Sim Cloud. + +To reach a service on the Docker host, pair the allowlist with the host alias that Compose already sets up: ```bash -ALLOW_PRIVATE_DATABASE_HOSTS=true +EGRESS_ALLOWED_HOSTS=host.docker.internal ``` - This loosens the SSRF boundary for every workflow author on the instance. Enable it only on a trusted private network, and prefer pairing it with a NetworkPolicy that constrains what the app can actually reach. + An allowlist widens what every workflow author on the instance can reach. Name specific hosts and narrow ranges rather than whole private networks, and pair it with a NetworkPolicy that constrains what the app can actually reach. ## Client IP and forwarded headers @@ -192,5 +210,5 @@ The service bundles ~2.2 GB of spaCy models, so first start takes around three m { question: "Can I rotate ENCRYPTION_KEY?", answer: "Not without re-encrypting everything it protects. Changing it makes workspace environment variables, stored provider API keys, MCP OAuth credentials, and deployment secrets permanently unreadable. Treat it as a permanent, backed-up value rather than a rotating secret."}, { question: "Where does user-authored code run?", answer: "By default in an in-process V8 isolate inside the app container, which isolates at the JS-engine level but shares the container's network and filesystem context. For untrusted authors, or to run Python at all, use E2B or Daytona so each execution runs in a remote sandbox."}, { question: "Why does the chart's NetworkPolicy allow traffic from any pod?", answer: "networkPolicy.ingressFrom defaults to an empty peer selector as a simple default that works on any cluster. On a shared cluster you should scope it to your ingress controller's namespace."}, - { question: "What does ALLOW_PRIVATE_DATABASE_HOSTS change?", answer: "It lets database and connector tools reach private, reserved, and loopback addresses — needed to connect to an internal database by Kubernetes service name. It also widens the SSRF boundary for every workflow author, so enable it only on a trusted network."}, + { question: "How do I reach an internal service from a workflow?", answer: "Name it in EGRESS_ALLOWED_HOSTS (hostnames, leading wildcard allowed) or EGRESS_ALLOWED_IP_RANGES (CIDRs). That permits plain HTTP to it and lifts the blocked-port list for it. Cloud metadata endpoints stay blocked regardless, content fetches never use the allowlist, and both variables are ignored on Sim Cloud."}, ]} /> diff --git a/apps/sim/.env.example b/apps/sim/.env.example index 443ff1d2da9..1ec0347adac 100644 --- a/apps/sim/.env.example +++ b/apps/sim/.env.example @@ -13,7 +13,8 @@ BETTER_AUTH_URL=http://localhost:3000 # DISABLE_AUTH=true # Uncomment to bypass authentication entirely. Creates an anonymous session for all requests. # Private Database Hosts (Optional - for self-hosted deployments only) -# ALLOW_PRIVATE_DATABASE_HOSTS=true # Uncomment to let database/connector tools reach private/reserved/loopback hosts (e.g. Docker/K8s service names, localhost). Loosens the SSRF boundary; only enable on a trusted private network. +# EGRESS_ALLOWED_HOSTS=host.docker.internal,*.svc.cluster.local # Uncomment to let outbound requests reach these hosts on a private network. Widens the SSRF boundary; only use on a trusted private network. +# EGRESS_ALLOWED_IP_RANGES=10.0.0.0/8 # Same, by CIDR. Cloud metadata endpoints stay blocked regardless. # NextJS (Required) NEXT_PUBLIC_APP_URL=http://localhost:3000 diff --git a/apps/sim/app/api/auth/sso/register/route.ts b/apps/sim/app/api/auth/sso/register/route.ts index c009e56b915..b7b120b215a 100644 --- a/apps/sim/app/api/auth/sso/register/route.ts +++ b/apps/sim/app/api/auth/sso/register/route.ts @@ -56,13 +56,18 @@ type DiscoveryResult = const OIDC_DISCOVERY_TIMEOUT_MS = 10000 async function fetchOIDCDiscoveryDocument(discoveryUrl: string): Promise { - const urlValidation = await validateUrlWithDNS(discoveryUrl, 'OIDC discovery URL') + const urlValidation = await validateUrlWithDNS( + discoveryUrl, + 'OIDC discovery URL', + 'configuredEndpoint' + ) if (!urlValidation.isValid || !urlValidation.resolvedIP) { return { ok: false, error: urlValidation.error ?? 'SSRF validation failed' } } try { const response = await secureFetchWithPinnedIP(discoveryUrl, urlValidation.resolvedIP, { + profile: 'configuredEndpoint', headers: { Accept: 'application/json' }, timeout: OIDC_DISCOVERY_TIMEOUT_MS, }) @@ -317,7 +322,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => { for (const [name, endpointUrl] of Object.entries(userProvidedEndpoints)) { if (endpointUrl) { - const endpointValidation = await validateUrlWithDNS(endpointUrl, `OIDC ${name}`) + const endpointValidation = await validateUrlWithDNS( + endpointUrl, + `OIDC ${name}`, + 'configuredEndpoint' + ) if (!endpointValidation.isValid) { logger.warn('Explicitly provided OIDC endpoint failed SSRF validation', { endpoint: name, @@ -369,7 +378,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => { for (const [key, value] of Object.entries(discoveredEndpoints)) { if (typeof value === 'string') { - const endpointValidation = await validateUrlWithDNS(value, `OIDC ${key}`) + const endpointValidation = await validateUrlWithDNS( + value, + `OIDC ${key}`, + 'contentFetch' + ) if (!endpointValidation.isValid) { logger.warn('OIDC discovered endpoint failed SSRF validation', { endpoint: key, diff --git a/apps/sim/app/api/link-preview/route.ts b/apps/sim/app/api/link-preview/route.ts index 5dc8c230df1..75fb85bae54 100644 --- a/apps/sim/app/api/link-preview/route.ts +++ b/apps/sim/app/api/link-preview/route.ts @@ -53,6 +53,7 @@ function parsePreview(html: string): LinkPreview { async function fetchPreview(url: string): Promise { const response = await secureFetchWithValidation(url, { + profile: 'requestTarget', timeout: FETCH_TIMEOUT_MS, maxRedirects: MAX_REDIRECTS, maxResponseBytes: MAX_RESPONSE_BYTES, diff --git a/apps/sim/app/api/tools/zoho_desk/agents/route.ts b/apps/sim/app/api/tools/zoho_desk/agents/route.ts index 40aef00acad..1d79c04f158 100644 --- a/apps/sim/app/api/tools/zoho_desk/agents/route.ts +++ b/apps/sim/app/api/tools/zoho_desk/agents/route.ts @@ -94,6 +94,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { // the resolved IP, block private/reserved hops, and drop the token if a // Zoho-side redirect leaves the original origin. const response = await secureFetchWithValidation(agentsUrl.toString(), { + profile: 'configuredEndpoint', method: 'GET', headers, timeout: 15_000, diff --git a/apps/sim/app/api/tools/zoho_desk/departments/route.ts b/apps/sim/app/api/tools/zoho_desk/departments/route.ts index 3edb135d314..cc2a5c64777 100644 --- a/apps/sim/app/api/tools/zoho_desk/departments/route.ts +++ b/apps/sim/app/api/tools/zoho_desk/departments/route.ts @@ -78,6 +78,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { // IP, block private/reserved hops, and drop the token if a Zoho-side // redirect leaves the original origin. const response = await secureFetchWithValidation(departmentsUrl.toString(), { + profile: 'configuredEndpoint', method: 'GET', headers, timeout: 15_000, diff --git a/apps/sim/app/api/tools/zoho_desk/organizations/route.ts b/apps/sim/app/api/tools/zoho_desk/organizations/route.ts index 907d7030d94..27dee5147e3 100644 --- a/apps/sim/app/api/tools/zoho_desk/organizations/route.ts +++ b/apps/sim/app/api/tools/zoho_desk/organizations/route.ts @@ -64,6 +64,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { // IP, blocks private/reserved targets on every hop, and drops the token if // a redirect leaves the original origin. const response = await secureFetchWithValidation(organizationsUrl.toString(), { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: `Zoho-oauthtoken ${accessToken}`, diff --git a/apps/sim/connectors/gitlab/gitlab.ts b/apps/sim/connectors/gitlab/gitlab.ts index e94c3a8d28f..694e0280863 100644 --- a/apps/sim/connectors/gitlab/gitlab.ts +++ b/apps/sim/connectors/gitlab/gitlab.ts @@ -134,6 +134,7 @@ function parseNextLink(linkHeader: string | null): string | undefined { */ async function fetchListing(url: string, accessToken: string): Promise { const response = await secureFetchWithRetry(url, { + profile: 'configuredEndpoint', method: 'GET', headers: authHeaders(accessToken), }) @@ -150,7 +151,11 @@ async function fetchListing(url: string, accessToken: string): Promise { return secureFetchWithRetry( `${apiBase}/projects/${encodedProject}`, - { method: 'GET', headers: authHeaders(accessToken) }, + { profile: 'configuredEndpoint', method: 'GET', headers: authHeaders(accessToken) }, retryOptions ) } @@ -816,6 +821,7 @@ export const gitlabConnector: ConnectorConfig = { logger.info('Listing GitLab wiki pages', { host, project: encodedProject }) const response = await secureFetchWithRetry(url, { + profile: 'configuredEndpoint', method: 'GET', headers: authHeaders(accessToken), }) @@ -965,6 +971,7 @@ export const gitlabConnector: ConnectorConfig = { const url = `${apiBase}/projects/${encodedProject}/wikis/${encodeURIComponent(slug)}?render_html=false` const response = await secureFetchWithRetry(url, { + profile: 'configuredEndpoint', method: 'GET', headers: authHeaders(accessToken), }) @@ -986,6 +993,7 @@ export const gitlabConnector: ConnectorConfig = { const url = `${apiBase}/projects/${encodedProject}/issues/${iid}` const response = await secureFetchWithRetry(url, { + profile: 'configuredEndpoint', method: 'GET', headers: authHeaders(accessToken), }) @@ -1013,6 +1021,7 @@ export const gitlabConnector: ConnectorConfig = { ) const url = `${apiBase}/projects/${encodedProject}/repository/files/${encodeURIComponent(path)}?ref=${encodeURIComponent(ref)}` const response = await secureFetchWithRetry(url, { + profile: 'configuredEndpoint', method: 'GET', headers: authHeaders(accessToken), }) @@ -1108,7 +1117,7 @@ export const gitlabConnector: ConnectorConfig = { if (userRef && activePhases(choice).includes('repo')) { const refResponse = await secureFetchWithRetry( `${apiBase}/projects/${encodedProject}/repository/commits/${encodeURIComponent(userRef)}`, - { method: 'GET', headers: authHeaders(accessToken) }, + { profile: 'configuredEndpoint', method: 'GET', headers: authHeaders(accessToken) }, VALIDATE_RETRY_OPTIONS ) if (refResponse.status === 404) { diff --git a/apps/sim/connectors/mintlify/mintlify.ts b/apps/sim/connectors/mintlify/mintlify.ts index 4b3aa3fe5d8..d8420ae57c3 100644 --- a/apps/sim/connectors/mintlify/mintlify.ts +++ b/apps/sim/connectors/mintlify/mintlify.ts @@ -111,7 +111,7 @@ function resolveSite(rawUrl: string | undefined): MintlifySite { url = `https://${url}` } - const validation = validateExternalUrl(url, 'siteUrl') + const validation = validateExternalUrl(url, 'siteUrl', 'configuredEndpoint') if (!validation.isValid) { throw new Error(validation.error || 'Invalid documentation site URL') } @@ -172,6 +172,7 @@ async function fetchSiteText( const response = await secureFetchWithRetry( url, { + profile: 'configuredEndpoint', method: 'GET', headers: siteHeaders(accessToken, accept), stripAuthOnRedirect: true, diff --git a/apps/sim/connectors/obsidian/obsidian.ts b/apps/sim/connectors/obsidian/obsidian.ts index 22f1167b8e3..6993a92e7b8 100644 --- a/apps/sim/connectors/obsidian/obsidian.ts +++ b/apps/sim/connectors/obsidian/obsidian.ts @@ -49,7 +49,7 @@ function resolveVaultEndpoint(rawUrl: string | undefined): string { if (url && !url.startsWith('https://') && !url.startsWith('http://')) { url = `https://${url}` } - const validation = validateExternalUrl(url, 'vaultUrl') + const validation = validateExternalUrl(url, 'vaultUrl', 'configuredEndpoint') if (!validation.isValid) { throw new Error(validation.error || 'Invalid vault URL') } @@ -112,6 +112,7 @@ async function listDirectory( const response = await secureFetchWithRetry( endpoint, { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, @@ -201,6 +202,7 @@ async function fetchNote( filePath: string ): Promise { const response = await secureFetchWithRetry(noteUrl(baseUrl, filePath), { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, @@ -385,6 +387,7 @@ export const obsidianConnector: ConnectorConfig = { const response = await secureFetchWithRetry( `${baseUrl}/`, { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: `Bearer ${accessToken}` }, stripAuthOnRedirect: true, diff --git a/apps/sim/connectors/s3/s3.ts b/apps/sim/connectors/s3/s3.ts index bb746b02fd1..351a553ace5 100644 --- a/apps/sim/connectors/s3/s3.ts +++ b/apps/sim/connectors/s3/s3.ts @@ -523,7 +523,7 @@ async function listObjectsPage( const response = await secureFetchWithRetry( url, - { method: 'GET', headers, stripAuthOnRedirect: true }, + { profile: 'configuredEndpoint', method: 'GET', headers, stripAuthOnRedirect: true }, retryOptions ) @@ -616,6 +616,7 @@ export const s3Connector: ConnectorConfig = { const url = buildUrl(ctx, encodedPath, '') const response = await secureFetchWithRetry(url, { + profile: 'configuredEndpoint', method: 'GET', headers, stripAuthOnRedirect: true, diff --git a/apps/sim/connectors/sentry/sentry.ts b/apps/sim/connectors/sentry/sentry.ts index 8f1069af1f0..bc2fdf95ff1 100644 --- a/apps/sim/connectors/sentry/sentry.ts +++ b/apps/sim/connectors/sentry/sentry.ts @@ -372,6 +372,7 @@ async function fetchLatestEvent( const url = `${apiBase}/organizations/${encodeURIComponent(organization)}/issues/${encodeURIComponent(issueId)}/events/latest/` const response = await secureFetchWithRetry(url, { + profile: 'configuredEndpoint', method: 'GET', headers: authHeaders(accessToken), }) @@ -456,6 +457,7 @@ export const sentryConnector: ConnectorConfig = { }) const response = await secureFetchWithRetry(url.toString(), { + profile: 'configuredEndpoint', method: 'GET', headers: authHeaders(accessToken), }) @@ -511,6 +513,7 @@ export const sentryConnector: ConnectorConfig = { const url = `${apiBase}/organizations/${encodeURIComponent(organization)}/issues/${encodeURIComponent(externalId)}/` const response = await secureFetchWithRetry(url, { + profile: 'configuredEndpoint', method: 'GET', headers: authHeaders(accessToken), }) @@ -570,10 +573,7 @@ export const sentryConnector: ConnectorConfig = { */ const projectResponse = await secureFetchWithRetry( `${apiBase}/projects/${encodeURIComponent(organization)}/${encodeURIComponent(project)}/`, - { - method: 'GET', - headers: authHeaders(accessToken), - }, + { profile: 'configuredEndpoint', method: 'GET', headers: authHeaders(accessToken) }, VALIDATE_RETRY_OPTIONS ) @@ -612,10 +612,7 @@ export const sentryConnector: ConnectorConfig = { const issuesResponse = await secureFetchWithRetry( issuesProbeUrl.toString(), - { - method: 'GET', - headers: authHeaders(accessToken), - }, + { profile: 'configuredEndpoint', method: 'GET', headers: authHeaders(accessToken) }, VALIDATE_RETRY_OPTIONS ) diff --git a/apps/sim/connectors/zendesk/zendesk.ts b/apps/sim/connectors/zendesk/zendesk.ts index b8381ca574a..84f17530c15 100644 --- a/apps/sim/connectors/zendesk/zendesk.ts +++ b/apps/sim/connectors/zendesk/zendesk.ts @@ -135,6 +135,7 @@ async function zendeskApiGet( const response = await secureFetchWithRetry( url, { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: `Basic ${Buffer.from(`${email}/token:${accessToken}`, 'utf8').toString('base64')}`, diff --git a/apps/sim/executor/handlers/api/api-handler.test.ts b/apps/sim/executor/handlers/api/api-handler.test.ts index e2eeca7f795..febf487014c 100644 --- a/apps/sim/executor/handlers/api/api-handler.test.ts +++ b/apps/sim/executor/handlers/api/api-handler.test.ts @@ -136,32 +136,24 @@ describe('ApiBlockHandler', () => { expect(result).toEqual(expectedOutput) }) - it('should throw error for invalid URL format (no protocol)', async () => { - const inputs = { url: 'example.com/api' } + it('leaves egress validation to the tool layer rather than re-resolving here', async () => { + // `executeToolRequest` validates the fully composed URL and connects to the + // address it pinned. Validating the pre-templating string here would resolve + // DNS a second time and check a URL that is not the one dialed. + await handler.execute(mockContext, mockBlock, { url: 'https://api.example.com/data' }) - mockValidateUrlWithDNS.mockResolvedValueOnce({ - isValid: false, - error: 'url must be a valid URL', - }) - - await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toThrow( - 'url must be a valid URL' - ) - expect(mockExecuteTool).not.toHaveBeenCalled() + expect(mockValidateUrlWithDNS).not.toHaveBeenCalled() + expect(mockExecuteTool).toHaveBeenCalled() }) - it('should throw error for generally invalid URL format', async () => { - const inputs = { url: 'htp:/invalid-url' } + it('strips quotes a block reference left around the URL', async () => { + await handler.execute(mockContext, mockBlock, { url: '"https://api.example.com/data"' }) - mockValidateUrlWithDNS.mockResolvedValueOnce({ - isValid: false, - error: 'url must use https:// protocol', - }) - - await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toThrow( - 'url must use https:// protocol' + expect(mockExecuteTool).toHaveBeenCalledWith( + 'http_request', + expect.objectContaining({ url: 'https://api.example.com/data' }), + expect.anything() ) - expect(mockExecuteTool).not.toHaveBeenCalled() }) it('should parse JSON string body correctly', async () => { diff --git a/apps/sim/executor/handlers/api/api-handler.ts b/apps/sim/executor/handlers/api/api-handler.ts index 8595ef00a4b..1db45b81246 100644 --- a/apps/sim/executor/handlers/api/api-handler.ts +++ b/apps/sim/executor/handlers/api/api-handler.ts @@ -1,5 +1,4 @@ import { createLogger } from '@sim/logger' -import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import { BlockType, HTTP } from '@/executor/constants' import type { BlockHandler, ExecutionContext } from '@/executor/types' import type { SerializedBlock } from '@/serializer/types' @@ -30,21 +29,20 @@ export class ApiBlockHandler implements BlockHandler { return { data: null, status: HTTP.STATUS.OK, headers: {} } } - if (tool.name?.includes('HTTP') && inputs.url) { - let urlToValidate = inputs.url - if (typeof urlToValidate === 'string') { - if ( - (urlToValidate.startsWith('"') && urlToValidate.endsWith('"')) || - (urlToValidate.startsWith("'") && urlToValidate.endsWith("'")) - ) { - urlToValidate = urlToValidate.slice(1, -1) - inputs.url = urlToValidate - } - } - - const urlValidation = await validateUrlWithDNS(urlToValidate, 'url') - if (!urlValidation.isValid) { - throw new Error(urlValidation.error) + // Templating can leave the URL wrapped in quotes the author typed around a + // block reference. Strip them before the tool layer composes the request. + // + // Egress is deliberately not validated here. `executeToolRequest` validates + // the fully composed URL and connects to the address it pinned; repeating the + // check on this pre-templating string would resolve DNS a second time, throw + // the pinned address away, and validate a URL that is not the one dialed. + if (tool.name?.includes('HTTP') && typeof inputs.url === 'string') { + const raw = inputs.url + if ( + (raw.startsWith('"') && raw.endsWith('"')) || + (raw.startsWith("'") && raw.endsWith("'")) + ) { + inputs.url = raw.slice(1, -1) } } diff --git a/apps/sim/lib/a2a/client.ts b/apps/sim/lib/a2a/client.ts index f4dea96d1fd..4f84e5c822a 100644 --- a/apps/sim/lib/a2a/client.ts +++ b/apps/sim/lib/a2a/client.ts @@ -133,6 +133,7 @@ function createPinnedFetch( : undefined) const res = await secureFetchWithPinnedIP(url, resolvedIP, { + profile: 'requestTarget', method, headers, body, @@ -196,7 +197,7 @@ export async function createA2AClient( apiKey?: string, options: { signal?: AbortSignal } = {} ): Promise { - const validation = await validateUrlWithDNS(agentUrl, 'agentUrl') + const validation = await validateUrlWithDNS(agentUrl, 'agentUrl', 'requestTarget') if (!validation.isValid || !validation.resolvedIP) { throw new Error(validation.error || 'Agent URL validation failed') } diff --git a/apps/sim/lib/api/contracts/data-drains.ts b/apps/sim/lib/api/contracts/data-drains.ts index c4cb63c640b..ace4ddeb017 100644 --- a/apps/sim/lib/api/contracts/data-drains.ts +++ b/apps/sim/lib/api/contracts/data-drains.ts @@ -127,7 +127,7 @@ const s3ConfigBodySchema = z.object({ .string() .url() .refine((v) => v.startsWith('https://'), { message: 'endpoint must use https://' }) - .refine((value) => validateExternalUrl(value, 'endpoint').isValid, { + .refine((value) => validateExternalUrl(value, 'endpoint', 'configuredEndpoint').isValid, { message: 'endpoint must be HTTPS and not point at a private, loopback, or metadata address', }) .optional(), @@ -304,7 +304,7 @@ const webhookConfigBodySchema = z.object({ .string() .url('url must be a valid URL') .max(2048, 'url must be at most 2048 characters') - .refine((value) => validateExternalUrl(value, 'url').isValid, { + .refine((value) => validateExternalUrl(value, 'url', 'configuredEndpoint').isValid, { message: 'url must be HTTPS and not point at a private, loopback, or metadata address', }), signatureHeader: z diff --git a/apps/sim/lib/copilot/tools/server/files/download-to-workspace-file.ts b/apps/sim/lib/copilot/tools/server/files/download-to-workspace-file.ts index 71118e7d390..5f949e3e978 100644 --- a/apps/sim/lib/copilot/tools/server/files/download-to-workspace-file.ts +++ b/apps/sim/lib/copilot/tools/server/files/download-to-workspace-file.ts @@ -159,6 +159,7 @@ export const downloadToWorkspaceFileServerTool: BaseServerTool< // secureFetchWithValidation handles: DNS resolution, private IP blocking (via ipaddr.js), // SSRF-safe redirect following, and streaming size enforcement const response = await secureFetchWithValidation(params.url, { + profile: 'contentFetch', maxResponseBytes: MAX_DOWNLOAD_BYTES, }) diff --git a/apps/sim/lib/core/config/env-flags.ts b/apps/sim/lib/core/config/env-flags.ts index f337d67cc82..cf1f28b25f6 100644 --- a/apps/sim/lib/core/config/env-flags.ts +++ b/apps/sim/lib/core/config/env-flags.ts @@ -134,27 +134,74 @@ if (isTruthy(env.DISABLE_AUTH)) { }) } +const legacyPrivateHostsAllowed = isTruthy(env.ALLOW_PRIVATE_DATABASE_HOSTS) + /** - * Whether database/connector tools may connect to private, reserved, or loopback - * hosts (e.g. Docker/K8s service names, localhost). Off by default: the SSRF guard - * in {@link validateDatabaseHost} blocks these so an untrusted user cannot pivot - * into the deployment's internal network. Self-hosted operators can opt in when - * their database lives on the same private network. Blocked on the hosted platform - * regardless of the env var, mirroring {@link isAuthDisabled}. + * Ranges the deprecated `ALLOW_PRIVATE_DATABASE_HOSTS` stood for: it was a + * blanket "reach anything private", so it maps to the private and loopback + * space in full. Appended to whatever the operator listed explicitly, which is + * why an unset flag contributes nothing. Cloud metadata stays unreachable + * regardless — that block is not lifted by any allowlist. */ -export const isPrivateDatabaseHostsAllowed = isTruthy(env.ALLOW_PRIVATE_DATABASE_HOSTS) && !isHosted +const LEGACY_PRIVATE_RANGES = + '10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,127.0.0.0/8,169.254.0.0/16,::1/128,fc00::/7,fe80::/10' + +function joinConfig(...parts: Array): string | undefined { + const joined = parts.filter((part) => part && part.trim().length > 0).join(',') + return joined.length > 0 ? joined : undefined +} + +/** + * Destinations on a private network that outbound requests may reach, as raw + * operator config. Empty on the hosted platform regardless of what is set, so a + * tenant can never pivot into Sim's own network — mirroring {@link isAuthDisabled}. + * + * Parsing and enforcement live in `@sim/security/egress`; these are passed + * through unparsed because `env-flags` is loaded by `next.config.ts` before the + * `@/` alias exists and must stay dependency-light. + */ +export const egressAllowedHosts = isHosted + ? undefined + : joinConfig(env.EGRESS_ALLOWED_HOSTS, legacyPrivateHostsAllowed ? 'localhost' : undefined) + +export const egressAllowedIpRanges = isHosted + ? undefined + : joinConfig( + env.EGRESS_ALLOWED_IP_RANGES, + legacyPrivateHostsAllowed ? LEGACY_PRIVATE_RANGES : undefined + ) + +if (legacyPrivateHostsAllowed) { + import('@sim/logger') + .then(({ createLogger }) => { + const logger = createLogger('EnvFlags') + if (isHosted) { + logger.error( + 'ALLOW_PRIVATE_DATABASE_HOSTS is set but ignored on hosted environment. Private, reserved, and loopback destinations remain blocked for security.' + ) + } else { + logger.warn( + 'ALLOW_PRIVATE_DATABASE_HOSTS is deprecated and opens the whole private address space. Replace it with EGRESS_ALLOWED_HOSTS / EGRESS_ALLOWED_IP_RANGES naming only the destinations you need.' + ) + } + }) + .catch(() => { + // Fallback during config compilation when logger is unavailable + }) +} -if (isTruthy(env.ALLOW_PRIVATE_DATABASE_HOSTS)) { +if (env.EGRESS_ALLOWED_HOSTS || env.EGRESS_ALLOWED_IP_RANGES) { import('@sim/logger') .then(({ createLogger }) => { const logger = createLogger('EnvFlags') if (isHosted) { logger.error( - 'ALLOW_PRIVATE_DATABASE_HOSTS is set but ignored on hosted environment. Private/reserved database hosts remain blocked for security.' + 'EGRESS_ALLOWED_HOSTS/EGRESS_ALLOWED_IP_RANGES are set but ignored on hosted environment. Private, reserved, and loopback destinations remain blocked for security.' ) } else { logger.warn( - 'ALLOW_PRIVATE_DATABASE_HOSTS is enabled. Database/connector tools may reach private, reserved, and loopback hosts. Only use this in trusted private networks.' + 'Private-network egress allowlist is configured. Outbound requests may reach the listed destinations. Only use this on a trusted private network.', + { hosts: env.EGRESS_ALLOWED_HOSTS, ipRanges: env.EGRESS_ALLOWED_IP_RANGES } ) } }) diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 74462eb6c72..ee6f4dfadec 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -119,7 +119,9 @@ export const env = createEnv({ DISABLE_REGISTRATION: z.boolean().optional(), // Flag to disable new user registration EMAIL_PASSWORD_SIGNUP_ENABLED: z.boolean().optional().default(true), // Enable email/password authentication (server-side enforcement) DISABLE_AUTH: z.boolean().optional(), // Bypass authentication entirely (self-hosted only, creates anonymous session) - ALLOW_PRIVATE_DATABASE_HOSTS: z.boolean().optional(), // Opt-in (self-hosted only): let database/connector tools reach private/reserved/loopback hosts (e.g. Docker/K8s service names). Loosens the SSRF boundary; ignored on the hosted platform. + ALLOW_PRIVATE_DATABASE_HOSTS: z.boolean().optional(), // Deprecated alias for the egress allowlist, kept so existing self-hosted deployments keep working. Equivalent to allowing every private, reserved, and loopback destination. Prefer EGRESS_ALLOWED_HOSTS / EGRESS_ALLOWED_IP_RANGES, which name specific destinations. + EGRESS_ALLOWED_HOSTS: z.string().optional(), // Comma-separated hostnames outbound requests may reach on a private network, leading wildcard allowed (e.g. "host.docker.internal,*.svc.cluster.local"). Self-hosted only; ignored on the hosted platform. Replaces ALLOW_PRIVATE_DATABASE_HOSTS. + EGRESS_ALLOWED_IP_RANGES: z.string().optional(), // Comma-separated CIDRs or IPs outbound requests may reach on a private network (e.g. "10.0.0.0/8,192.168.65.254/32"). Self-hosted only; never lifts the cloud-metadata block. ALLOWED_LOGIN_EMAILS: z.string().optional(), // Comma-separated list of allowed email addresses for login ALLOWED_LOGIN_DOMAINS: z.string().optional(), // Comma-separated list of allowed email domains for login BLOCKED_SIGNUP_DOMAINS: z.string().optional(), // Comma-separated list of email domains blocked from signing up (e.g., "gmail.com,yahoo.com") diff --git a/apps/sim/lib/core/security/egress/profiles.ts b/apps/sim/lib/core/security/egress/profiles.ts new file mode 100644 index 00000000000..2da4618714b --- /dev/null +++ b/apps/sim/lib/core/security/egress/profiles.ts @@ -0,0 +1,180 @@ +/** + * Egress profiles: the deployment's answer to "how far may this kind of request + * reach?". + * + * The taxonomy is **where the URL came from**, because provenance is what + * determines how much trust a destination has earned. A base URL an operator + * typed during setup is not the same input as a link harvested from a + * third-party API response, and treating them identically is what left Sim + * simultaneously too strict for the first and too loose for the last. + * + * This module is the only place deployment posture and operator configuration + * are read. Everything below it takes an {@link EgressPolicy} value. + */ + +import { + createEgressPolicy, + type EgressDecision, + type EgressPolicy, + type InsecureHttpPolicy, +} from '@sim/security/egress' +import { egressAllowedHosts, egressAllowedIpRanges, isHosted } from '@/lib/core/config/env-flags' + +/** + * Where the URL for an outbound request came from. + * + * - `configuredEndpoint` — a base or server URL entered during setup: a + * self-hosted vLLM or Jupyter instance, GitHub Enterprise, Grafana, + * ClickHouse, an MCP server, a data-drain destination, a connector's host. + * - `requestTarget` — supplied per run by the workflow author: the HTTP block's + * `url`, an A2A agent URL, an RSS feed, a Function block's `fetch`. + * - `contentFetch` — harvested from content, a third-party response, or model + * output: an image URL, a file imported by URL, a Slack `url_private`, an + * endpoint read out of an OIDC discovery document. + * - `databaseHost` — a datastore host for a database, cache, or mail connector. + * Configured like the first, but without its loopback carve-out: loopback is + * exactly where Sim's own database and Redis listen, so reaching them has to + * be named rather than assumed. + * - `proxy` — the egress proxy itself. Held to the strictest rule of all, + * because it is the component that decides where everything else may go: plain + * HTTP by protocol, but public destinations only, and no allowlist. + */ +export type EgressProfile = + | 'configuredEndpoint' + | 'requestTarget' + | 'contentFetch' + | 'databaseHost' + | 'proxy' + +interface ProfileSpec { + /** + * Whether this profile consults the operator's private-network allowlist. + * + * `contentFetch` never does, and that is the point of the taxonomy: it is the + * class where SSRF is actually exploited, so it must stay locked even on a + * deployment whose operator has allowlisted their entire internal range. + */ + readonly honorsAllowlist: boolean + /** When plain HTTP is acceptable for this provenance. */ + readonly insecureHttp: InsecureHttpPolicy + /** + * Whether loopback is reachable without being allowlisted. True off the hosted + * platform for the two profiles whose URLs someone deliberately configured — + * a single-tenant deployment pointing at its own `localhost` (Ollama, a local + * Jupyter, a sidecar) is the ordinary case. Never true for `contentFetch`, and + * never true on the hosted platform, where `localhost` is Sim's own process. + */ + readonly allowLoopback: boolean +} + +const PROFILE_SPECS: Record = { + configuredEndpoint: { + honorsAllowlist: true, + insecureHttp: 'whenVouched', + allowLoopback: !isHosted, + }, + requestTarget: { honorsAllowlist: true, insecureHttp: 'whenVouched', allowLoopback: !isHosted }, + contentFetch: { honorsAllowlist: false, insecureHttp: 'never', allowLoopback: false }, + databaseHost: { honorsAllowlist: true, insecureHttp: 'whenVouched', allowLoopback: false }, + proxy: { honorsAllowlist: false, insecureHttp: 'always', allowLoopback: false }, +} + +const SOURCE_NAMES = { + hosts: 'EGRESS_ALLOWED_HOSTS', + ranges: 'EGRESS_ALLOWED_IP_RANGES', +} as const + +function buildPolicies(hosts: string | undefined, ranges: string | undefined) { + return Object.fromEntries( + (Object.keys(PROFILE_SPECS) as EgressProfile[]).map((profile) => { + const spec = PROFILE_SPECS[profile] + return [ + profile, + createEgressPolicy({ + allowedHosts: spec.honorsAllowlist ? hosts : undefined, + allowedRanges: spec.honorsAllowlist ? ranges : undefined, + insecureHttp: spec.insecureHttp, + allowLoopback: spec.allowLoopback, + sourceNames: SOURCE_NAMES, + }), + ] + }) + ) as Record +} + +/** + * Policies are built eagerly so a malformed allowlist entry throws at startup + * rather than at whichever request first happens to touch it, and cached against + * the configuration they were built from so that changing it rebuilds rather + * than silently serving a stale policy. Caching on the value rather than "built + * once" is what keeps the allowlist reachable from a test without a module-level + * reset hook. + */ +let cache = { + hosts: egressAllowedHosts, + ranges: egressAllowedIpRanges, + policies: buildPolicies(egressAllowedHosts, egressAllowedIpRanges), +} + +/** + * The policy governing requests of the given provenance on this deployment. + * + * An unrecognized profile resolves to the strictest one rather than to + * `undefined`: the callers are on the request path, and a missing policy there + * should refuse the destination, not throw somewhere further down where the + * cause is no longer visible. + */ +export function resolveEgressPolicy(profile: EgressProfile): EgressPolicy { + if (cache.hosts !== egressAllowedHosts || cache.ranges !== egressAllowedIpRanges) { + cache = { + hosts: egressAllowedHosts, + ranges: egressAllowedIpRanges, + policies: buildPolicies(egressAllowedHosts, egressAllowedIpRanges), + } + } + return cache.policies[profile] ?? cache.policies.contentFetch +} + +/** True when this deployment has any private-network allowlist configured. */ +function hasAllowlist(): boolean { + return Boolean(egressAllowedHosts || egressAllowedIpRanges) +} + +/** + * Turns a refusal into a message the person who hit it can act on. + * + * The message this replaced said `url must use https:// protocol`, which was + * worse than unhelpful: it implied switching scheme would fix a destination that + * the address check was going to refuse anyway. Each reason here names the + * actual blocker and, where one exists, the remedy. + */ +export function describeEgressDenial( + decision: Extract, + paramName: string, + profile: EgressProfile +): string { + // Mirrors resolveEgressPolicy: an unrecognized profile is described with the + // strictest spec, so a bad profile can never advertise a remedy that does not + // apply to the policy that actually refused the request. + const spec = PROFILE_SPECS[profile] ?? PROFILE_SPECS.contentFetch + const remedy = spec.honorsAllowlist + ? hasAllowlist() + ? ` It is not covered by ${SOURCE_NAMES.hosts} or ${SOURCE_NAMES.ranges}.` + : ` Self-hosted deployments can permit specific destinations with ${SOURCE_NAMES.hosts} or ${SOURCE_NAMES.ranges}.` + : '' + + switch (decision.reason) { + case 'scheme-not-permitted': + return `${paramName} must use http:// or https:// (got ${decision.detail})` + case 'insecure-scheme': + return `${paramName} must use https:// to a public destination.${remedy}` + case 'port-denied': + return `${paramName} uses a blocked port (${decision.detail}).${remedy}` + case 'address-loopback': + return `${paramName} resolves to loopback (${decision.detail}), which inside a container is the container itself, not the host.${remedy}` + case 'address-blocked': + return `${paramName} resolves to a private or reserved address (${decision.detail}).${remedy}` + case 'address-metadata': + return `${paramName} resolves to a cloud metadata endpoint (${decision.detail}), which is never reachable and cannot be allowlisted.` + } +} diff --git a/apps/sim/lib/core/security/egress/validate.ts b/apps/sim/lib/core/security/egress/validate.ts new file mode 100644 index 00000000000..059b420ad78 --- /dev/null +++ b/apps/sim/lib/core/security/egress/validate.ts @@ -0,0 +1,154 @@ +/** + * The DNS-resolving egress gate. + * + * Resolution and classification are one step here on purpose. A hostname says + * nothing about where it points, so the only safe sequence is resolve, classify + * every address, then connect to an address that was actually classified — which + * is why this returns the address it approved. A caller that resolves again + * instead of pinning what it was handed reopens the DNS-rebinding window this + * exists to close. + */ + +import { createLogger } from '@sim/logger' +import { preferIpv4, resolveHostAddresses } from '@sim/security/dns' +import { + type EgressDecision, + evaluateAddress, + evaluateUrl, + isLiftableByVouching, + policyCanVouch, +} from '@sim/security/egress' +import { isIpLiteral, unwrapIpv6Brackets } from '@sim/security/ssrf' +import { toError } from '@sim/utils/errors' +import { + describeEgressDenial, + type EgressProfile, + resolveEgressPolicy, +} from '@/lib/core/security/egress/profiles' + +const logger = createLogger('Egress') + +export interface EgressValidationSuccess { + readonly isValid: true + /** The address that was classified, and therefore the only one safe to dial. */ + readonly resolvedIP: string + readonly originalHostname: string +} + +export interface EgressValidationFailure { + readonly isValid: false + readonly error: string +} + +export type EgressValidationResult = EgressValidationSuccess | EgressValidationFailure + +type EgressDenial = Extract + +function fail( + decision: EgressDenial, + url: string, + paramName: string, + profile: EgressProfile +): EgressValidationFailure { + logger.warn('Blocked outbound request', { + profile, + reason: decision.reason, + detail: decision.detail, + paramName, + }) + return { isValid: false, error: describeEgressDenial(decision, paramName, profile) } +} + +/** + * Validates a destination and returns the address to pin. + * + * `profile` is required rather than defaulted: a new call site must state where + * its URL came from, because that is the only input to the trust decision and + * guessing it wrong is silent in both directions. + */ +export async function validateEgressUrl( + url: string | null | undefined, + paramName: string, + profile: EgressProfile +): Promise { + if (!url || typeof url !== 'string') { + return { isValid: false, error: `${paramName} is required and must be a string` } + } + + let parsed: URL + try { + parsed = new URL(url) + } catch { + return { isValid: false, error: `${paramName} must be a valid URL` } + } + + const policy = resolveEgressPolicy(profile) + + const host = unwrapIpv6Brackets(parsed.hostname) + const isLiteral = isIpLiteral(host) + + const preflight = evaluateUrl(parsed, policy) + if (!preflight.allowed) { + // For a literal the pre-flight already had the address, so its verdict is + // final. For a hostname it judged the destination as unvouched; if this + // policy could still vouch for it, resolve and let the address rule. + const final = isLiteral || !policyCanVouch(policy) || !isLiftableByVouching(preflight.reason) + if (final) return fail(preflight, url, paramName, profile) + } + + // An IP literal was already classified by evaluateUrl; resolving it would only + // hand DNS a chance to answer with something else. + if (isLiteral) { + return { isValid: true, resolvedIP: host, originalHostname: parsed.hostname } + } + + let addresses: string[] + try { + addresses = (await resolveHostAddresses(host)).addresses + } catch (error) { + logger.warn('DNS lookup failed', { paramName, host, error: toError(error).message }) + return { isValid: false, error: `${paramName} hostname could not be resolved` } + } + + // Refused records are filtered rather than failing the whole host: pinning to a + // surviving permitted address is just as safe as refusing outright, and a + // split-horizon resolver that answers with a private record alongside the + // public one would otherwise be unusable. + let refusal: EgressDenial | undefined + const usable = addresses.filter((address) => { + const decision = evaluateAddress(parsed, address, policy) + if (decision.allowed) return true + refusal ??= decision + return false + }) + + if (usable.length === 0) { + return fail( + refusal ?? { allowed: false, reason: 'address-blocked', detail: 'no addresses resolved' }, + url, + paramName, + profile + ) + } + + return { + isValid: true, + // Re-preferred over the surviving set so the pin is never an address the + // filter above just refused. + resolvedIP: preferIpv4(usable as [string, ...string[]]), + originalHostname: parsed.hostname, + } +} + +/** + * Re-checks a destination that has already been resolved — a redirect hop whose + * target is an IP literal, or any point where the address is known and a fresh + * lookup would be the wrong thing to do. + */ +export function checkResolvedEgress( + url: URL, + address: string, + profile: EgressProfile +): EgressDecision { + return evaluateAddress(url, address, resolveEgressPolicy(profile)) +} diff --git a/apps/sim/lib/core/security/input-validation.server.test.ts b/apps/sim/lib/core/security/input-validation.server.test.ts index a8cffca7888..1475a0c6b6d 100644 --- a/apps/sim/lib/core/security/input-validation.server.test.ts +++ b/apps/sim/lib/core/security/input-validation.server.test.ts @@ -13,7 +13,8 @@ vi.mock('@sim/security/dns', () => ({ vi.mock('@/lib/core/config/env-flags', () => ({ isHosted: false, - isPrivateDatabaseHostsAllowed: false, + egressAllowedHosts: undefined, + egressAllowedIpRanges: undefined, getProxyUrl: () => undefined, })) @@ -39,7 +40,11 @@ describe('validateUrlWithDNS address classification', () => { // got judged was a matter of resolver order. mockResolve.mockResolvedValue(resolved(['93.184.216.34', '10.0.0.5'])) - const result = await validateUrlWithDNS('https://mixed.example/api') + const result = await validateUrlWithDNS( + 'https://mixed.example/api', + 'url', + 'configuredEndpoint' + ) expect(result.isValid).toBe(true) expect(result.resolvedIP).toBe('93.184.216.34') @@ -48,10 +53,17 @@ describe('validateUrlWithDNS address classification', () => { it('rejects when every record is private', async () => { mockResolve.mockResolvedValue(resolved(['10.0.0.5', '192.168.1.9'])) - const result = await validateUrlWithDNS('https://internal.example/api') + const result = await validateUrlWithDNS( + 'https://internal.example/api', + 'url', + 'configuredEndpoint' + ) expect(result.isValid).toBe(false) - expect(result.error).toContain('blocked IP address') + // The message now names the offending address and the setting that would + // permit it, instead of the old undifferentiated 'blocked IP address'. + expect(result.error).toContain('private or reserved address (10.0.0.5)') + expect(result.error).toContain('EGRESS_ALLOWED_IP_RANGES') }) it('never pins an address the filter refused', async () => { @@ -59,7 +71,11 @@ describe('validateUrlWithDNS address classification', () => { // the unfiltered set would land on 10.0.0.5. mockResolve.mockResolvedValue(resolved(['10.0.0.5', '2606:2800:220:1::248'])) - const result = await validateUrlWithDNS('https://mixed.example/api') + const result = await validateUrlWithDNS( + 'https://mixed.example/api', + 'url', + 'configuredEndpoint' + ) expect(result.isValid).toBe(true) expect(result.resolvedIP).toBe('2606:2800:220:1::248') @@ -68,7 +84,7 @@ describe('validateUrlWithDNS address classification', () => { it('accepts a host whose every record is public, pinning the preferred one', async () => { mockResolve.mockResolvedValue(resolved(['93.184.216.34', '93.184.216.35'])) - const result = await validateUrlWithDNS('https://example.com/api') + const result = await validateUrlWithDNS('https://example.com/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(true) expect(result.resolvedIP).toBe('93.184.216.34') @@ -77,7 +93,9 @@ describe('validateUrlWithDNS address classification', () => { it('keeps the self-hosted localhost carve-out when every record is loopback', async () => { mockResolve.mockResolvedValue(resolved(['127.0.0.1', '::1'])) - expect((await validateUrlWithDNS('https://localhost/api')).isValid).toBe(true) + expect( + (await validateUrlWithDNS('https://localhost/api', 'url', 'configuredEndpoint')).isValid + ).toBe(true) }) it('drops an off-loopback record from localhost rather than pinning it', async () => { @@ -85,7 +103,7 @@ describe('validateUrlWithDNS address classification', () => { // the pin stays on the machine the carve-out was written for. mockResolve.mockResolvedValue(resolved(['127.0.0.1', '10.0.0.5'])) - const result = await validateUrlWithDNS('https://localhost/api') + const result = await validateUrlWithDNS('https://localhost/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(true) expect(result.resolvedIP).toBe('127.0.0.1') @@ -94,6 +112,8 @@ describe('validateUrlWithDNS address classification', () => { it('reports an unresolvable host rather than treating it as public', async () => { mockResolve.mockRejectedValue(new Error('ENOTFOUND')) - expect((await validateUrlWithDNS('https://missing.example/api')).isValid).toBe(false) + expect( + (await validateUrlWithDNS('https://missing.example/api', 'url', 'configuredEndpoint')).isValid + ).toBe(false) }) }) diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index 7a2046fdb5a..c44d31aa5bf 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -6,7 +6,7 @@ import https from 'https' import type { LookupFunction } from 'net' import { createLogger } from '@sim/logger' import { preferIpv4, resolveHostAddresses } from '@sim/security/dns' -import { isLoopbackIp, isPrivateIp, isPrivateIpHost, unwrapIpv6Brackets } from '@sim/security/ssrf' +import { isIpLiteral, isPrivateIp, unwrapIpv6Brackets } from '@sim/security/ssrf' import { toError } from '@sim/utils/errors' import { HttpProxyAgent } from 'http-proxy-agent' import { HttpsProxyAgent } from 'https-proxy-agent' @@ -17,9 +17,10 @@ import { type RequestInit as UndiciRequestInit, request as undiciRequest, } from 'undici' -import { isHosted, isPrivateDatabaseHostsAllowed } from '@/lib/core/config/env-flags' +import { describeEgressDenial, type EgressProfile } from '@/lib/core/security/egress/profiles' +import { checkResolvedEgress, validateEgressUrl } from '@/lib/core/security/egress/validate' import type { HttpRedirectPolicy } from '@/lib/core/security/http-redirect-policy' -import { type ValidationResult, validateExternalUrl } from '@/lib/core/security/input-validation' +import type { ValidationResult } from '@/lib/core/security/input-validation' import { nodeReadableToWebStream } from '@/lib/core/utils/node-stream' import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' @@ -34,78 +35,27 @@ export interface AsyncValidationResult extends ValidationResult { } /** - * Validates a URL and resolves its DNS to prevent SSRF via DNS rebinding + * Validates a URL, resolves its DNS, and returns the address to pin. * - * This function: - * 1. Performs basic URL validation (protocol, format) - * 2. Resolves the hostname to an IP address - * 3. Validates the resolved IP is not private/reserved - * 4. Returns the resolved IP for use in the actual request + * `profile` states where the URL came from — see {@link EgressProfile}. It is + * required because provenance is the only input to the trust decision, and a + * wrong guess is silent in both directions: too strict breaks a self-hosted + * integration, too loose hands an attacker the internal network. * * @param url - The URL to validate * @param paramName - Name of the parameter for error messages + * @param profile - Where this URL came from * @returns AsyncValidationResult with resolved IP for DNS pinning */ export async function validateUrlWithDNS( url: string | null | undefined, - paramName = 'url', - options: { allowHttp?: boolean } = {} + paramName: string, + profile: EgressProfile ): Promise { - const basicValidation = validateExternalUrl(url, paramName, options) - if (!basicValidation.isValid) { - return basicValidation - } - - const parsedUrl = new URL(url!) - const hostname = parsedUrl.hostname - - const hostnameLower = hostname.toLowerCase() - const cleanHostname = unwrapIpv6Brackets(hostnameLower) - - // Whole loopback range — see the matching note in input-validation.ts. - const isLocalhost = cleanHostname === 'localhost' || isLoopbackIp(cleanHostname) - - try { - // Refused records are filtered rather than failing the whole host, matching - // createSsrfGuardedLookup below. Pinning to a surviving public address is - // just as safe as refusing outright, and rejecting the host would break a - // split-horizon resolver that answers with a private record alongside the - // public one — with no operator opt-out on this path. - const { addresses } = await resolveHostAddresses(cleanHostname) - const usable = addresses.filter( - (address) => !isPrivateIp(address) || (isLocalhost && !isHosted && isLoopbackIp(address)) - ) - - if (usable.length === 0) { - logger.warn('URL resolves to blocked IP address', { - paramName, - hostname, - resolvedIP: addresses.find((address) => isPrivateIp(address)), - }) - return { - isValid: false, - error: `${paramName} resolves to a blocked IP address`, - } - } - - return { - isValid: true, - // Re-preferred over the surviving set so the pin is never an address the - // filter above just refused. - resolvedIP: preferIpv4(usable as [string, ...string[]]), - originalHostname: hostname, - } - } catch (error) { - logger.warn('DNS lookup failed for URL', { - paramName, - hostname, - error: toError(error).message, - }) - return { - isValid: false, - error: `${paramName} hostname could not be resolved`, - } - } + const result = await validateEgressUrl(url, paramName, profile) + return result.isValid + ? { isValid: true, resolvedIP: result.resolvedIP, originalHostname: result.originalHostname } + : { isValid: false, error: result.error } } /** @@ -154,19 +104,17 @@ export async function validateAndPinProxyUrl( } } - const validation = await validateUrlWithDNS(proxyUrl, 'proxyUrl', { allowHttp: true }) + // The `proxy` profile is what holds a proxy to a stricter rule than the + // destinations it fronts: plain HTTP by protocol, but public addresses only, + // and no operator allowlist — a private proxy host stays blocked even on a + // deployment that has allowlisted that range for everything else. + const validation = await validateUrlWithDNS(proxyUrl, 'proxyUrl', 'proxy') if (!validation.isValid) { return { isValid: false, error: validation.error } } const resolvedIP = validation.resolvedIP! - // validateUrlWithDNS permits loopback for self-hosted dev targets; a proxy governs - // egress, so loopback/private proxy hosts stay blocked unconditionally. - if (isPrivateIp(resolvedIP)) { - return { isValid: false, error: 'proxyUrl resolves to a blocked IP address' } - } - // Bracket IPv6 literals: assigning an unbracketed IPv6 address to URL.hostname // is a no-op, which would leave the DNS hostname in place and reopen rebinding. parsed.hostname = resolvedIP.includes(':') ? `[${resolvedIP}]` : resolvedIP @@ -182,11 +130,16 @@ export async function validateAndPinProxyUrl( * database hostnames (e.g. underscores in Docker/K8s service names). It only * blocks localhost and private/reserved IPs. * - * Self-hosted operators can set `ALLOW_PRIVATE_DATABASE_HOSTS` to reach databases - * on their private network (e.g. a Docker/Swarm service name that resolves to an - * internal IP). The opt-in only bypasses the private/reserved/loopback block; DNS - * is still resolved so the caller can pin the connection to the resolved IP. The - * bypass is never honored on the hosted platform (see {@link isPrivateDatabaseHostsAllowed}). + * Self-hosted operators reach a database on their private network (e.g. a + * Docker/Swarm service name that resolves to an internal IP) by naming it in the + * shared egress allowlist — the same one that governs HTTP destinations, because + * "may this deployment talk to that host" is one question, not one per protocol. + * DNS is still resolved so the caller can pin the connection to the resolved IP, + * and the allowlist is never honored on the hosted platform. + * + * A database host carries no scheme or port of its own, so it is evaluated as an + * `https` destination: the address rules and the allowlist apply, the HTTP-only + * scheme and port rules do not. * * @param host - The database hostname to validate * @param paramName - Name of the parameter for error messages @@ -202,35 +155,47 @@ export async function validateDatabaseHost( const cleanHost = unwrapIpv6Brackets(host.toLowerCase()) - if (cleanHost === 'localhost' && !isPrivateDatabaseHostsAllowed) { - return { isValid: false, error: `${paramName} cannot be localhost` } + let asUrl: URL + try { + asUrl = new URL( + `https://${isIpLiteral(cleanHost) && cleanHost.includes(':') ? `[${cleanHost}]` : cleanHost}` + ) + } catch { + return { isValid: false, error: `${paramName} is not a valid host` } } - if (isPrivateIpHost(cleanHost) && !isPrivateDatabaseHostsAllowed) { - return { isValid: false, error: `${paramName} cannot be a private IP address` } + if (isIpLiteral(cleanHost)) { + const decision = checkResolvedEgress(asUrl, cleanHost, 'databaseHost') + if (!decision.allowed) { + return { isValid: false, error: describeEgressDenial(decision, paramName, 'databaseHost') } + } + return { isValid: true, resolvedIP: cleanHost, originalHostname: host } } try { - const { addresses, preferred } = await resolveHostAddresses(cleanHost) - const blockedAddress = isPrivateDatabaseHostsAllowed - ? undefined - : addresses.find((candidate) => isPrivateIp(candidate)) + const { addresses } = await resolveHostAddresses(cleanHost) + const blocked = addresses.find( + (candidate) => !checkResolvedEgress(asUrl, candidate, 'databaseHost').allowed + ) - if (blockedAddress !== undefined) { + if (blocked !== undefined) { + const decision = checkResolvedEgress(asUrl, blocked, 'databaseHost') logger.warn('Database host resolves to blocked IP address', { paramName, hostname: host, - resolvedIP: blockedAddress, + resolvedIP: blocked, }) return { isValid: false, - error: `${paramName} resolves to a blocked IP address`, + error: decision.allowed + ? `${paramName} resolves to a blocked IP address` + : describeEgressDenial(decision, paramName, 'databaseHost'), } } return { isValid: true, - resolvedIP: preferred, + resolvedIP: preferIpv4(addresses as [string, ...string[]]), originalHostname: host, } } catch (error) { @@ -380,6 +345,13 @@ export interface SecureFetchOptions { * bypassed (the proxy resolves the target). */ proxyUrl?: string + /** + * Where this request's URL came from. Carried on the options so the same + * policy is re-applied to every redirect hop rather than re-derived — a hop + * evaluated under a laxer policy than the origin is how a redirect chain + * escapes the guard it started under. + */ + profile: EgressProfile } export class SecureFetchHeaders { @@ -1012,7 +984,7 @@ export function createPinnedFetchWithDispatcher( export async function secureFetchWithPinnedIP( url: string, resolvedIP: string, - options: SecureFetchOptions & { allowHttp?: boolean } = {}, + options: SecureFetchOptions, redirectCount = 0 ): Promise { const maxRedirects = options.maxRedirects ?? DEFAULT_MAX_REDIRECTS @@ -1067,7 +1039,7 @@ export async function secureFetchWithPinnedIP( settledReject(error) return } - validateUrlWithDNS(redirectUrl, 'redirectUrl', { allowHttp: options.allowHttp }) + validateUrlWithDNS(redirectUrl, 'redirectUrl', options.profile) .then((validation) => { if (!validation.isValid) { settledReject(new Error(`Redirect blocked: ${validation.error}`)) @@ -1102,7 +1074,7 @@ export async function secureFetchWithPinnedIP( if (redirectHeaders && options.stripAuthOnRedirect) { redirectHeaders = stripHeaders(redirectHeaders, ['authorization']) } - const redirectOptions: SecureFetchOptions & { allowHttp?: boolean } = { + const redirectOptions: SecureFetchOptions = { ...options, method: hop.method, body: hop.dropBody ? undefined : options.body, @@ -1302,19 +1274,17 @@ export async function secureFetchWithPinnedIP( * Combines validateUrlWithDNS and secureFetchWithPinnedIP for convenience. * * @param url - The URL to fetch - * @param options - Fetch options (method, headers, body, etc.) + * @param options - Fetch options, including the required egress `profile` * @param paramName - Name of the parameter for error messages (default: 'url') * @returns SecureFetchResponse * @throws Error if URL validation fails */ export async function secureFetchWithValidation( url: string, - options: SecureFetchOptions & { allowHttp?: boolean } = {}, + options: SecureFetchOptions, paramName = 'url' ): Promise { - const validation = await validateUrlWithDNS(url, paramName, { - allowHttp: options.allowHttp, - }) + const validation = await validateUrlWithDNS(url, paramName, options.profile) if (!validation.isValid) { throw new Error(validation.error) } diff --git a/apps/sim/lib/core/security/input-validation.test.ts b/apps/sim/lib/core/security/input-validation.test.ts index 3d548cfdec1..3fd24267239 100644 --- a/apps/sim/lib/core/security/input-validation.test.ts +++ b/apps/sim/lib/core/security/input-validation.test.ts @@ -7,25 +7,16 @@ import { validateCallbackUrl, validateEnum, validateExternalUrl, - validateFileExtension, - validateGoogleCalendarId, validateGoogleCloudLocation, validateGoogleCloudProject, - validateHostname, - validateImageUrl, - validateInteger, validateJiraCloudId, validateJiraIssueKey, validateMicrosoftGraphId, - validateMondayColumnId, - validateMondayGroupId, validateMondayNumericId, validateNumericId, validatePathSegment, - validateProxyUrl, validateS3BucketName, validateServiceNowInstanceUrl, - validateSupabaseProjectId, validateWorkdayTenantUrl, } from '@/lib/core/security/input-validation' import { @@ -386,141 +377,6 @@ describe('validateEnum', () => { }) }) -describe('validateHostname', () => { - describe('valid hostnames', () => { - it.concurrent('should accept valid domain names', () => { - const result = validateHostname('example.com') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept subdomains', () => { - const result = validateHostname('api.example.com') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept domains with hyphens', () => { - const result = validateHostname('my-domain.com') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept multi-level domains', () => { - const result = validateHostname('api.v2.example.co.uk') - expect(result.isValid).toBe(true) - }) - }) - - describe('invalid hostnames - private IPs', () => { - it.concurrent('should reject localhost', () => { - const result = validateHostname('localhost') - expect(result.isValid).toBe(false) - expect(result.error).toContain('private IP') - }) - - it.concurrent('should reject 127.0.0.1', () => { - const result = validateHostname('127.0.0.1') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject 10.x.x.x private range', () => { - const result = validateHostname('10.0.0.1') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject 192.168.x.x private range', () => { - const result = validateHostname('192.168.1.1') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject 172.16-31.x.x private range', () => { - const result = validateHostname('172.16.0.1') - expect(result.isValid).toBe(false) - const result2 = validateHostname('172.31.255.255') - expect(result2.isValid).toBe(false) - }) - - it.concurrent('should reject link-local addresses', () => { - const result = validateHostname('169.254.169.254') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject IPv6 loopback', () => { - const result = validateHostname('::1') - expect(result.isValid).toBe(false) - }) - }) - - describe('invalid hostnames - format', () => { - it.concurrent('should reject invalid characters', () => { - const result = validateHostname('example_domain.com') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject hostnames starting with hyphen', () => { - const result = validateHostname('-example.com') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject hostnames ending with hyphen', () => { - const result = validateHostname('example-.com') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject empty string', () => { - const result = validateHostname('') - expect(result.isValid).toBe(false) - }) - }) -}) - -describe('validateFileExtension', () => { - const allowedExtensions = ['jpg', 'png', 'gif', 'pdf'] as const - - describe('valid extensions', () => { - it.concurrent('should accept allowed extensions', () => { - const result = validateFileExtension('jpg', allowedExtensions) - expect(result.isValid).toBe(true) - expect(result.sanitized).toBe('jpg') - }) - - it.concurrent('should accept extensions with leading dot', () => { - const result = validateFileExtension('.png', allowedExtensions) - expect(result.isValid).toBe(true) - expect(result.sanitized).toBe('png') - }) - - it.concurrent('should normalize to lowercase', () => { - const result = validateFileExtension('JPG', allowedExtensions) - expect(result.isValid).toBe(true) - expect(result.sanitized).toBe('jpg') - }) - - it.concurrent('should accept all allowed extensions', () => { - for (const ext of allowedExtensions) { - const result = validateFileExtension(ext, allowedExtensions) - expect(result.isValid).toBe(true) - } - }) - }) - - describe('invalid extensions', () => { - it.concurrent('should reject extensions not in allowed list', () => { - const result = validateFileExtension('exe', allowedExtensions) - expect(result.isValid).toBe(false) - expect(result.error).toContain('jpg, png, gif, pdf') - }) - - it.concurrent('should reject empty string', () => { - const result = validateFileExtension('', allowedExtensions) - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject null', () => { - const result = validateFileExtension(null, allowedExtensions) - expect(result.isValid).toBe(false) - }) - }) -}) - describe('sanitizeForLogging', () => { it.concurrent('should truncate long strings', () => { const longString = 'a'.repeat(200) @@ -571,54 +427,98 @@ describe('sanitizeForLogging', () => { describe('validateUrlWithDNS', () => { describe('basic validation', () => { it('should reject invalid URLs', async () => { - const result = await validateUrlWithDNS('not-a-url') + const result = await validateUrlWithDNS('not-a-url', 'url', 'configuredEndpoint') expect(result.isValid).toBe(false) expect(result.error).toContain('valid URL') }) - it('should reject http:// URLs', async () => { - const result = await validateUrlWithDNS('http://example.com') + it('should reject http:// URLs to a public host', async () => { + const result = await validateUrlWithDNS('http://example.com', 'url', 'configuredEndpoint') expect(result.isValid).toBe(false) expect(result.error).toContain('https://') }) it('should accept https localhost URLs (self-hosted)', async () => { - const result = await validateUrlWithDNS('https://localhost/api') + const result = await validateUrlWithDNS('https://localhost/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(true) expect(result.resolvedIP).toBeDefined() }) it('should accept http localhost URLs (self-hosted)', async () => { - const result = await validateUrlWithDNS('http://localhost/api') + const result = await validateUrlWithDNS('http://localhost/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(true) expect(result.resolvedIP).toBeDefined() }) it('should accept IPv4 loopback URLs (self-hosted)', async () => { - const result = await validateUrlWithDNS('http://127.0.0.1/api') + const result = await validateUrlWithDNS('http://127.0.0.1/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(true) expect(result.resolvedIP).toBeDefined() }) it('should accept IPv6 loopback URLs (self-hosted)', async () => { - const result = await validateUrlWithDNS('http://[::1]/api') + const result = await validateUrlWithDNS('http://[::1]/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(true) expect(result.resolvedIP).toBeDefined() }) it('should reject private IP URLs', async () => { - const result = await validateUrlWithDNS('https://192.168.1.1/api') - expect(result.isValid).toBe(false) - expect(result.error).toContain('private IP') + const result = await validateUrlWithDNS( + 'https://192.168.1.1/api', + 'url', + 'configuredEndpoint' + ) + expect(result.isValid).toBe(false) + expect(result.error).toContain('private or reserved address') + }) + + it('permits a private IP once the operator allowlists its range', async () => { + envFlagsMock.egressAllowedIpRanges = '192.168.0.0/16' + try { + const result = await validateUrlWithDNS( + 'http://192.168.1.1/api', + 'url', + 'configuredEndpoint' + ) + expect(result.isValid).toBe(true) + expect(result.resolvedIP).toBe('192.168.1.1') + } finally { + envFlagsMock.egressAllowedIpRanges = undefined + } + }) + + it('never lets an allowlisted range reach a content-provenance URL', async () => { + envFlagsMock.egressAllowedIpRanges = '192.168.0.0/16' + try { + const result = await validateUrlWithDNS('https://192.168.1.1/api', 'url', 'contentFetch') + expect(result.isValid).toBe(false) + } finally { + envFlagsMock.egressAllowedIpRanges = undefined + } + }) + + it('never lets an allowlisted range reach cloud metadata', async () => { + envFlagsMock.egressAllowedIpRanges = '169.254.0.0/16' + try { + const result = await validateUrlWithDNS( + 'http://169.254.169.254/latest/meta-data/', + 'url', + 'configuredEndpoint' + ) + expect(result.isValid).toBe(false) + expect(result.error).toContain('metadata') + } finally { + envFlagsMock.egressAllowedIpRanges = undefined + } }) it('should reject null', async () => { - const result = await validateUrlWithDNS(null) + const result = await validateUrlWithDNS(null, 'url', 'configuredEndpoint') expect(result.isValid).toBe(false) }) it('should reject empty string', async () => { - const result = await validateUrlWithDNS('') + const result = await validateUrlWithDNS('', 'url', 'configuredEndpoint') expect(result.isValid).toBe(false) }) }) @@ -626,7 +526,8 @@ describe('validateUrlWithDNS', () => { describe('validateDatabaseHost', () => { afterEach(() => { - envFlagsMock.isPrivateDatabaseHostsAllowed = false + envFlagsMock.egressAllowedHosts = undefined + envFlagsMock.egressAllowedIpRanges = undefined }) describe('default (SSRF guard on)', () => { @@ -639,25 +540,26 @@ describe('validateDatabaseHost', () => { it('rejects localhost', async () => { const result = await validateDatabaseHost('localhost') expect(result.isValid).toBe(false) - expect(result.error).toContain('localhost') + expect(result.error).toContain('loopback') }) it('rejects a literal private IP', async () => { const result = await validateDatabaseHost('10.0.0.5') expect(result.isValid).toBe(false) - expect(result.error).toContain('private IP') + expect(result.error).toContain('private or reserved address') }) it('rejects a literal loopback IP', async () => { const result = await validateDatabaseHost('127.0.0.1') expect(result.isValid).toBe(false) - expect(result.error).toContain('private IP') + // A database host gets no loopback carve-out: Sim's own datastore is there. + expect(result.error).toContain('loopback') }) it('rejects a bracketed IPv6 loopback as a private IP (not unresolvable)', async () => { const result = await validateDatabaseHost('[::1]') expect(result.isValid).toBe(false) - expect(result.error).toContain('private IP') + expect(result.error).toContain('loopback') }) it('accepts a public IP and pins the resolved address', async () => { @@ -667,9 +569,36 @@ describe('validateDatabaseHost', () => { }) }) - describe('self-host opt-in (ALLOW_PRIVATE_DATABASE_HOSTS)', () => { + describe('deprecated ALLOW_PRIVATE_DATABASE_HOSTS alias', () => { + afterEach(() => { + envFlagsMock.egressAllowedHosts = undefined + envFlagsMock.egressAllowedIpRanges = undefined + }) + + it('keeps working for a deployment that still sets only the old flag', async () => { + // env-flags expands the flag into the full private space, so the alias + // reproduces the behavior those deployments have today. + envFlagsMock.egressAllowedHosts = 'localhost' + envFlagsMock.egressAllowedIpRanges = + '10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,127.0.0.0/8,169.254.0.0/16,::1/128,fc00::/7,fe80::/10' + + expect((await validateDatabaseHost('localhost')).isValid).toBe(true) + expect((await validateDatabaseHost('10.0.0.5')).isValid).toBe(true) + expect((await validateDatabaseHost('127.0.0.1')).isValid).toBe(true) + }) + + it('still cannot reach cloud metadata through the alias', async () => { + envFlagsMock.egressAllowedIpRanges = '169.254.0.0/16' + const result = await validateDatabaseHost('169.254.169.254') + expect(result.isValid).toBe(false) + expect(result.error).toContain('cloud metadata endpoint') + }) + }) + + describe('self-host opt-in (EGRESS_ALLOWED_HOSTS / EGRESS_ALLOWED_IP_RANGES)', () => { beforeEach(() => { - envFlagsMock.isPrivateDatabaseHostsAllowed = true + envFlagsMock.egressAllowedHosts = 'localhost' + envFlagsMock.egressAllowedIpRanges = '10.0.0.0/8,127.0.0.0/8,::1/128' }) it('allows localhost and still resolves an IP to pin', async () => { @@ -731,22 +660,35 @@ describe('validateAndPinProxyUrl', () => { it('should reject a proxy host that is a private IP', async () => { const result = await validateAndPinProxyUrl('http://user:pass@192.168.1.1:8080') expect(result.isValid).toBe(false) - expect(result.error).toMatch(/private IP|blocked IP/) + expect(result.error).toContain('private or reserved address') + // The proxy profile honors no allowlist, so the message must not offer one + // as a remedy — there is nothing the operator could set to permit this. + expect(result.error).not.toContain('EGRESS_ALLOWED') }) it('should reject a loopback proxy host even off the hosted platform', async () => { const localhost = await validateAndPinProxyUrl('http://localhost:3128') expect(localhost.isValid).toBe(false) - expect(localhost.error).toContain('blocked IP') + expect(localhost.error).toContain('loopback') const loopback = await validateAndPinProxyUrl('http://127.0.0.1:3128') expect(loopback.isValid).toBe(false) - expect(loopback.error).toContain('blocked IP') + expect(loopback.error).toContain('loopback') }) it('should reject a proxy host that is the metadata IP', async () => { const result = await validateAndPinProxyUrl('http://169.254.169.254:80') expect(result.isValid).toBe(false) - expect(result.error).toMatch(/private IP|blocked IP/) + expect(result.error).toContain('metadata') + }) + + it('rejects a private proxy even when the operator allowlists its range', async () => { + envFlagsMock.egressAllowedIpRanges = '192.168.0.0/16' + try { + const result = await validateAndPinProxyUrl('http://192.168.1.1:8080') + expect(result.isValid).toBe(false) + } finally { + envFlagsMock.egressAllowedIpRanges = undefined + } }) it('should accept a public proxy host and pin the hostname to the resolved IP, preserving creds/port', async () => { @@ -771,98 +713,6 @@ describe('validateAndPinProxyUrl', () => { }) }) -describe('validateInteger', () => { - describe('valid integers', () => { - it.concurrent('should accept positive integers', () => { - const result = validateInteger(42, 'count') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept zero', () => { - const result = validateInteger(0, 'count') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept negative integers', () => { - const result = validateInteger(-10, 'offset') - expect(result.isValid).toBe(true) - }) - }) - - describe('invalid integers', () => { - it.concurrent('should reject null', () => { - const result = validateInteger(null, 'value') - expect(result.isValid).toBe(false) - expect(result.error).toContain('required') - }) - - it.concurrent('should reject undefined', () => { - const result = validateInteger(undefined, 'value') - expect(result.isValid).toBe(false) - expect(result.error).toContain('required') - }) - - it.concurrent('should reject strings', () => { - const result = validateInteger('42' as any, 'value') - expect(result.isValid).toBe(false) - expect(result.error).toContain('must be a number') - }) - - it.concurrent('should reject floating point numbers', () => { - const result = validateInteger(3.14, 'value') - expect(result.isValid).toBe(false) - expect(result.error).toContain('must be an integer') - }) - - it.concurrent('should reject NaN', () => { - const result = validateInteger(Number.NaN, 'value') - expect(result.isValid).toBe(false) - expect(result.error).toContain('valid number') - }) - - it.concurrent('should reject Infinity', () => { - const result = validateInteger(Number.POSITIVE_INFINITY, 'value') - expect(result.isValid).toBe(false) - expect(result.error).toContain('valid number') - }) - - it.concurrent('should reject negative Infinity', () => { - const result = validateInteger(Number.NEGATIVE_INFINITY, 'value') - expect(result.isValid).toBe(false) - expect(result.error).toContain('valid number') - }) - }) - - describe('min/max constraints', () => { - it.concurrent('should accept values within range', () => { - const result = validateInteger(50, 'value', { min: 0, max: 100 }) - expect(result.isValid).toBe(true) - }) - - it.concurrent('should reject values below min', () => { - const result = validateInteger(-1, 'value', { min: 0 }) - expect(result.isValid).toBe(false) - expect(result.error).toContain('at least 0') - }) - - it.concurrent('should reject values above max', () => { - const result = validateInteger(101, 'value', { max: 100 }) - expect(result.isValid).toBe(false) - expect(result.error).toContain('at most 100') - }) - - it.concurrent('should accept value equal to min', () => { - const result = validateInteger(0, 'value', { min: 0 }) - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept value equal to max', () => { - const result = validateInteger(100, 'value', { max: 100 }) - expect(result.isValid).toBe(true) - }) - }) -}) - describe('validateMicrosoftGraphId', () => { describe('valid IDs', () => { it.concurrent('should accept simple alphanumeric IDs', () => { @@ -1033,46 +883,50 @@ describe('validateJiraIssueKey', () => { describe('validateExternalUrl', () => { describe('valid URLs', () => { it.concurrent('should accept https URLs', () => { - const result = validateExternalUrl('https://example.com') + const result = validateExternalUrl('https://example.com', 'url', 'configuredEndpoint') expect(result.isValid).toBe(true) }) it.concurrent('should accept URLs with paths', () => { - const result = validateExternalUrl('https://api.example.com/v1/data') + const result = validateExternalUrl( + 'https://api.example.com/v1/data', + 'url', + 'configuredEndpoint' + ) expect(result.isValid).toBe(true) }) it.concurrent('should accept URLs with query strings', () => { - const result = validateExternalUrl('https://example.com?foo=bar') + const result = validateExternalUrl('https://example.com?foo=bar', 'url', 'configuredEndpoint') expect(result.isValid).toBe(true) }) it.concurrent('should accept URLs with standard ports', () => { - const result = validateExternalUrl('https://example.com:443/api') + const result = validateExternalUrl('https://example.com:443/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(true) }) }) describe('invalid URLs', () => { it.concurrent('should reject null', () => { - const result = validateExternalUrl(null) + const result = validateExternalUrl(null, 'url', 'configuredEndpoint') expect(result.isValid).toBe(false) expect(result.error).toContain('required') }) it.concurrent('should reject empty string', () => { - const result = validateExternalUrl('') + const result = validateExternalUrl('', 'url', 'configuredEndpoint') expect(result.isValid).toBe(false) }) it.concurrent('should reject http URLs', () => { - const result = validateExternalUrl('http://example.com') + const result = validateExternalUrl('http://example.com', 'url', 'configuredEndpoint') expect(result.isValid).toBe(false) expect(result.error).toContain('https://') }) it.concurrent('should reject invalid URLs', () => { - const result = validateExternalUrl('not-a-url') + const result = validateExternalUrl('not-a-url', 'url', 'configuredEndpoint') expect(result.isValid).toBe(false) expect(result.error).toContain('valid URL') }) @@ -1080,22 +934,22 @@ describe('validateExternalUrl', () => { describe('localhost and loopback addresses (self-hosted)', () => { it.concurrent('should accept https localhost', () => { - const result = validateExternalUrl('https://localhost/api') + const result = validateExternalUrl('https://localhost/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(true) }) it.concurrent('should accept http localhost', () => { - const result = validateExternalUrl('http://localhost/api') + const result = validateExternalUrl('http://localhost/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(true) }) it.concurrent('should accept https 127.0.0.1', () => { - const result = validateExternalUrl('https://127.0.0.1/api') + const result = validateExternalUrl('https://127.0.0.1/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(true) }) it.concurrent('should accept http 127.0.0.1', () => { - const result = validateExternalUrl('http://127.0.0.1/api') + const result = validateExternalUrl('http://127.0.0.1/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(true) }) @@ -1106,205 +960,124 @@ describe('validateExternalUrl', () => { * address was localhost to one caller and a plain http URL to the other. */ it.concurrent('should treat the rest of the loopback range as localhost too', () => { - expect(validateExternalUrl('http://127.0.0.2/api').isValid).toBe(true) - expect(validateExternalUrl('http://127.1.2.3/api').isValid).toBe(true) + expect(validateExternalUrl('http://127.0.0.2/api', 'url', 'configuredEndpoint').isValid).toBe( + true + ) + expect(validateExternalUrl('http://127.1.2.3/api', 'url', 'configuredEndpoint').isValid).toBe( + true + ) // Still only loopback — neighbouring private ranges stay rejected. - expect(validateExternalUrl('http://10.0.0.1/api').isValid).toBe(false) - expect(validateExternalUrl('http://192.168.1.1/api').isValid).toBe(false) + expect(validateExternalUrl('http://10.0.0.1/api', 'url', 'configuredEndpoint').isValid).toBe( + false + ) + expect( + validateExternalUrl('http://192.168.1.1/api', 'url', 'configuredEndpoint').isValid + ).toBe(false) }) it.concurrent('should accept https IPv6 loopback', () => { - const result = validateExternalUrl('https://[::1]/api') + const result = validateExternalUrl('https://[::1]/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(true) }) it.concurrent('should accept http IPv6 loopback', () => { - const result = validateExternalUrl('http://[::1]/api') + const result = validateExternalUrl('http://[::1]/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(true) }) it.concurrent('should reject 0.0.0.0', () => { - const result = validateExternalUrl('https://0.0.0.0/api') + const result = validateExternalUrl('https://0.0.0.0/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(false) - expect(result.error).toContain('private IP') + expect(result.error).toContain('private or reserved address') }) }) describe('private IP ranges', () => { it.concurrent('should reject 10.x.x.x', () => { - const result = validateExternalUrl('https://10.0.0.1/api') + const result = validateExternalUrl('https://10.0.0.1/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(false) - expect(result.error).toContain('private IP') + expect(result.error).toContain('private or reserved address') }) it.concurrent('should reject 172.16.x.x', () => { - const result = validateExternalUrl('https://172.16.0.1/api') + const result = validateExternalUrl('https://172.16.0.1/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(false) - expect(result.error).toContain('private IP') + expect(result.error).toContain('private or reserved address') }) it.concurrent('should reject 192.168.x.x', () => { - const result = validateExternalUrl('https://192.168.1.1/api') + const result = validateExternalUrl('https://192.168.1.1/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(false) - expect(result.error).toContain('private IP') + expect(result.error).toContain('private or reserved address') }) it.concurrent('should reject link-local 169.254.x.x', () => { - const result = validateExternalUrl('https://169.254.169.254/api') + const result = validateExternalUrl('https://169.254.169.254/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(false) - expect(result.error).toContain('private IP') + // The metadata endpoint gets its own, more specific refusal. + expect(result.error).toContain('cloud metadata endpoint') }) }) describe('blocked ports', () => { it.concurrent('should reject SSH port 22', () => { - const result = validateExternalUrl('https://example.com:22/api') + const result = validateExternalUrl('https://example.com:22/api', 'url', 'configuredEndpoint') expect(result.isValid).toBe(false) expect(result.error).toContain('blocked port') }) it.concurrent('should reject MySQL port 3306', () => { - const result = validateExternalUrl('https://example.com:3306/api') + const result = validateExternalUrl( + 'https://example.com:3306/api', + 'url', + 'configuredEndpoint' + ) expect(result.isValid).toBe(false) expect(result.error).toContain('blocked port') }) it.concurrent('should reject PostgreSQL port 5432', () => { - const result = validateExternalUrl('https://example.com:5432/api') + const result = validateExternalUrl( + 'https://example.com:5432/api', + 'url', + 'configuredEndpoint' + ) expect(result.isValid).toBe(false) expect(result.error).toContain('blocked port') }) it.concurrent('should reject Redis port 6379', () => { - const result = validateExternalUrl('https://example.com:6379/api') + const result = validateExternalUrl( + 'https://example.com:6379/api', + 'url', + 'configuredEndpoint' + ) expect(result.isValid).toBe(false) expect(result.error).toContain('blocked port') }) it.concurrent('should reject MongoDB port 27017', () => { - const result = validateExternalUrl('https://example.com:27017/api') + const result = validateExternalUrl( + 'https://example.com:27017/api', + 'url', + 'configuredEndpoint' + ) expect(result.isValid).toBe(false) expect(result.error).toContain('blocked port') }) it.concurrent('should reject Elasticsearch port 9200', () => { - const result = validateExternalUrl('https://example.com:9200/api') + const result = validateExternalUrl( + 'https://example.com:9200/api', + 'url', + 'configuredEndpoint' + ) expect(result.isValid).toBe(false) expect(result.error).toContain('blocked port') }) }) }) -describe('validateImageUrl', () => { - it.concurrent('should accept valid image URLs', () => { - const result = validateImageUrl('https://example.com/image.png') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept localhost URLs (self-hosted)', () => { - const result = validateImageUrl('https://localhost/image.png') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should use imageUrl as default param name', () => { - const result = validateImageUrl(null) - expect(result.error).toContain('imageUrl') - }) -}) - -describe('validateProxyUrl', () => { - it.concurrent('should accept valid proxy URLs', () => { - const result = validateProxyUrl('https://proxy.example.com/api') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should reject private IPs', () => { - const result = validateProxyUrl('https://192.168.1.1:8080') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should use proxyUrl as default param name', () => { - const result = validateProxyUrl(null) - expect(result.error).toContain('proxyUrl') - }) -}) - -describe('validateGoogleCalendarId', () => { - describe('valid calendar IDs', () => { - it.concurrent('should accept "primary"', () => { - const result = validateGoogleCalendarId('primary') - expect(result.isValid).toBe(true) - expect(result.sanitized).toBe('primary') - }) - - it.concurrent('should accept email addresses', () => { - const result = validateGoogleCalendarId('user@example.com') - expect(result.isValid).toBe(true) - expect(result.sanitized).toBe('user@example.com') - }) - - it.concurrent('should accept Google calendar format', () => { - const result = validateGoogleCalendarId('en.usa#holiday@group.v.calendar.google.com') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept alphanumeric IDs with allowed characters', () => { - const result = validateGoogleCalendarId('abc123_def-456') - expect(result.isValid).toBe(true) - }) - }) - - describe('invalid calendar IDs', () => { - it.concurrent('should reject null', () => { - const result = validateGoogleCalendarId(null) - expect(result.isValid).toBe(false) - expect(result.error).toContain('required') - }) - - it.concurrent('should reject empty string', () => { - const result = validateGoogleCalendarId('') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject path traversal', () => { - const result = validateGoogleCalendarId('../etc/passwd') - expect(result.isValid).toBe(false) - expect(result.error).toContain('path traversal') - }) - - it.concurrent('should reject URL-encoded path traversal', () => { - const result = validateGoogleCalendarId('%2e%2e%2f') - expect(result.isValid).toBe(false) - expect(result.error).toContain('path traversal') - }) - - it.concurrent('should reject null bytes', () => { - const result = validateGoogleCalendarId('test\0value') - expect(result.isValid).toBe(false) - expect(result.error).toContain('control characters') - }) - - it.concurrent('should reject newline characters', () => { - const result = validateGoogleCalendarId('test\nvalue') - expect(result.isValid).toBe(false) - expect(result.error).toContain('control characters') - }) - - it.concurrent('should reject IDs exceeding 255 characters', () => { - const longId = 'a'.repeat(256) - const result = validateGoogleCalendarId(longId) - expect(result.isValid).toBe(false) - expect(result.error).toContain('maximum length') - }) - - it.concurrent('should reject invalid characters', () => { - const result = validateGoogleCalendarId('test') - expect(result.isValid).toBe(false) - expect(result.error).toContain('format is invalid') - }) - }) -}) - describe('validateAirtableId', () => { describe('valid base IDs (app prefix)', () => { it.concurrent('should accept valid base ID', () => { @@ -1867,264 +1640,6 @@ describe('validateMondayNumericId', () => { }) }) -describe('validateMondayGroupId', () => { - describe('valid inputs', () => { - it.concurrent('should accept simple group IDs', () => { - const result = validateMondayGroupId('topics') - expect(result.isValid).toBe(true) - expect(result.sanitized).toBe('topics') - }) - - it.concurrent('should accept group IDs with underscores', () => { - const result = validateMondayGroupId('new_group') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept group IDs with spaces', () => { - const result = validateMondayGroupId('test group id') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept group IDs with uppercase letters', () => { - const result = validateMondayGroupId('Group One') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept group IDs with digits', () => { - const result = validateMondayGroupId('group123') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept auto-generated group IDs', () => { - const result = validateMondayGroupId('group_title') - expect(result.isValid).toBe(true) - }) - }) - - describe('invalid inputs', () => { - it.concurrent('should reject null', () => { - const result = validateMondayGroupId(null) - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject empty string', () => { - const result = validateMondayGroupId('') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject strings with brackets', () => { - const result = validateMondayGroupId('group"]){id}#') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject strings with quotes', () => { - const result = validateMondayGroupId('group")') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject control characters', () => { - const result = validateMondayGroupId('group\x00id') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject strings exceeding max length', () => { - const result = validateMondayGroupId('a'.repeat(256)) - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject strings with special characters', () => { - const result = validateMondayGroupId('group;DROP') - expect(result.isValid).toBe(false) - }) - }) -}) - -describe('validateMondayColumnId', () => { - describe('valid inputs', () => { - it.concurrent('should accept simple column IDs', () => { - const result = validateMondayColumnId('status') - expect(result.isValid).toBe(true) - expect(result.sanitized).toBe('status') - }) - - it.concurrent('should accept column IDs with digits', () => { - const result = validateMondayColumnId('date4') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept auto-generated column IDs', () => { - const result = validateMondayColumnId('email_mksr9hcd') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept column IDs with underscores', () => { - const result = validateMondayColumnId('color_mksreyj6') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept single character column IDs', () => { - const result = validateMondayColumnId('a') - expect(result.isValid).toBe(true) - }) - }) - - describe('invalid inputs', () => { - it.concurrent('should reject null', () => { - const result = validateMondayColumnId(null) - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject empty string', () => { - const result = validateMondayColumnId('') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject uppercase letters', () => { - const result = validateMondayColumnId('Status') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject spaces', () => { - const result = validateMondayColumnId('my column') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject hyphens', () => { - const result = validateMondayColumnId('my-column') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject special characters', () => { - const result = validateMondayColumnId('col;DROP') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject strings exceeding max length', () => { - const result = validateMondayColumnId('a'.repeat(256)) - expect(result.isValid).toBe(false) - }) - }) - - describe('validateSupabaseProjectId', () => { - describe('valid inputs', () => { - it.concurrent('should accept a typical 20-char lowercase alphanumeric project ID', () => { - const result = validateSupabaseProjectId('jdrkgepadsdopsntdlom') - expect(result.isValid).toBe(true) - expect(result.sanitized).toBe('jdrkgepadsdopsntdlom') - }) - - it.concurrent('should accept project IDs with digits', () => { - const result = validateSupabaseProjectId('abc123def456ghi789jk') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept IDs at the minimum length boundary (10)', () => { - const result = validateSupabaseProjectId('abcdefghij') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept IDs at the maximum length boundary (40)', () => { - const result = validateSupabaseProjectId('a'.repeat(40)) - expect(result.isValid).toBe(true) - }) - }) - - describe('SSRF attack vectors', () => { - it.concurrent('should reject fragment injection (#)', () => { - const result = validateSupabaseProjectId('evil#attacker.com') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject @ for authority injection', () => { - const result = validateSupabaseProjectId('evil@attacker.com') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject path traversal with slashes', () => { - const result = validateSupabaseProjectId('evil/../../etc/passwd') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject dots (subdomain manipulation)', () => { - const result = validateSupabaseProjectId('evil.attacker.com') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject backslashes', () => { - const result = validateSupabaseProjectId('evil\\path') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject colons (port injection)', () => { - const result = validateSupabaseProjectId('evil:8080') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject URL-encoded characters', () => { - const result = validateSupabaseProjectId('evil%23attacker') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject spaces', () => { - const result = validateSupabaseProjectId('evil host') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject newlines (header injection)', () => { - const result = validateSupabaseProjectId('evil\r\nHost: attacker.com') - expect(result.isValid).toBe(false) - }) - }) - - describe('invalid formats', () => { - it.concurrent('should reject null', () => { - const result = validateSupabaseProjectId(null) - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject undefined', () => { - const result = validateSupabaseProjectId(undefined) - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject empty string', () => { - const result = validateSupabaseProjectId('') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject uppercase letters', () => { - const result = validateSupabaseProjectId('JDRKGEPADSDOPSNTDLOM') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject mixed case', () => { - const result = validateSupabaseProjectId('jdrkGEPadsdOPSntdlom') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject hyphens', () => { - const result = validateSupabaseProjectId('jdrk-gepa-dsdo-psnt') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject underscores', () => { - const result = validateSupabaseProjectId('jdrk_gepa_dsdo_psnt') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject IDs shorter than 10 characters', () => { - const result = validateSupabaseProjectId('abcdefghi') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject IDs longer than 40 characters', () => { - const result = validateSupabaseProjectId('a'.repeat(41)) - expect(result.isValid).toBe(false) - }) - }) - }) -}) - describe('validateCallbackUrl', () => { const ORIGIN = 'https://sim.app' const originalWindow = (globalThis as { window?: unknown }).window @@ -2291,7 +1806,7 @@ describe('validateServiceNowInstanceUrl', () => { it.concurrent('should reject private IPs', () => { const result = validateServiceNowInstanceUrl('https://192.168.1.1') expect(result.isValid).toBe(false) - expect(result.error).toContain('private IP') + expect(result.error).toContain('private or reserved address') }) it.concurrent('should reject link-local metadata IP', () => { @@ -2389,7 +1904,7 @@ describe('validateWorkdayTenantUrl', () => { it.concurrent('should reject private IPs', () => { const result = validateWorkdayTenantUrl('https://192.168.1.1') expect(result.isValid).toBe(false) - expect(result.error).toContain('private IP') + expect(result.error).toContain('private or reserved address') }) it.concurrent('should reject link-local metadata IP (SSRF classic)', () => { diff --git a/apps/sim/lib/core/security/input-validation.ts b/apps/sim/lib/core/security/input-validation.ts index fb448e04a56..e96f2e542b1 100644 --- a/apps/sim/lib/core/security/input-validation.ts +++ b/apps/sim/lib/core/security/input-validation.ts @@ -1,7 +1,10 @@ import { createLogger } from '@sim/logger' -import { isLoopbackIp, isPrivateIp, unwrapIpv6Brackets } from '@sim/security/ssrf' -import * as ipaddr from 'ipaddr.js' -import { isHosted } from '@/lib/core/config/env-flags' +import { evaluateUrl } from '@sim/security/egress' +import { + describeEgressDenial, + type EgressProfile, + resolveEgressPolicy, +} from '@/lib/core/security/egress/profiles' import { getBaseUrl } from '@/lib/core/utils/urls' const logger = createLogger('InputValidation') @@ -12,7 +15,8 @@ export interface ValidationResult { sanitized?: string } -export interface PathSegmentOptions { +/** Options for {@link validatePathSegment}. */ +interface PathSegmentOptions { /** Name of the parameter for error messages */ paramName?: string /** Maximum length allowed (default: 255) */ @@ -246,80 +250,6 @@ export function validateNumericId( return { isValid: true, sanitized: num.toString() } } -/** - * Validates an integer value (from JSON body or other sources) - * - * This is stricter than validateNumericId - it requires: - * - Value must already be a number type (not string) - * - Must be an integer (no decimals) - * - Must be finite (not NaN or Infinity) - * - * @param value - The value to validate - * @param paramName - Name of the parameter for error messages - * @param options - Additional options (min, max) - * @returns ValidationResult - * - * @example - * ```typescript - * const result = validateInteger(failedCount, 'failedCount', { min: 0 }) - * if (!result.isValid) { - * return NextResponse.json({ error: result.error }, { status: 400 }) - * } - * ``` - */ -export function validateInteger( - value: unknown, - paramName = 'value', - options: { min?: number; max?: number } = {} -): ValidationResult { - if (value === null || value === undefined) { - return { - isValid: false, - error: `${paramName} is required`, - } - } - - if (typeof value !== 'number') { - logger.warn('Value is not a number', { paramName, valueType: typeof value }) - return { - isValid: false, - error: `${paramName} must be a number`, - } - } - - if (Number.isNaN(value) || !Number.isFinite(value)) { - logger.warn('Invalid number value', { paramName, value }) - return { - isValid: false, - error: `${paramName} must be a valid number`, - } - } - - if (!Number.isInteger(value)) { - logger.warn('Value is not an integer', { paramName, value }) - return { - isValid: false, - error: `${paramName} must be an integer`, - } - } - - if (options.min !== undefined && value < options.min) { - return { - isValid: false, - error: `${paramName} must be at least ${options.min}`, - } - } - - if (options.max !== undefined && value > options.max) { - return { - isValid: false, - error: `${paramName} must be at most ${options.max}`, - } - } - - return { isValid: true } -} - /** * Validates that a value is in an allowed list (enum validation) * @@ -363,121 +293,6 @@ export function validateEnum( return { isValid: true, sanitized: value } } -/** - * Validates a hostname to prevent SSRF attacks - * - * This function checks that a hostname is not a private IP, localhost, or other reserved address. - * It complements the validateProxyUrl function by providing hostname-specific validation. - * - * @param hostname - The hostname to validate - * @param paramName - Name of the parameter for error messages - * @returns ValidationResult - * - * @example - * ```typescript - * const result = validateHostname(webhookDomain, 'webhook domain') - * if (!result.isValid) { - * return NextResponse.json({ error: result.error }, { status: 400 }) - * } - * ``` - */ -export function validateHostname( - hostname: string | null | undefined, - paramName = 'hostname' -): ValidationResult { - if (hostname === null || hostname === undefined || hostname === '') { - return { - isValid: false, - error: `${paramName} is required`, - } - } - - const lowerHostname = hostname.toLowerCase() - - if (lowerHostname === 'localhost') { - logger.warn('Hostname is localhost', { paramName }) - return { - isValid: false, - error: `${paramName} cannot be a private IP address or localhost`, - } - } - - if (ipaddr.isValid(lowerHostname)) { - if (isPrivateIp(lowerHostname)) { - logger.warn('Hostname matches blocked IP range', { - paramName, - hostname: hostname.substring(0, 100), - }) - return { - isValid: false, - error: `${paramName} cannot be a private IP address or localhost`, - } - } - } - - const hostnamePattern = - /^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*$/i - - if (!hostnamePattern.test(hostname)) { - logger.warn('Invalid hostname format', { - paramName, - hostname: hostname.substring(0, 100), - }) - return { - isValid: false, - error: `${paramName} is not a valid hostname`, - } - } - - return { isValid: true, sanitized: hostname } -} - -/** - * Validates a file extension - * - * @param extension - The file extension (with or without leading dot) - * @param allowedExtensions - Array of allowed extensions (without dots) - * @param paramName - Name of the parameter for error messages - * @returns ValidationResult - * - * @example - * ```typescript - * const result = validateFileExtension(ext, ['jpg', 'png', 'gif'], 'file extension') - * if (!result.isValid) { - * return NextResponse.json({ error: result.error }, { status: 400 }) - * } - * ``` - */ -export function validateFileExtension( - extension: string | null | undefined, - allowedExtensions: readonly string[], - paramName = 'file extension' -): ValidationResult { - if (extension === null || extension === undefined || extension === '') { - return { - isValid: false, - error: `${paramName} is required`, - } - } - - const ext = extension.startsWith('.') ? extension.slice(1) : extension - const normalizedExt = ext.toLowerCase() - - if (!allowedExtensions.map((e) => e.toLowerCase()).includes(normalizedExt)) { - logger.warn('File extension not in allowed list', { - paramName, - extension: ext, - allowedExtensions, - }) - return { - isValid: false, - error: `${paramName} must be one of: ${allowedExtensions.join(', ')}`, - } - } - - return { isValid: true, sanitized: normalizedExt } -} - /** * Validates Microsoft Graph API resource IDs * @@ -629,27 +444,6 @@ export function validateJiraCloudId( }) } -/** - * Validates an Atlassian Assets workspace ID (a UUID-shaped, hyphenated - * alphanumeric identifier) before it is interpolated into an API path. - * - * @param value - The Assets workspace ID to validate - * @param paramName - Name of the parameter for error messages - * @returns ValidationResult - */ -export function validateAssetsWorkspaceId( - value: string | null | undefined, - paramName = 'workspaceId' -): ValidationResult { - return validatePathSegment(value, { - paramName, - allowHyphens: true, - allowUnderscores: false, - allowDots: false, - maxLength: 100, - }) -} - /** * Validates Jira issue keys (format: PROJECT-123 or PROJECT-KEY-123) * @@ -679,20 +473,22 @@ export function validateJiraIssueKey( } /** - * Validates a URL to prevent SSRF attacks + * Synchronous, pre-DNS egress check for a URL. * - * This function checks that URLs: - * - Use https:// protocol only - * - Do not point to private IP ranges or localhost - * - Do not use suspicious ports + * This is the cheap half of the guard: it rejects a bad scheme, a denied port, + * and a disallowed IP literal without a lookup. A hostname it accepts is NOT + * cleared to be dialed — only {@link validateUrlWithDNS} can do that, because + * only a resolved address can be classified. Use this for form/contract + * validation; use the DNS-resolving variant before connecting. * * @param url - The URL to validate * @param paramName - Name of the parameter for error messages + * @param profile - Where this URL came from; see {@link EgressProfile} * @returns ValidationResult * * @example * ```typescript - * const result = validateExternalUrl(url, 'fileUrl') + * const result = validateExternalUrl(url, 'fileUrl', 'configuredEndpoint') * if (!result.isValid) { * return NextResponse.json({ error: result.error }, { status: 400 }) * } @@ -700,100 +496,24 @@ export function validateJiraIssueKey( */ export function validateExternalUrl( url: string | null | undefined, - paramName = 'url', - options: { allowHttp?: boolean } = {} + paramName: string, + profile: EgressProfile ): ValidationResult { if (!url || typeof url !== 'string') { - return { - isValid: false, - error: `${paramName} is required and must be a string`, - } + return { isValid: false, error: `${paramName} is required and must be a string` } } - let parsedUrl: URL + let parsed: URL try { - parsedUrl = new URL(url) + parsed = new URL(url) } catch { - return { - isValid: false, - error: `${paramName} must be a valid URL`, - } - } - - const protocol = parsedUrl.protocol - const hostname = parsedUrl.hostname.toLowerCase() - - const cleanHostname = unwrapIpv6Brackets(hostname) - - // The whole loopback range, not just 127.0.0.1: 127.0.0.2 is the same - // machine, and matching two literals made this validator disagree with MCP's - // domain-check about what "localhost" means. Both directions stay coherent — - // hosted rejects the wider set, self-hosted permits http on it. - const isLocalhost = cleanHostname === 'localhost' || isLoopbackIp(cleanHostname) - - if (isLocalhost && isHosted) { - return { - isValid: false, - error: `${paramName} cannot point to localhost`, - } - } - - if (options.allowHttp) { - if (protocol !== 'https:' && protocol !== 'http:') { - return { - isValid: false, - error: `${paramName} must use http:// or https:// protocol`, - } - } - } else if (protocol !== 'https:' && !(protocol === 'http:' && isLocalhost && !isHosted)) { - return { - isValid: false, - error: `${paramName} must use https:// protocol`, - } + return { isValid: false, error: `${paramName} must be a valid URL` } } - if (!isLocalhost && ipaddr.isValid(cleanHostname)) { - if (isPrivateIp(cleanHostname)) { - return { - isValid: false, - error: `${paramName} cannot point to private IP addresses`, - } - } - } - - const port = parsedUrl.port - const blockedPorts = ['22', '23', '25', '3306', '5432', '6379', '27017', '9200'] - - if (port && blockedPorts.includes(port)) { - return { - isValid: false, - error: `${paramName} uses a blocked port`, - } - } - - return { isValid: true } -} - -/** - * Validates an image URL to prevent SSRF attacks - * Alias for validateExternalUrl for backward compatibility - */ -export function validateImageUrl( - url: string | null | undefined, - paramName = 'imageUrl' -): ValidationResult { - return validateExternalUrl(url, paramName) -} - -/** - * Validates a proxy URL to prevent SSRF attacks - * Alias for validateExternalUrl for backward compatibility - */ -export function validateProxyUrl( - url: string | null | undefined, - paramName = 'proxyUrl' -): ValidationResult { - return validateExternalUrl(url, paramName) + const decision = evaluateUrl(parsed, resolveEgressPolicy(profile)) + return decision.allowed + ? { isValid: true } + : { isValid: false, error: describeEgressDenial(decision, paramName, profile) } } /** @@ -1054,115 +774,6 @@ export function validateS3BucketName( return { isValid: true, sanitized: value } } -/** - * Validates a Google Calendar ID - * - * Google Calendar IDs can be: - * - "primary" (literal string for the user's primary calendar) - * - Email addresses (for user calendars) - * - Alphanumeric strings with hyphens, underscores, and dots (for other calendars) - * - * This validator allows these legitimate formats while blocking path traversal and injection attempts. - * - * @param value - The calendar ID to validate - * @param paramName - Name of the parameter for error messages - * @returns ValidationResult - * - * @example - * ```typescript - * const result = validateGoogleCalendarId(calendarId, 'calendarId') - * if (!result.isValid) { - * return NextResponse.json({ error: result.error }, { status: 400 }) - * } - * ``` - */ -export function validateGoogleCalendarId( - value: string | null | undefined, - paramName = 'calendarId' -): ValidationResult { - if (value === null || value === undefined || value === '') { - return { - isValid: false, - error: `${paramName} is required`, - } - } - - if (value === 'primary') { - return { isValid: true, sanitized: value } - } - - const pathTraversalPatterns = [ - '../', - '..\\', - '%2e%2e%2f', - '%2e%2e/', - '..%2f', - '%2e%2e%5c', - '%2e%2e\\', - '..%5c', - '%252e%252e%252f', - ] - - const lowerValue = value.toLowerCase() - for (const pattern of pathTraversalPatterns) { - if (lowerValue.includes(pattern)) { - logger.warn('Path traversal attempt in Google Calendar ID', { - paramName, - value: value.substring(0, 100), - }) - return { - isValid: false, - error: `${paramName} contains invalid path traversal sequence`, - } - } - } - - if (/[\x00-\x1f\x7f]/.test(value) || value.includes('%00')) { - logger.warn('Control characters in Google Calendar ID', { paramName }) - return { - isValid: false, - error: `${paramName} contains invalid control characters`, - } - } - - if (value.includes('\n') || value.includes('\r')) { - return { - isValid: false, - error: `${paramName} contains invalid newline characters`, - } - } - - const emailPattern = /^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/ - if (emailPattern.test(value)) { - return { isValid: true, sanitized: value } - } - - const calendarIdPattern = /^[a-zA-Z0-9._@%#+-]+$/ - if (!calendarIdPattern.test(value)) { - logger.warn('Invalid Google Calendar ID format', { - paramName, - value: value.substring(0, 100), - }) - return { - isValid: false, - error: `${paramName} format is invalid. Must be "primary", an email address, or an alphanumeric ID`, - } - } - - if (value.length > 255) { - logger.warn('Google Calendar ID exceeds maximum length', { - paramName, - length: value.length, - }) - return { - isValid: false, - error: `${paramName} exceeds maximum length of 255 characters`, - } - } - - return { isValid: true, sanitized: value } -} - /** * Validates a pagination cursor token * @@ -1401,125 +1012,6 @@ export function validateMondayNumericId( return { isValid: true, sanitized: str } } -/** - * Validates a Monday.com group ID. - * - * Monday.com group IDs are strings that can contain lowercase/uppercase letters, - * digits, underscores, and spaces. They are user-visible identifiers like - * "topics", "new_group", or "test group id". Auto-generated IDs may also - * include "group_title" patterns. - * - * @param value - The group ID to validate - * @param paramName - Name of the parameter for error messages - * @returns ValidationResult - * - * @example - * ```typescript - * const result = validateMondayGroupId(groupId, 'groupId') - * if (!result.isValid) { - * return NextResponse.json({ error: result.error }, { status: 400 }) - * } - * ``` - */ -export function validateMondayGroupId( - value: string | null | undefined, - paramName = 'groupId' -): ValidationResult { - if (value === null || value === undefined || value === '') { - return { - isValid: false, - error: `${paramName} is required`, - } - } - - if (value.length > 255) { - logger.warn('Monday.com group ID exceeds maximum length', { - paramName, - length: value.length, - }) - return { - isValid: false, - error: `${paramName} exceeds maximum length of 255 characters`, - } - } - - if (/[\x00-\x1f\x7f]/.test(value) || value.includes('%00')) { - logger.warn('Monday.com group ID contains control characters', { paramName }) - return { - isValid: false, - error: `${paramName} contains invalid control characters`, - } - } - - if (!/^[a-zA-Z0-9_ ]+$/.test(value)) { - logger.warn('Monday.com group ID contains disallowed characters', { - paramName, - value: value.substring(0, 100), - }) - return { - isValid: false, - error: `${paramName} can only contain letters, digits, underscores, and spaces`, - } - } - - return { isValid: true, sanitized: value } -} - -/** - * Validates a Monday.com column ID. - * - * Column IDs are strings containing lowercase letters (a-z), digits (0-9), - * and underscores. User-specified IDs are 1-20 characters of [a-z_]. - * Auto-generated IDs follow patterns like "status", "date4", "email_mksr9hcd". - * - * @param value - The column ID to validate - * @param paramName - Name of the parameter for error messages - * @returns ValidationResult - * - * @example - * ```typescript - * const result = validateMondayColumnId(columnId, 'columnId') - * if (!result.isValid) { - * return NextResponse.json({ error: result.error }, { status: 400 }) - * } - * ``` - */ -export function validateMondayColumnId( - value: string | null | undefined, - paramName = 'columnId' -): ValidationResult { - if (value === null || value === undefined || value === '') { - return { - isValid: false, - error: `${paramName} is required`, - } - } - - if (value.length > 255) { - logger.warn('Monday.com column ID exceeds maximum length', { - paramName, - length: value.length, - }) - return { - isValid: false, - error: `${paramName} exceeds maximum length of 255 characters`, - } - } - - if (!/^[a-z0-9_]+$/.test(value)) { - logger.warn('Monday.com column ID contains disallowed characters', { - paramName, - value: value.substring(0, 100), - }) - return { - isValid: false, - error: `${paramName} can only contain lowercase letters, digits, and underscores`, - } - } - - return { isValid: true, sanitized: value } -} - /** * Validates a Supabase project ID. * @@ -1587,6 +1079,66 @@ const SERVICENOW_ALLOWED_HOST_SUFFIXES = [ '.servicenowservices.com', ] as const +/** + * Validates a vendor-hosted URL: an ordinary egress check, then a hostname + * allowlist that pins it to the vendor's own domains. + * + * The allowlist is what makes these connectors safe to point at a + * customer-supplied tenant: egress validation alone would accept any public + * host, so a tenant field would otherwise be an open redirect for credentials + * scoped to that vendor. + * + * @param url - The URL or bare host to validate + * @param options.suffixes - Permitted host suffixes, each written with a leading dot + * @param options.vendor - Vendor name, used in the error message + * @param options.paramName - Name of the parameter for error messages + * @param options.assumeHttps - Accept a bare host by prepending `https://` + * @param options.sanitize - What to return as `sanitized`: the input, or the parsed origin + */ +function validateVendorHostedUrl( + url: string | null | undefined, + options: { + suffixes: readonly string[] + vendor: string + paramName: string + assumeHttps?: boolean + sanitize?: 'input' | 'origin' + } +): ValidationResult { + const { suffixes, vendor, paramName, assumeHttps = false, sanitize = 'input' } = options + + const raw = typeof url === 'string' ? url.trim() : '' + if (!raw) { + return { isValid: false, error: `${paramName} is required` } + } + + const candidate = assumeHttps && !/^https?:\/\//i.test(raw) ? `https://${raw}` : raw + + const urlResult = validateExternalUrl(candidate, paramName, 'configuredEndpoint') + if (!urlResult.isValid) return urlResult + + const parsed = new URL(candidate) + const hostname = parsed.hostname.toLowerCase() + const allowed = suffixes.some( + (suffix) => hostname === suffix.slice(1) || hostname.endsWith(suffix) + ) + + if (!allowed) { + logger.warn(`${vendor} host not on allowlist`, { + paramName, + hostname: hostname.substring(0, 100), + }) + return { + isValid: false, + error: `${paramName} must be a ${vendor}-hosted domain (e.g., ${suffixes + .map((suffix) => `*${suffix}`) + .join(', ')})`, + } + } + + return { isValid: true, sanitized: sanitize === 'origin' ? parsed.origin : candidate } +} + /** * Validates a ServiceNow instance URL to prevent SSRF attacks. * @@ -1620,26 +1172,11 @@ export function validateServiceNowInstanceUrl( url: string | null | undefined, paramName = 'instanceUrl' ): ValidationResult { - const urlResult = validateExternalUrl(url, paramName) - if (!urlResult.isValid) return urlResult - - const hostname = new URL(url as string).hostname.toLowerCase() - const isAllowedHost = SERVICENOW_ALLOWED_HOST_SUFFIXES.some( - (suffix) => hostname === suffix.slice(1) || hostname.endsWith(suffix) - ) - - if (!isAllowedHost) { - logger.warn('ServiceNow instance URL hostname not on allowlist', { - paramName, - hostname: hostname.substring(0, 100), - }) - return { - isValid: false, - error: `${paramName} must be a ServiceNow-hosted domain (e.g., *.service-now.com, *.servicenow.com, or *.servicenowservices.com)`, - } - } - - return { isValid: true, sanitized: url as string } + return validateVendorHostedUrl(url, { + suffixes: SERVICENOW_ALLOWED_HOST_SUFFIXES, + vendor: 'ServiceNow', + paramName, + }) } const WORKDAY_ALLOWED_HOST_SUFFIXES = ['.workday.com', '.myworkday.com'] as const @@ -1673,26 +1210,11 @@ export function validateWorkdayTenantUrl( url: string | null | undefined, paramName = 'tenantUrl' ): ValidationResult { - const urlResult = validateExternalUrl(url, paramName) - if (!urlResult.isValid) return urlResult - - const hostname = new URL(url as string).hostname.toLowerCase() - const isAllowedHost = WORKDAY_ALLOWED_HOST_SUFFIXES.some( - (suffix) => hostname === suffix.slice(1) || hostname.endsWith(suffix) - ) - - if (!isAllowedHost) { - logger.warn('Workday tenant URL hostname not on allowlist', { - paramName, - hostname: hostname.substring(0, 100), - }) - return { - isValid: false, - error: `${paramName} must be a Workday-hosted domain (e.g., *.workday.com or *.myworkday.com)`, - } - } - - return { isValid: true, sanitized: url as string } + return validateVendorHostedUrl(url, { + suffixes: WORKDAY_ALLOWED_HOST_SUFFIXES, + vendor: 'Workday', + paramName, + }) } /** @@ -1750,32 +1272,13 @@ export function validateDatabricksWorkspaceHost( host: string | null | undefined, paramName = 'workspaceHost' ): ValidationResult { - const raw = typeof host === 'string' ? host.trim() : '' - if (!raw) { - return { isValid: false, error: `${paramName} is required` } - } - - const withScheme = /^https?:\/\//i.test(raw) ? raw : `https://${raw}` - - const urlResult = validateExternalUrl(withScheme, paramName) - if (!urlResult.isValid) return urlResult - - const parsed = new URL(withScheme) - const hostname = parsed.hostname.toLowerCase() - const isAllowedHost = DATABRICKS_ALLOWED_HOST_SUFFIXES.some((suffix) => hostname.endsWith(suffix)) - - if (!isAllowedHost) { - logger.warn('Databricks workspace host not on allowlist', { - paramName, - hostname: hostname.substring(0, 100), - }) - return { - isValid: false, - error: `${paramName} must be a Databricks-hosted domain (e.g., *.cloud.databricks.com, *.azuredatabricks.net, or *.gcp.databricks.com)`, - } - } - - return { isValid: true, sanitized: parsed.origin } + return validateVendorHostedUrl(host, { + suffixes: DATABRICKS_ALLOWED_HOST_SUFFIXES, + vendor: 'Databricks', + paramName, + assumeHttps: true, + sanitize: 'origin', + }) } /** diff --git a/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts b/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts index cccf9ddfa8d..e05746c9d24 100644 --- a/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts +++ b/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts @@ -15,7 +15,8 @@ vi.mock('@sim/security/dns', () => ({ vi.mock('@/lib/core/config/env-flags', () => ({ isHosted: false, - isPrivateDatabaseHostsAllowed: false, + egressAllowedHosts: undefined, + egressAllowedIpRanges: undefined, getProxyUrl: () => undefined, })) @@ -70,7 +71,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { await expect( secureFetchWithPinnedIP(origin, '127.0.0.1', { - allowHttp: true, + profile: 'configuredEndpoint', assertRedirectTarget, }) ).rejects.toThrow('redirect target rejected') @@ -96,7 +97,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { 'Content-Type': 'application/json', Host: 'legacy.example', }, - allowHttp: true, + profile: 'configuredEndpoint', }) expect(response.status).toBe(200) @@ -130,7 +131,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { sendCredentialsOnCrossOriginRedirect: false, sensitiveHeaders: ['x-api-key'], }, - allowHttp: true, + profile: 'configuredEndpoint', }) expect(hops).toHaveLength(1) @@ -166,7 +167,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { sendCredentialsOnCrossOriginRedirect: false, sensitiveHeaders: ['x-api-key'], }, - allowHttp: true, + profile: 'configuredEndpoint', }) expect(response.status).toBe(200) @@ -202,7 +203,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false, }, - allowHttp: true, + profile: 'configuredEndpoint', }) expect(hops).toHaveLength(1) @@ -227,7 +228,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false, }, - allowHttp: true, + profile: 'configuredEndpoint', }) expect(hops).toHaveLength(1) @@ -255,7 +256,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false, }, - allowHttp: true, + profile: 'configuredEndpoint', }) expect(hops).toHaveLength(1) @@ -283,7 +284,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { mode: 'standard', sendCredentialsOnCrossOriginRedirect: true, }, - allowHttp: true, + profile: 'configuredEndpoint', }) expect(hops).toHaveLength(1) @@ -321,7 +322,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false, }, - allowHttp: true, + profile: 'configuredEndpoint', }) expect(response.status).toBe(200) @@ -354,7 +355,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { method: 'GET', headers: { Authorization: 'Bearer strip-me', 'X-Trace': 'keep-me' }, stripAuthOnRedirect: true, - allowHttp: true, + profile: 'configuredEndpoint', }) expect(hops).toHaveLength(1) diff --git a/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts b/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts index 78d6f21805d..7a362e1c5ed 100644 --- a/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts +++ b/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts @@ -12,7 +12,8 @@ vi.mock('@sim/security/dns', () => ({ vi.mock('@/lib/core/config/env-flags', () => ({ isHosted: false, - isPrivateDatabaseHostsAllowed: false, + egressAllowedHosts: undefined, + egressAllowedIpRanges: undefined, getProxyUrl: () => undefined, })) diff --git a/apps/sim/lib/data-drains/destinations/s3.ts b/apps/sim/lib/data-drains/destinations/s3.ts index 32727bfb16c..de0e15dafd6 100644 --- a/apps/sim/lib/data-drains/destinations/s3.ts +++ b/apps/sim/lib/data-drains/destinations/s3.ts @@ -85,7 +85,7 @@ const s3ConfigSchema = z.object({ .string() .url() .refine((v) => v.startsWith('https://'), { message: 'endpoint must use https://' }) - .refine((value) => validateExternalUrl(value, 'endpoint').isValid, { + .refine((value) => validateExternalUrl(value, 'endpoint', 'configuredEndpoint').isValid, { message: 'endpoint must be HTTPS and not point at a private, loopback, or metadata address', }) .optional(), @@ -128,7 +128,7 @@ function isS3ServiceException(error: unknown): error is S3ServiceException { /** DNS-aware SSRF check: catches hostnames that resolve to internal IPs (the schema check only catches IP literals). */ async function assertEndpointIsPublic(endpoint: string | undefined): Promise { if (!endpoint) return - const result = await validateUrlWithDNS(endpoint, 'endpoint') + const result = await validateUrlWithDNS(endpoint, 'endpoint', 'configuredEndpoint') if (!result.isValid) { throw new Error(result.error ?? 'S3 endpoint failed SSRF validation') } diff --git a/apps/sim/lib/data-drains/destinations/webhook.ts b/apps/sim/lib/data-drains/destinations/webhook.ts index 4e9cd289550..1403bd3bd34 100644 --- a/apps/sim/lib/data-drains/destinations/webhook.ts +++ b/apps/sim/lib/data-drains/destinations/webhook.ts @@ -46,7 +46,7 @@ const RESERVED_SIGNATURE_HEADER_NAMES = new Set([ const HEADER_INJECTION_PATTERN = /[\r\n\0]/ async function resolvePublicTarget(url: string): Promise { - const result = await validateUrlWithDNS(url, 'url') + const result = await validateUrlWithDNS(url, 'url', 'configuredEndpoint') if (!result.isValid || !result.resolvedIP) { throw new Error(result.error ?? 'Webhook URL failed SSRF validation') } @@ -58,7 +58,7 @@ const webhookConfigSchema = z.object({ .string() .url('url must be a valid URL') .max(2048, 'url must be at most 2048 characters') - .refine((value) => validateExternalUrl(value, 'url').isValid, { + .refine((value) => validateExternalUrl(value, 'url', 'configuredEndpoint').isValid, { message: 'url must be HTTPS and not point at a private, loopback, or metadata address', }), signatureHeader: z @@ -165,6 +165,7 @@ export const webhookDestination: DrainDestination< isProbe: true, }) const response = await secureFetchWithPinnedIP(config.url, resolvedIP, { + profile: 'configuredEndpoint', method: 'POST', body: new Uint8Array(probe), headers, @@ -193,6 +194,7 @@ export const webhookDestination: DrainDestination< let response: Awaited> | undefined try { response = await secureFetchWithPinnedIP(config.url, resolvedIP, { + profile: 'configuredEndpoint', method: 'POST', body: new Uint8Array(body), headers, diff --git a/apps/sim/lib/execution/isolated-vm.ts b/apps/sim/lib/execution/isolated-vm.ts index ba9832fe41f..93bbf55fedb 100644 --- a/apps/sim/lib/execution/isolated-vm.ts +++ b/apps/sim/lib/execution/isolated-vm.ts @@ -251,9 +251,14 @@ function truncateString(value: string, maxChars: number): { value: string; trunc } function normalizeFetchOptions(options?: IsolatedFetchOptions): SecureFetchOptions { - if (!options) return { maxResponseBytes: MAX_FETCH_RESPONSE_BYTES } + // The Function block's `fetch()` reaches whatever the workflow author's script + // asks for, so it is governed as a request target rather than a configured one. + if (!options) { + return { profile: 'requestTarget', maxResponseBytes: MAX_FETCH_RESPONSE_BYTES } + } const normalized: SecureFetchOptions = { + profile: 'requestTarget', maxResponseBytes: MAX_FETCH_RESPONSE_BYTES, } diff --git a/apps/sim/lib/function-execution/execute-request.test.ts b/apps/sim/lib/function-execution/execute-request.test.ts index 2fd1b35eaa5..06b55600472 100644 --- a/apps/sim/lib/function-execution/execute-request.test.ts +++ b/apps/sim/lib/function-execution/execute-request.test.ts @@ -143,7 +143,7 @@ vi.mock('@/lib/uploads', () => ({ vi.mock('@/lib/workflows/utils', () => workflowsUtilsMock) -import { validateProxyUrl } from '@/lib/core/security/input-validation' +import { validateExternalUrl } from '@/lib/core/security/input-validation' import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache' import { isLargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest-metadata' import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref' @@ -576,23 +576,25 @@ describe('Function execution request', () => { expect(data.output.result).toBe('undefined') }) + const proxyTarget = (url: string) => validateExternalUrl(url, 'url', 'proxy') + it.concurrent('should block SSRF attacks through secure fetch wrapper', async () => { - expect(validateProxyUrl('http://169.254.169.254/latest/meta-data/').isValid).toBe(false) - expect(validateProxyUrl('http://127.0.0.1:8080/admin').isValid).toBe(true) - expect(validateProxyUrl('http://192.168.1.1/config').isValid).toBe(false) - expect(validateProxyUrl('http://10.0.0.1/internal').isValid).toBe(false) + expect(proxyTarget('http://169.254.169.254/latest/meta-data/').isValid).toBe(false) + expect(proxyTarget('http://127.0.0.1:8080/admin').isValid).toBe(false) + expect(proxyTarget('http://192.168.1.1/config').isValid).toBe(false) + expect(proxyTarget('http://10.0.0.1/internal').isValid).toBe(false) }) it.concurrent('should allow legitimate external URLs', async () => { - expect(validateProxyUrl('https://api.github.com/user').isValid).toBe(true) - expect(validateProxyUrl('https://httpbin.org/get').isValid).toBe(true) - expect(validateProxyUrl('https://example.com/api').isValid).toBe(true) + expect(proxyTarget('https://api.github.com/user').isValid).toBe(true) + expect(proxyTarget('https://httpbin.org/get').isValid).toBe(true) + expect(proxyTarget('https://example.com/api').isValid).toBe(true) }) it.concurrent('should block dangerous protocols', async () => { - expect(validateProxyUrl('file:///etc/passwd').isValid).toBe(false) - expect(validateProxyUrl('ftp://internal.server/files').isValid).toBe(false) - expect(validateProxyUrl('gopher://old.server/menu').isValid).toBe(false) + expect(proxyTarget('file:///etc/passwd').isValid).toBe(false) + expect(proxyTarget('ftp://internal.server/files').isValid).toBe(false) + expect(proxyTarget('gopher://old.server/menu').isValid).toBe(false) }) }) diff --git a/apps/sim/lib/internal/agiloft/client.test.ts b/apps/sim/lib/internal/agiloft/client.test.ts index b20c68312ef..a6ca3f50b86 100644 --- a/apps/sim/lib/internal/agiloft/client.test.ts +++ b/apps/sim/lib/internal/agiloft/client.test.ts @@ -79,7 +79,8 @@ describe('executeAgiloftRequest', () => { expect(mockValidateUrlWithDNS).toHaveBeenCalledWith( 'https://example.agiloft.com', - 'instanceUrl' + 'instanceUrl', + 'configuredEndpoint' ) const calls = mockSecureFetch.mock.calls diff --git a/apps/sim/lib/internal/agiloft/client.ts b/apps/sim/lib/internal/agiloft/client.ts index a837c890f38..43cbbdada3e 100644 --- a/apps/sim/lib/internal/agiloft/client.ts +++ b/apps/sim/lib/internal/agiloft/client.ts @@ -31,7 +31,7 @@ export async function resolveAgiloftInstance( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const validation = await validateUrlWithDNS(instanceUrl, 'instanceUrl') + const validation = await validateUrlWithDNS(instanceUrl, 'instanceUrl', 'configuredEndpoint') signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) { throw new Error(validation.error || 'Invalid Agiloft instance URL') @@ -81,6 +81,7 @@ export async function agiloftLoginPinned( const base = params.instanceUrl.replace(/\/$/, '') const response = await secureFetchWithPinnedIP(`${base}/ewws/EWLogin`, resolvedIP, { + profile: 'configuredEndpoint', method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: formEncode( @@ -139,6 +140,7 @@ export async function agiloftLogoutPinned( `${base}/ewws/EWLogout?$KB=${kb}&$lang=${AGILOFT_LANG}`, resolvedIP, { + profile: 'configuredEndpoint', method: 'POST', headers: { Authorization: authorization }, maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, @@ -182,6 +184,7 @@ export async function executeAgiloftRequest( try { const req = buildRequest(base) const response = await secureFetchWithPinnedIP(req.url, resolvedIP, { + profile: 'configuredEndpoint', method: req.method, headers: { ...req.headers, @@ -280,6 +283,7 @@ export async function executeAlrestRequest( try { const req = buildRequest(agiloftAlrestBase(params.instanceUrl, params.knowledgeBase)) const response = await secureFetchWithPinnedIP(req.url, resolvedIP, { + profile: 'configuredEndpoint', method: req.method, headers: { ...req.headers, Authorization: session.authorization }, body: req.body, @@ -313,6 +317,7 @@ export async function executeEwRequest( const resolvedIP = await resolveAgiloftInstance(params.instanceUrl, signal) const req = buildRequest(params.instanceUrl.replace(/\/$/, '')) const response = await secureFetchWithPinnedIP(req.url, resolvedIP, { + profile: 'configuredEndpoint', method: req.method, headers: req.headers, body: req.body, diff --git a/apps/sim/lib/internal/agiloft/operations.test.ts b/apps/sim/lib/internal/agiloft/operations.test.ts index 922e1b33a57..a9e373f3244 100644 --- a/apps/sim/lib/internal/agiloft/operations.test.ts +++ b/apps/sim/lib/internal/agiloft/operations.test.ts @@ -178,6 +178,7 @@ describe('Agiloft operations', () => { expect.stringContaining('/ewws/EWRetrieve'), '203.0.113.10', { + profile: 'configuredEndpoint', method: 'GET', maxResponseBytes: 25 * 1024 * 1024, signal: controller.signal, diff --git a/apps/sim/lib/internal/agiloft/operations.ts b/apps/sim/lib/internal/agiloft/operations.ts index 91c983b44b5..a346fd60ef8 100644 --- a/apps/sim/lib/internal/agiloft/operations.ts +++ b/apps/sim/lib/internal/agiloft/operations.ts @@ -853,6 +853,7 @@ export async function executeAgiloftAttachFile( buildAttachFileUrl(input.instanceUrl.replace(/\/$/, ''), input, fileName), resolvedIP, { + profile: 'configuredEndpoint', method: 'PUT', headers: { 'Content-Type': 'application/octet-stream' }, body: buffer, @@ -903,7 +904,12 @@ export async function executeAgiloftRetrieveAttachment( const response = await secureFetchWithPinnedIP( buildRetrieveAttachmentUrl(input.instanceUrl.replace(/\/$/, ''), input), resolvedIP, - { method: 'GET', maxResponseBytes: AGILOFT_MAX_ATTACHMENT_BYTES, signal: context.signal } + { + profile: 'configuredEndpoint', + method: 'GET', + maxResponseBytes: AGILOFT_MAX_ATTACHMENT_BYTES, + signal: context.signal, + } ) if (!response.ok) { const text = await response.text() diff --git a/apps/sim/lib/internal/azure-data-explorer/client.ts b/apps/sim/lib/internal/azure-data-explorer/client.ts index eaca58026b7..c5c71b957b7 100644 --- a/apps/sim/lib/internal/azure-data-explorer/client.ts +++ b/apps/sim/lib/internal/azure-data-explorer/client.ts @@ -84,6 +84,7 @@ async function fetchAccessToken( const response = await secureFetchWithValidation( `${authority}/${encodeURIComponent(input.tenantId)}/oauth2/token`, { + profile: 'configuredEndpoint', method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -269,6 +270,7 @@ export async function requestAzureDataExplorer( const response = await secureFetchWithValidation( `${clusterUrl.origin}/v1/rest/${input.endpoint}`, { + profile: 'configuredEndpoint', method: 'POST', headers, body: JSON.stringify({ diff --git a/apps/sim/lib/internal/brex/client.test.ts b/apps/sim/lib/internal/brex/client.test.ts index 1c07cfa4b7d..cad850caf7b 100644 --- a/apps/sim/lib/internal/brex/client.test.ts +++ b/apps/sim/lib/internal/brex/client.test.ts @@ -73,6 +73,7 @@ describe('BrexReceiptClient', () => { ) expect(mocks.pinnedFetch).toHaveBeenCalledWith('https://upload.example/file', '52.216.0.1', { + profile: 'contentFetch', method: 'PUT', headers: { 'Content-Length': String(buffer.byteLength) }, body: new Uint8Array(buffer), diff --git a/apps/sim/lib/internal/brex/client.ts b/apps/sim/lib/internal/brex/client.ts index eab3a5b0edf..0877bf5ded6 100644 --- a/apps/sim/lib/internal/brex/client.ts +++ b/apps/sim/lib/internal/brex/client.ts @@ -71,12 +71,13 @@ export class BrexReceiptClient { async uploadReceipt(uri: string, buffer: Buffer): Promise { this.signal?.throwIfAborted() - const validation = await validateUrlWithDNS(uri, 'uri') + const validation = await validateUrlWithDNS(uri, 'uri', 'contentFetch') this.signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) { throw new BrexReceiptError('Brex returned an invalid upload URL', 502) } const response = await secureFetchWithPinnedIP(uri, validation.resolvedIP, { + profile: 'contentFetch', method: 'PUT', headers: { 'Content-Length': String(buffer.byteLength) }, body: new Uint8Array(buffer), diff --git a/apps/sim/lib/internal/buffer/operations.ts b/apps/sim/lib/internal/buffer/operations.ts index 71c7f6c0262..a6ca65c13e2 100644 --- a/apps/sim/lib/internal/buffer/operations.ts +++ b/apps/sim/lib/internal/buffer/operations.ts @@ -72,10 +72,11 @@ async function resolveMediaKind(args: { if (extensionKind) return extensionKind try { - const validation = await validateUrlWithDNS(fileUrl, 'media') + const validation = await validateUrlWithDNS(fileUrl, 'media', 'contentFetch') context.signal?.throwIfAborted() if (validation.isValid && validation.resolvedIP) { const probe = await secureFetchWithPinnedIP(fileUrl, validation.resolvedIP, { + profile: 'contentFetch', method: 'HEAD', timeout: MEDIA_PROBE_TIMEOUT_MS, signal: context.signal, diff --git a/apps/sim/lib/internal/clickhouse/client.test.ts b/apps/sim/lib/internal/clickhouse/client.test.ts index 2c77749162e..04a0b34464c 100644 --- a/apps/sim/lib/internal/clickhouse/client.test.ts +++ b/apps/sim/lib/internal/clickhouse/client.test.ts @@ -95,7 +95,7 @@ describe('clickhouseRequest DNS pinning', () => { const [url, , options] = mockSecureFetchWithPinnedIP.mock.calls[0] expect(url).toMatch(/^https:\/\//) - expect(options.allowHttp).toBe(false) + expect(options.profile).toBe('configuredEndpoint') }) it('allows http for the initial request when secure is false', async () => { @@ -103,7 +103,7 @@ describe('clickhouseRequest DNS pinning', () => { const [url, , options] = mockSecureFetchWithPinnedIP.mock.calls[0] expect(url).toMatch(/^http:\/\//) - expect(options.allowHttp).toBe(true) + expect(options.profile).toBe('configuredEndpoint') }) it('brackets an unbracketed IPv6 literal when constructing the request URL', async () => { diff --git a/apps/sim/lib/internal/clickhouse/client.ts b/apps/sim/lib/internal/clickhouse/client.ts index cde12f9ff3f..46494c9a5be 100644 --- a/apps/sim/lib/internal/clickhouse/client.ts +++ b/apps/sim/lib/internal/clickhouse/client.ts @@ -74,7 +74,7 @@ export async function requestClickHouse( }, body: statement, timeout: REQUEST_TIMEOUT_MS, - allowHttp: !config.secure, + profile: 'configuredEndpoint', maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, redirectPolicy: { mode: 'standard', diff --git a/apps/sim/lib/internal/cursor/operations.test.ts b/apps/sim/lib/internal/cursor/operations.test.ts index 1105d9df52c..1eff865fe46 100644 --- a/apps/sim/lib/internal/cursor/operations.test.ts +++ b/apps/sim/lib/internal/cursor/operations.test.ts @@ -45,7 +45,7 @@ describe('downloadCursorArtifact', () => { expect(mocks.secureFetchWithPinnedIP).toHaveBeenCalledWith( 'https://download.example/artifact', '203.0.113.1', - { signal: controller.signal } + { profile: 'contentFetch', signal: controller.signal } ) expect(result.output.file).toEqual({ name: 'index.ts', diff --git a/apps/sim/lib/internal/cursor/operations.ts b/apps/sim/lib/internal/cursor/operations.ts index 2c1ce81b477..86d0b8cff1c 100644 --- a/apps/sim/lib/internal/cursor/operations.ts +++ b/apps/sim/lib/internal/cursor/operations.ts @@ -65,12 +65,13 @@ export async function downloadCursorArtifact( throw new CursorOperationError('No download URL returned for artifact', 400) } - const validation = await validateUrlWithDNS(downloadUrl, 'downloadUrl') + const validation = await validateUrlWithDNS(downloadUrl, 'downloadUrl', 'contentFetch') context.signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) { throw new CursorOperationError(validation.error || 'Invalid download URL', 400) } const downloadResponse = await secureFetchWithPinnedIP(downloadUrl, validation.resolvedIP, { + profile: 'contentFetch', signal: context.signal, }) if (!downloadResponse.ok) { diff --git a/apps/sim/lib/internal/extend/client.ts b/apps/sim/lib/internal/extend/client.ts index df11e20eca3..0b26f0eb73e 100644 --- a/apps/sim/lib/internal/extend/client.ts +++ b/apps/sim/lib/internal/extend/client.ts @@ -21,7 +21,11 @@ export async function submitExtendParse( signal?: AbortSignal ): Promise> { signal?.throwIfAborted() - const validation = await validateUrlWithDNS(EXTEND_ENDPOINT, 'Extend API URL') + const validation = await validateUrlWithDNS( + EXTEND_ENDPOINT, + 'Extend API URL', + 'configuredEndpoint' + ) signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) { throw new ExtendOperationError(502, { success: false, error: 'Failed to reach Extend API' }) @@ -30,6 +34,7 @@ export async function submitExtendParse( let response: Awaited> try { response = await secureFetchWithPinnedIP(EXTEND_ENDPOINT, validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/lib/internal/github/operations.ts b/apps/sim/lib/internal/github/operations.ts index 49522552564..7759ea395e4 100644 --- a/apps/sim/lib/internal/github/operations.ts +++ b/apps/sim/lib/internal/github/operations.ts @@ -319,10 +319,11 @@ async function fetchChangedFileContent( ): Promise { if (file.status === 'removed' || !file.raw_url || remainingBytes <= 0) return undefined try { - const validation = await validateUrlWithDNS(file.raw_url, 'rawUrl') + const validation = await validateUrlWithDNS(file.raw_url, 'rawUrl', 'contentFetch') context.signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) return undefined const response = await secureFetchWithPinnedIP(file.raw_url, validation.resolvedIP, { + profile: 'contentFetch', headers: { Authorization: `Bearer ${apiKey}`, 'X-GitHub-Api-Version': '2022-11-28', @@ -356,13 +357,14 @@ export async function getGitHubLatestCommit( const repo = encodeURIComponent(input.repo) const revision = encodeURIComponent(input.branch || 'HEAD') const commitUrl = `https://api.github.com/repos/${owner}/${repo}/commits/${revision}` - const validation = await validateUrlWithDNS(commitUrl, 'commitUrl') + const validation = await validateUrlWithDNS(commitUrl, 'commitUrl', 'configuredEndpoint') context.signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) { throw new GitHubOperationError(validation.error || 'Invalid GitHub commit URL', 400) } const response = await secureFetchWithPinnedIP(commitUrl, validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'GET', headers: { Accept: 'application/vnd.github.v3+json', diff --git a/apps/sim/lib/internal/google-drive/client.test.ts b/apps/sim/lib/internal/google-drive/client.test.ts index c2f1f965248..a66d48d2d8a 100644 --- a/apps/sim/lib/internal/google-drive/client.test.ts +++ b/apps/sim/lib/internal/google-drive/client.test.ts @@ -36,7 +36,8 @@ describe('requestGoogleDrive', () => { expect(mocks.validateUrl).toHaveBeenCalledWith( 'https://www.googleapis.com/drive/v3/files/file-1', - 'metadataUrl' + 'metadataUrl', + 'configuredEndpoint' ) expect(mocks.secureFetch).toHaveBeenCalledWith( 'https://www.googleapis.com/drive/v3/files/file-1', diff --git a/apps/sim/lib/internal/google-drive/client.ts b/apps/sim/lib/internal/google-drive/client.ts index 3397fcf1fb6..f393cc733ab 100644 --- a/apps/sim/lib/internal/google-drive/client.ts +++ b/apps/sim/lib/internal/google-drive/client.ts @@ -25,7 +25,7 @@ export async function requestGoogleDrive( options: GoogleDriveRequestOptions ): Promise { options.signal?.throwIfAborted() - const validation = await validateUrlWithDNS(options.url, options.label) + const validation = await validateUrlWithDNS(options.url, options.label, 'configuredEndpoint') options.signal?.throwIfAborted() if (!validation.isValid) { throw new GoogleDriveOperationError(400, { @@ -35,6 +35,7 @@ export async function requestGoogleDrive( } return secureFetchWithPinnedIP(options.url, validation.resolvedIP!, { + profile: 'configuredEndpoint', method: options.method, headers: { Authorization: `Bearer ${options.accessToken}`, diff --git a/apps/sim/lib/internal/google-slides/operations.ts b/apps/sim/lib/internal/google-slides/operations.ts index 9aa5158533e..044241a9cde 100644 --- a/apps/sim/lib/internal/google-slides/operations.ts +++ b/apps/sim/lib/internal/google-slides/operations.ts @@ -42,7 +42,11 @@ export async function exportGoogleSlidesPresentation( const exportFormat = input.exportFormat ?? 'PDF' const mimeType = FORMAT_TO_MIME[exportFormat] const exportUrl = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(input.presentationId)}/export?mimeType=${encodeURIComponent(mimeType)}` - const validation = await validateUrlWithDNS(exportUrl, 'googleSlidesExportUrl') + const validation = await validateUrlWithDNS( + exportUrl, + 'googleSlidesExportUrl', + 'configuredEndpoint' + ) context.signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) { throw new GoogleSlidesOperationError( @@ -52,6 +56,7 @@ export async function exportGoogleSlidesPresentation( } const response = await secureFetchWithPinnedIP(exportUrl, validation.resolvedIP, { + profile: 'configuredEndpoint', headers: { Authorization: `Bearer ${input.accessToken}` }, maxResponseBytes: MAX_GOOGLE_SLIDES_EXPORT_BYTES, signal: context.signal, diff --git a/apps/sim/lib/internal/google-vault/operations.test.ts b/apps/sim/lib/internal/google-vault/operations.test.ts index 69f16be89ef..fb7d34b9045 100644 --- a/apps/sim/lib/internal/google-vault/operations.test.ts +++ b/apps/sim/lib/internal/google-vault/operations.test.ts @@ -46,6 +46,7 @@ describe('downloadGoogleVaultExportFile', () => { expect.stringContaining('/storage/v1/b/bucket-1/o/exports%2Fresult.zip?alt=media'), '203.0.113.1', { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: 'Bearer token' }, maxResponseBytes: MAX_BUFFERED_TRANSFER_BYTES, diff --git a/apps/sim/lib/internal/google-vault/operations.ts b/apps/sim/lib/internal/google-vault/operations.ts index 14762da9e66..e74a9cc1048 100644 --- a/apps/sim/lib/internal/google-vault/operations.ts +++ b/apps/sim/lib/internal/google-vault/operations.ts @@ -42,7 +42,7 @@ export async function downloadGoogleVaultExportFile( const bucket = encodeURIComponent(input.bucketName) const object = encodeURIComponent(input.objectName) const downloadUrl = `https://storage.googleapis.com/storage/v1/b/${bucket}/o/${object}?alt=media` - const validation = await validateUrlWithDNS(downloadUrl, 'downloadUrl') + const validation = await validateUrlWithDNS(downloadUrl, 'downloadUrl', 'configuredEndpoint') context.signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) { throw new GoogleVaultOperationError( @@ -52,6 +52,7 @@ export async function downloadGoogleVaultExportFile( } const response = await secureFetchWithPinnedIP(downloadUrl, validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: `Bearer ${input.accessToken}` }, maxResponseBytes: MAX_BUFFERED_TRANSFER_BYTES, diff --git a/apps/sim/lib/internal/grafana/client.test.ts b/apps/sim/lib/internal/grafana/client.test.ts index e041a9ecb21..a1d098c757a 100644 --- a/apps/sim/lib/internal/grafana/client.test.ts +++ b/apps/sim/lib/internal/grafana/client.test.ts @@ -40,7 +40,8 @@ describe('GrafanaClient', () => { expect(mocks.validateUrlWithDNS).toHaveBeenCalledWith( 'https://grafana.example.com/api/folders/folder-1', - 'baseUrl' + 'baseUrl', + 'configuredEndpoint' ) expect(mocks.secureFetchWithPinnedIP).toHaveBeenCalledWith( 'https://grafana.example.com/api/folders/folder-1', diff --git a/apps/sim/lib/internal/grafana/client.ts b/apps/sim/lib/internal/grafana/client.ts index bccf83c8f95..c8b40350c7d 100644 --- a/apps/sim/lib/internal/grafana/client.ts +++ b/apps/sim/lib/internal/grafana/client.ts @@ -28,7 +28,7 @@ export class GrafanaClient { ): Promise { this.signal?.throwIfAborted() const url = `${this.baseUrl}${path}` - const validation = await validateUrlWithDNS(url, 'baseUrl') + const validation = await validateUrlWithDNS(url, 'baseUrl', 'configuredEndpoint') this.signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) { return { success: false, error: `Invalid Grafana baseUrl: ${validation.error}` } @@ -43,6 +43,7 @@ export class GrafanaClient { if (this.organizationId) headers['X-Grafana-Org-Id'] = this.organizationId const response = await secureFetchWithPinnedIP(url, validation.resolvedIP, { + profile: 'configuredEndpoint', method: options.method, headers, ...(options.body === undefined ? {} : { body: JSON.stringify(options.body) }), diff --git a/apps/sim/lib/internal/image/fetch.ts b/apps/sim/lib/internal/image/fetch.ts index 23d8760d647..5691a756a83 100644 --- a/apps/sim/lib/internal/image/fetch.ts +++ b/apps/sim/lib/internal/image/fetch.ts @@ -32,13 +32,14 @@ export async function fetchRemoteImage( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const validation = await validateUrlWithDNS(imageUrl, 'imageUrl') + const validation = await validateUrlWithDNS(imageUrl, 'imageUrl', 'contentFetch') if (!validation.isValid || !validation.resolvedIP) { throw new RemoteImageFetchError(validation.error || 'Invalid image URL', 403) } try { const response = await secureFetchWithPinnedIP(imageUrl, validation.resolvedIP, { + profile: 'contentFetch', method: 'GET', maxResponseBytes: MAX_REMOTE_IMAGE_BYTES, signal, diff --git a/apps/sim/lib/internal/image/operations.ts b/apps/sim/lib/internal/image/operations.ts index 591b4979168..c09d1ffba57 100644 --- a/apps/sim/lib/internal/image/operations.ts +++ b/apps/sim/lib/internal/image/operations.ts @@ -376,12 +376,13 @@ async function bufferFromImageUrl( } } - const urlValidation = await validateUrlWithDNS(url, 'imageUrl') + const urlValidation = await validateUrlWithDNS(url, 'imageUrl', 'contentFetch') if (!urlValidation.isValid || !urlValidation.resolvedIP) { throw new Error(urlValidation.error || 'Generated image URL failed validation') } const imageResponse = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP, { + profile: 'contentFetch', method: 'GET', maxResponseBytes: MAX_IMAGE_BYTES, signal, diff --git a/apps/sim/lib/internal/jsm/client.ts b/apps/sim/lib/internal/jsm/client.ts index 6a2ada6c6aa..1dfb8df2665 100644 --- a/apps/sim/lib/internal/jsm/client.ts +++ b/apps/sim/lib/internal/jsm/client.ts @@ -1,7 +1,4 @@ -import { - validateAssetsWorkspaceId, - validateJiraCloudId, -} from '@/lib/core/security/input-validation' +import { validateJiraCloudId } from '@/lib/core/security/input-validation' import { JsmOperationError } from '@/lib/internal/jsm/errors' import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' import { resolveAssetsContext } from '@/tools/jsm/utils' @@ -175,7 +172,7 @@ export async function createJsmAssetsClient( signal?.throwIfAborted() const cloudId = validateJiraCloudId(context.cloudId, 'cloudId') if (!cloudId.isValid) throw new JsmOperationError(cloudId.error || 'Invalid cloudId', 400) - const workspaceId = validateAssetsWorkspaceId(context.workspaceId, 'workspaceId') + const workspaceId = validateJiraCloudId(context.workspaceId, 'workspaceId') if (!workspaceId.isValid) { throw new JsmOperationError(workspaceId.error || 'Invalid workspaceId', 400) } diff --git a/apps/sim/lib/internal/jupyter/client.test.ts b/apps/sim/lib/internal/jupyter/client.test.ts index c88f3694337..15e70dc7c29 100644 --- a/apps/sim/lib/internal/jupyter/client.test.ts +++ b/apps/sim/lib/internal/jupyter/client.test.ts @@ -43,7 +43,7 @@ describe('Jupyter client', () => { expect(securityMocks.validateUrlWithDNS).toHaveBeenCalledWith( 'http://jupyter.example.com:8888/base/api/kernels', 'serverUrl', - { allowHttp: true } + 'configuredEndpoint' ) expect(securityMocks.secureFetchWithPinnedIP).toHaveBeenCalledWith( 'http://jupyter.example.com:8888/base/api/kernels', @@ -55,7 +55,7 @@ describe('Jupyter client', () => { 'Content-Type': 'application/json', }, body: JSON.stringify({ name: 'python3' }), - allowHttp: true, + profile: 'configuredEndpoint', maxRedirects: 0, maxResponseBytes: 10 * 1024 * 1024, signal: controller.signal, diff --git a/apps/sim/lib/internal/jupyter/client.ts b/apps/sim/lib/internal/jupyter/client.ts index b6f88b02358..40c988b4686 100644 --- a/apps/sim/lib/internal/jupyter/client.ts +++ b/apps/sim/lib/internal/jupyter/client.ts @@ -43,7 +43,7 @@ export async function requestJupyterApi( } const url = `${base}/api/${input.path}` - const urlValidation = await validateUrlWithDNS(url, 'serverUrl', { allowHttp: true }) + const urlValidation = await validateUrlWithDNS(url, 'serverUrl', 'configuredEndpoint') signal?.throwIfAborted() if (!urlValidation.isValid || !urlValidation.resolvedIP) { throw new InvalidJupyterTargetError(`Invalid Jupyter serverUrl: ${urlValidation.error}`) @@ -57,7 +57,7 @@ export async function requestJupyterApi( ...(hasBody ? { 'Content-Type': 'application/json' } : {}), }, body: hasBody ? JSON.stringify(input.body) : undefined, - allowHttp: true, + profile: 'configuredEndpoint', maxRedirects: 0, maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, signal, diff --git a/apps/sim/lib/internal/linq/client.ts b/apps/sim/lib/internal/linq/client.ts index 52dc3359407..a772f4bbe2f 100644 --- a/apps/sim/lib/internal/linq/client.ts +++ b/apps/sim/lib/internal/linq/client.ts @@ -86,12 +86,13 @@ export async function uploadLinqAttachmentBytes( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const validation = await validateUrlWithDNS(registration.uploadUrl, 'uploadUrl') + const validation = await validateUrlWithDNS(registration.uploadUrl, 'uploadUrl', 'contentFetch') signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) { throw new LinqOperationError(validation.error || 'Invalid Linq upload URL', 400) } const response = await secureFetchWithPinnedIP(registration.uploadUrl, validation.resolvedIP, { + profile: 'contentFetch', method: registration.httpMethod, headers: registration.requiredHeaders, body: new Uint8Array(buffer), diff --git a/apps/sim/lib/internal/microsoft-dataverse/client.ts b/apps/sim/lib/internal/microsoft-dataverse/client.ts index 7fc2c067d1d..2eedf0dfc86 100644 --- a/apps/sim/lib/internal/microsoft-dataverse/client.ts +++ b/apps/sim/lib/internal/microsoft-dataverse/client.ts @@ -19,6 +19,7 @@ export async function uploadDataverseFile( const response = await secureFetchWithValidation( input.uploadUrl, { + profile: 'contentFetch', method: 'PATCH', headers: { Authorization: `Bearer ${input.accessToken}`, diff --git a/apps/sim/lib/internal/microsoft-word/client.ts b/apps/sim/lib/internal/microsoft-word/client.ts index db123397537..c6a9e4df282 100644 --- a/apps/sim/lib/internal/microsoft-word/client.ts +++ b/apps/sim/lib/internal/microsoft-word/client.ts @@ -55,15 +55,18 @@ export class GraphRequestError extends Error { async function graphFetch( url: string, paramName: string, - options: NonNullable[2]> + options: Omit[2]>, 'profile'> ) { options.signal?.throwIfAborted() - const validation = await validateUrlWithDNS(url, paramName) + const validation = await validateUrlWithDNS(url, paramName, 'configuredEndpoint') options.signal?.throwIfAborted() if (!validation.isValid) { throw new GraphRequestError(validation.error || `Invalid ${paramName}`, 400) } - return secureFetchWithPinnedIP(url, validation.resolvedIP as string, options) + return secureFetchWithPinnedIP(url, validation.resolvedIP as string, { + ...options, + profile: 'configuredEndpoint', + }) } /** Reads a Graph error body and raises it as a {@link GraphRequestError}. */ diff --git a/apps/sim/lib/internal/mistral/client.ts b/apps/sim/lib/internal/mistral/client.ts index 593445a037b..3394880cd7f 100644 --- a/apps/sim/lib/internal/mistral/client.ts +++ b/apps/sim/lib/internal/mistral/client.ts @@ -17,7 +17,11 @@ export async function submitMistralOcr( maxResponseBytes = DEFAULT_MAX_RESPONSE_BYTES ): Promise { signal?.throwIfAborted() - const validation = await validateUrlWithDNS(MISTRAL_ENDPOINT, 'Mistral API URL') + const validation = await validateUrlWithDNS( + MISTRAL_ENDPOINT, + 'Mistral API URL', + 'configuredEndpoint' + ) signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) { throw new MistralOperationError(502, { @@ -27,6 +31,7 @@ export async function submitMistralOcr( } const response = await secureFetchWithPinnedIP(MISTRAL_ENDPOINT, validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/lib/internal/mistral/operations.ts b/apps/sim/lib/internal/mistral/operations.ts index 4ced4dab583..910cdaf4a30 100644 --- a/apps/sim/lib/internal/mistral/operations.ts +++ b/apps/sim/lib/internal/mistral/operations.ts @@ -162,7 +162,7 @@ async function buildUrlDocument( }) } else { const { validateUrlWithDNS } = await import('@/lib/core/security/input-validation.server') - const validation = await validateUrlWithDNS(fileUrl, 'filePath') + const validation = await validateUrlWithDNS(fileUrl, 'filePath', 'contentFetch') context.signal?.throwIfAborted() if (!validation.isValid) { throw new MistralOperationError(400, { success: false, error: validation.error }) diff --git a/apps/sim/lib/internal/onedrive/operations.ts b/apps/sim/lib/internal/onedrive/operations.ts index 0eaab6fb5f7..f8792ef4df6 100644 --- a/apps/sim/lib/internal/onedrive/operations.ts +++ b/apps/sim/lib/internal/onedrive/operations.ts @@ -104,14 +104,14 @@ async function readGraphJson( async function graphRequest( url: string, - init: Parameters[1], + init: Omit[1], 'profile'>, label: string, signal?: AbortSignal ): Promise { signal?.throwIfAborted() return secureFetchWithValidation( url, - { ...init, maxResponseBytes: MAX_GRAPH_JSON_BYTES, signal }, + { ...init, profile: 'configuredEndpoint', maxResponseBytes: MAX_GRAPH_JSON_BYTES, signal }, label ) } @@ -427,12 +427,13 @@ async function fetchGraph( maxResponseBytes: number, signal?: AbortSignal ) { - const validation = await validateUrlWithDNS(url, label) + const validation = await validateUrlWithDNS(url, label, 'contentFetch') signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) { throw new OneDriveOperationError(validation.error || `Invalid ${label}`, 400) } return secureFetchWithPinnedIP(url, validation.resolvedIP, { + profile: 'contentFetch', headers: { Authorization: `Bearer ${accessToken}` }, maxResponseBytes, signal, diff --git a/apps/sim/lib/internal/onepassword/client.test.ts b/apps/sim/lib/internal/onepassword/client.test.ts index 1dda28e2e38..784b1357c34 100644 --- a/apps/sim/lib/internal/onepassword/client.test.ts +++ b/apps/sim/lib/internal/onepassword/client.test.ts @@ -151,7 +151,7 @@ describe('connectRequest', () => { 'Content-Type': 'application/json', }, body: '{"title":"Example"}', - allowHttp: true, + profile: 'configuredEndpoint', maxResponseBytes: 10 * 1024 * 1024, signal: controller.signal, }) diff --git a/apps/sim/lib/internal/onepassword/client.ts b/apps/sim/lib/internal/onepassword/client.ts index 8ac448c09a8..aed98c5b806 100644 --- a/apps/sim/lib/internal/onepassword/client.ts +++ b/apps/sim/lib/internal/onepassword/client.ts @@ -379,7 +379,7 @@ export async function connectRequest(options: { method: options.method, headers, body: options.body ? JSON.stringify(options.body) : undefined, - allowHttp: true, + profile: 'configuredEndpoint', maxResponseBytes: options.maxResponseBytes ?? MAX_JSON_API_RESPONSE_BYTES, signal: options.signal, }) diff --git a/apps/sim/lib/internal/pipedrive/client.ts b/apps/sim/lib/internal/pipedrive/client.ts index c4427e3f26c..8caad8443d7 100644 --- a/apps/sim/lib/internal/pipedrive/client.ts +++ b/apps/sim/lib/internal/pipedrive/client.ts @@ -48,12 +48,13 @@ export async function listPipedriveFiles( if (input.sort) url.searchParams.set('sort', input.sort) if (input.limit) url.searchParams.set('limit', input.limit) if (input.start) url.searchParams.set('start', input.start) - const validation = await validateUrlWithDNS(url.toString(), 'apiUrl') + const validation = await validateUrlWithDNS(url.toString(), 'apiUrl', 'configuredEndpoint') signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) { throw new PipedriveOperationError(validation.error || 'Invalid Pipedrive API URL', 400) } const response = await secureFetchWithPinnedIP(url.toString(), validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'GET', headers: getPipedriveAuthHeaders(input), maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, @@ -92,7 +93,7 @@ export async function downloadPipedriveFile( signal?: AbortSignal ): Promise<{ buffer: Buffer; contentType: string | null } | null> { signal?.throwIfAborted() - const validation = await validateUrlWithDNS(fileUrl, 'fileUrl') + const validation = await validateUrlWithDNS(fileUrl, 'fileUrl', 'contentFetch') signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) return null const authHeaders: Record = @@ -100,6 +101,7 @@ export async function downloadPipedriveFile( ? { 'x-api-token': input.accessToken } : { Authorization: `Bearer ${input.accessToken}` } const response = await secureFetchWithPinnedIP(fileUrl, validation.resolvedIP, { + profile: 'contentFetch', method: 'GET', headers: isPipedriveHost(fileUrl) ? authHeaders : {}, maxResponseBytes: maxBytes, diff --git a/apps/sim/lib/internal/pulse/client.ts b/apps/sim/lib/internal/pulse/client.ts index c2deac2b4a2..85f8d299ef7 100644 --- a/apps/sim/lib/internal/pulse/client.ts +++ b/apps/sim/lib/internal/pulse/client.ts @@ -19,7 +19,7 @@ export async function submitPulseParse( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const validation = await validateUrlWithDNS(PULSE_ENDPOINT, 'Pulse API URL') + const validation = await validateUrlWithDNS(PULSE_ENDPOINT, 'Pulse API URL', 'configuredEndpoint') signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) { throw new PulseOperationError(502, { success: false, error: 'Failed to reach Pulse API' }) @@ -30,6 +30,7 @@ export async function submitPulseParse( const body = Buffer.from(await payload.arrayBuffer()) signal?.throwIfAborted() const response = await secureFetchWithPinnedIP(PULSE_ENDPOINT, validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'POST', headers: { 'x-api-key': apiKey, 'Content-Type': contentType }, body, diff --git a/apps/sim/lib/internal/reducto/client.ts b/apps/sim/lib/internal/reducto/client.ts index b5a79fb7383..bf324b903bf 100644 --- a/apps/sim/lib/internal/reducto/client.ts +++ b/apps/sim/lib/internal/reducto/client.ts @@ -19,7 +19,11 @@ export async function submitReductoParse( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const validation = await validateUrlWithDNS(REDUCTO_ENDPOINT, 'Reducto API URL') + const validation = await validateUrlWithDNS( + REDUCTO_ENDPOINT, + 'Reducto API URL', + 'configuredEndpoint' + ) signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) { throw new ReductoOperationError(502, { @@ -29,6 +33,7 @@ export async function submitReductoParse( } const response = await secureFetchWithPinnedIP(REDUCTO_ENDPOINT, validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/lib/internal/sap-concur/client.ts b/apps/sim/lib/internal/sap-concur/client.ts index 31c436e85fc..bb77379101a 100644 --- a/apps/sim/lib/internal/sap-concur/client.ts +++ b/apps/sim/lib/internal/sap-concur/client.ts @@ -307,6 +307,7 @@ async function requestAccessToken( const response = await secureFetchWithValidation( tokenUrl, { + profile: 'configuredEndpoint', method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -426,6 +427,7 @@ export async function invokeSapConcur( const response = await secureFetchWithValidation( url, { + profile: 'configuredEndpoint', method: input.method, headers, body: hasBody @@ -498,6 +500,7 @@ export async function invokeSapConcurMultipart( const response = await secureFetchWithValidation( url, { + profile: 'configuredEndpoint', method: 'POST', headers, body: bodyBuffer, diff --git a/apps/sim/lib/internal/sap-s4hana/client.ts b/apps/sim/lib/internal/sap-s4hana/client.ts index f4bd96fc4c3..2cce17bb257 100644 --- a/apps/sim/lib/internal/sap-s4hana/client.ts +++ b/apps/sim/lib/internal/sap-s4hana/client.ts @@ -73,6 +73,7 @@ export async function fetchSapAccessToken( const response = await secureFetchWithValidation( tokenUrl, { + profile: 'configuredEndpoint', method: 'POST', headers: { Authorization: `Basic ${basic}`, @@ -164,6 +165,7 @@ export async function fetchSapCsrf( const response = await secureFetchWithValidation( buildOdataUrl(input, '/$metadata'), { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: buildAuthHeader(input, accessToken), @@ -210,6 +212,7 @@ export async function callSapOdata( const response = await secureFetchWithValidation( buildOdataUrl(input), { + profile: 'configuredEndpoint', method: input.method, headers, body: hasBody ? JSON.stringify(input.body) : undefined, diff --git a/apps/sim/lib/internal/servicenow/client.ts b/apps/sim/lib/internal/servicenow/client.ts index e880dc9ee7a..415afc0db90 100644 --- a/apps/sim/lib/internal/servicenow/client.ts +++ b/apps/sim/lib/internal/servicenow/client.ts @@ -30,6 +30,7 @@ export async function uploadServiceNowAttachment( const response = await secureFetchWithValidation( `${baseUrl}/api/now/attachment/file?${params.toString()}`, { + profile: 'configuredEndpoint', method: 'POST', headers: { Authorization: createBasicAuthHeader(input.username, input.password), diff --git a/apps/sim/lib/internal/sharepoint/client.test.ts b/apps/sim/lib/internal/sharepoint/client.test.ts index 98f255b1481..d9c0a6cb7ca 100644 --- a/apps/sim/lib/internal/sharepoint/client.test.ts +++ b/apps/sim/lib/internal/sharepoint/client.test.ts @@ -42,6 +42,7 @@ describe('SharePointClient', () => { { headers: { Authorization: 'Bearer token' }, stripAuthOnRedirect: true, + profile: 'configuredEndpoint', maxResponseBytes: MAX_FILE_SIZE, signal: controller.signal, } @@ -63,6 +64,7 @@ describe('SharePointClient', () => { expect(mocks.validatedFetch).toHaveBeenCalledWith( 'https://graph.microsoft.com/upload', { + profile: 'contentFetch', method: 'PUT', headers: { Authorization: 'Bearer token', diff --git a/apps/sim/lib/internal/sharepoint/client.ts b/apps/sim/lib/internal/sharepoint/client.ts index 2697463f462..fc2016c6902 100644 --- a/apps/sim/lib/internal/sharepoint/client.ts +++ b/apps/sim/lib/internal/sharepoint/client.ts @@ -94,19 +94,21 @@ export class SharePointClient { return `Bearer ${this.accessToken}` } + /** Every URL reaching here is a fixed Microsoft Graph endpoint. */ private async pinnedFetch( url: string, paramName: string, - options: SecureFetchOptions + options: Omit ): Promise { this.signal?.throwIfAborted() - const validation = await validateUrlWithDNS(url, paramName) + const validation = await validateUrlWithDNS(url, paramName, 'configuredEndpoint') this.signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) { throw new SharePointGraphError(validation.error || `Invalid ${paramName}`, 400) } return secureFetchWithPinnedIP(url, validation.resolvedIP, { ...options, + profile: 'configuredEndpoint', signal: this.signal, }) } @@ -163,6 +165,7 @@ export class SharePointClient { const response = await secureFetchWithValidation( url, { + profile: 'contentFetch', method: 'PUT', headers: { Authorization: this.authorization, diff --git a/apps/sim/lib/internal/slack/operations.test.ts b/apps/sim/lib/internal/slack/operations.test.ts index 7d97783f171..8f269c2f2ae 100644 --- a/apps/sim/lib/internal/slack/operations.test.ts +++ b/apps/sim/lib/internal/slack/operations.test.ts @@ -229,6 +229,7 @@ describe('Slack operations', () => { '93.184.216.34', { headers: { Authorization: 'Bearer token' }, + profile: 'contentFetch', maxResponseBytes: MAX_FILE_SIZE, signal: controller.signal, } diff --git a/apps/sim/lib/internal/slack/operations.ts b/apps/sim/lib/internal/slack/operations.ts index b3649cdfdc4..831c1e092cc 100644 --- a/apps/sim/lib/internal/slack/operations.ts +++ b/apps/sim/lib/internal/slack/operations.ts @@ -325,6 +325,7 @@ async function uploadSlackFiles( const uploaded = await secureFetchWithValidation( uploadUrl, { + profile: 'contentFetch', method: 'POST', body: file.buffer, maxResponseBytes: 64 * 1024, @@ -444,10 +445,11 @@ export async function executeSlackDownload(input: SlackDownloadBody, signal?: Ab const urlPrivate = slackString(file, 'url_private') if (!urlPrivate) failure(400, 'File does not have a download URL') const downloadUrl = urlPrivate - const validation = await validateUrlWithDNS(downloadUrl, 'urlPrivate') + const validation = await validateUrlWithDNS(downloadUrl, 'urlPrivate', 'contentFetch') signal?.throwIfAborted() if (!validation.isValid) failure(400, validation.error || 'Invalid Slack file URL') const response = await secureFetchWithPinnedIP(downloadUrl, validation.resolvedIP!, { + profile: 'contentFetch', headers: { Authorization: `Bearer ${input.accessToken}` }, maxResponseBytes: MAX_FILE_SIZE, signal, diff --git a/apps/sim/lib/internal/stagehand/operations.ts b/apps/sim/lib/internal/stagehand/operations.ts index e12f9f5f66d..7cafab86303 100644 --- a/apps/sim/lib/internal/stagehand/operations.ts +++ b/apps/sim/lib/internal/stagehand/operations.ts @@ -130,7 +130,7 @@ export async function executeStagehandAgent( try { const startUrl = normalizeStagehandUrl(input.startUrl) - const urlValidation = await validateUrlWithDNS(startUrl, 'startUrl') + const urlValidation = await validateUrlWithDNS(startUrl, 'startUrl', 'requestTarget') context.signal?.throwIfAborted() if (!urlValidation.isValid) { return Response.json({ error: urlValidation.error }, { status: 400 }) @@ -251,7 +251,7 @@ export async function executeStagehandExtract( try { const url = normalizeStagehandUrl(input.url) - const urlValidation = await validateUrlWithDNS(url, 'url') + const urlValidation = await validateUrlWithDNS(url, 'url', 'requestTarget') context.signal?.throwIfAborted() if (!urlValidation.isValid) { return Response.json({ error: urlValidation.error }, { status: 400 }) diff --git a/apps/sim/lib/internal/stt/operations.ts b/apps/sim/lib/internal/stt/operations.ts index fb319967e94..4436dfb04d9 100644 --- a/apps/sim/lib/internal/stt/operations.ts +++ b/apps/sim/lib/internal/stt/operations.ts @@ -255,12 +255,13 @@ export async function executeSttOperation( } } - const urlValidation = await validateUrlWithDNS(audioUrl, 'audioUrl') + const urlValidation = await validateUrlWithDNS(audioUrl, 'audioUrl', 'contentFetch') if (!urlValidation.isValid) { return Response.json({ error: urlValidation.error }, { status: 400 }) } const response = await secureFetchWithPinnedIP(audioUrl, urlValidation.resolvedIP!, { + profile: 'contentFetch', method: 'GET', maxResponseBytes: MAX_FILE_SIZE, signal, diff --git a/apps/sim/lib/internal/textract/document-input.ts b/apps/sim/lib/internal/textract/document-input.ts index ebd82384cc4..f683f142dd1 100644 --- a/apps/sim/lib/internal/textract/document-input.ts +++ b/apps/sim/lib/internal/textract/document-input.ts @@ -41,12 +41,13 @@ async function fetchDocumentBytes( signal?: AbortSignal ): Promise<{ bytes: Buffer; contentType: string }> { signal?.throwIfAborted() - const urlValidation = await validateUrlWithDNS(url, 'Document URL') + const urlValidation = await validateUrlWithDNS(url, 'Document URL', 'contentFetch') if (!urlValidation.isValid) { throw new TextractOperationError(urlValidation.error || 'Invalid document URL', 400) } const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP!, { + profile: 'contentFetch', method: 'GET', signal, }) @@ -159,7 +160,7 @@ export async function resolveDocumentInput( ), } } else { - const urlValidation = await validateUrlWithDNS(fileUrl, 'Document URL') + const urlValidation = await validateUrlWithDNS(fileUrl, 'Document URL', 'contentFetch') if (!urlValidation.isValid) { logger.warn(`[${requestId}] SSRF attempt blocked`, { userId, diff --git a/apps/sim/lib/internal/twilio-voice/operations.ts b/apps/sim/lib/internal/twilio-voice/operations.ts index a94793bcf1c..827808bd7aa 100644 --- a/apps/sim/lib/internal/twilio-voice/operations.ts +++ b/apps/sim/lib/internal/twilio-voice/operations.ts @@ -50,12 +50,13 @@ async function fetchPinned( context: TwilioVoiceOperationContext, maxResponseBytes: number ) { - const validation = await validateUrlWithDNS(url, label) + const validation = await validateUrlWithDNS(url, label, 'configuredEndpoint') context.signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) { throw new TwilioVoiceOperationError(validation.error || `Invalid ${label}`, 400) } return secureFetchWithPinnedIP(url, validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: authHeader }, maxResponseBytes, diff --git a/apps/sim/lib/internal/typeform/operations.ts b/apps/sim/lib/internal/typeform/operations.ts index 32ade526aeb..9f5a7b24a19 100644 --- a/apps/sim/lib/internal/typeform/operations.ts +++ b/apps/sim/lib/internal/typeform/operations.ts @@ -44,12 +44,13 @@ export async function downloadTypeformFile( ): Promise { context.signal?.throwIfAborted() const fileUrl = buildTypeformFileUrl(input) - const validation = await validateUrlWithDNS(fileUrl, 'typeformFileUrl') + const validation = await validateUrlWithDNS(fileUrl, 'typeformFileUrl', 'configuredEndpoint') context.signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) { throw new TypeformOperationError(validation.error || 'Invalid Typeform file URL', 400) } const response = await secureFetchWithPinnedIP(fileUrl, validation.resolvedIP, { + profile: 'configuredEndpoint', headers: { Authorization: `Bearer ${input.apiKey}` }, maxResponseBytes: MAX_TYPEFORM_FILE_BYTES, signal: context.signal, diff --git a/apps/sim/lib/internal/vision/client.test.ts b/apps/sim/lib/internal/vision/client.test.ts index 78d52407f0f..d88a147695b 100644 --- a/apps/sim/lib/internal/vision/client.test.ts +++ b/apps/sim/lib/internal/vision/client.test.ts @@ -159,6 +159,7 @@ describe('Vision client', () => { 'https://images.example.com/a.png', '203.0.113.10', { + profile: 'contentFetch', method: 'GET', maxResponseBytes: MAX_BUFFERED_TRANSFER_BYTES, signal: controller.signal, diff --git a/apps/sim/lib/internal/vision/client.ts b/apps/sim/lib/internal/vision/client.ts index 7ff0a164d28..6c0b79088a2 100644 --- a/apps/sim/lib/internal/vision/client.ts +++ b/apps/sim/lib/internal/vision/client.ts @@ -94,6 +94,7 @@ async function fetchGeminiImage(input: VisionClientInput, signal?: AbortSignal): } const response = await secureFetchWithPinnedIP(input.imageSource, input.remoteImageResolvedIP, { + profile: 'contentFetch', method: 'GET', maxResponseBytes: MAX_BUFFERED_TRANSFER_BYTES, signal, diff --git a/apps/sim/lib/internal/vision/operations.test.ts b/apps/sim/lib/internal/vision/operations.test.ts index 8f8f6a9280e..1be0b594029 100644 --- a/apps/sim/lib/internal/vision/operations.test.ts +++ b/apps/sim/lib/internal/vision/operations.test.ts @@ -218,7 +218,8 @@ describe('Vision operations', () => { ) expect(mocks.validateUrlWithDNS).toHaveBeenCalledWith( 'https://storage.example.com/image.png', - 'imageUrl' + 'imageUrl', + 'contentFetch' ) expect(mocks.analyzeVision).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/apps/sim/lib/internal/vision/operations.ts b/apps/sim/lib/internal/vision/operations.ts index 7c4c219d0d3..de2b744be3c 100644 --- a/apps/sim/lib/internal/vision/operations.ts +++ b/apps/sim/lib/internal/vision/operations.ts @@ -101,7 +101,7 @@ async function resolveUrlImage( } context.signal?.throwIfAborted() - const validation = await validateUrlWithDNS(source, 'imageUrl') + const validation = await validateUrlWithDNS(source, 'imageUrl', 'contentFetch') context.signal?.throwIfAborted() if (!validation.isValid) { fail(validation.error || 'Invalid image URL', 400, { diff --git a/apps/sim/lib/internal/whatsapp/operations.ts b/apps/sim/lib/internal/whatsapp/operations.ts index 1af0d6f6042..bddd9b90e7b 100644 --- a/apps/sim/lib/internal/whatsapp/operations.ts +++ b/apps/sim/lib/internal/whatsapp/operations.ts @@ -221,11 +221,12 @@ export async function executeWhatsAppGetMedia( ) } - const urlValidation = await validateUrlWithDNS(metadata.url, 'mediaUrl') + const urlValidation = await validateUrlWithDNS(metadata.url, 'mediaUrl', 'contentFetch') if (!urlValidation.isValid) { return failureResponse(`Invalid WhatsApp media URL: ${urlValidation.error}`, 502) } const mediaResponse = await secureFetchWithPinnedIP(metadata.url, urlValidation.resolvedIP!, { + profile: 'contentFetch', method: 'GET', headers: { Authorization: authorization, 'User-Agent': DOWNLOAD_USER_AGENT }, maxResponseBytes: WHATSAPP_MEDIA_MAX_BYTES, diff --git a/apps/sim/lib/internal/windchill/client.ts b/apps/sim/lib/internal/windchill/client.ts index dc5f43bf60b..cfa9adaf3fc 100644 --- a/apps/sim/lib/internal/windchill/client.ts +++ b/apps/sim/lib/internal/windchill/client.ts @@ -116,6 +116,7 @@ export async function createWindchillSession( const response = await secureFetchWithValidation( `${ptcRoot(params.baseUrl)}/PTC/GetCSRFToken()`, { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: createBasicAuthHeader(params.username, params.password), @@ -172,6 +173,7 @@ export async function windchillMutationRequest({ const response = await secureFetchWithValidation( url, { + profile: 'configuredEndpoint', method, headers, body: body === undefined ? undefined : JSON.stringify(body), @@ -318,6 +320,7 @@ export async function uploadWindchillContent({ const stageTwoResponse = await secureFetchWithValidation( descriptor.replicaUrl, { + profile: 'configuredEndpoint', method: 'POST', headers: { 'Content-Type': multipart.contentType, Accept: 'application/json' }, body: multipart.body, @@ -377,6 +380,7 @@ export async function resolveWindchillContentUrl({ const response = await secureFetchWithValidation( `${contentPath}/PTC.ApplicationData/Content/URL`, { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: createBasicAuthHeader(params.username, params.password), @@ -435,6 +439,7 @@ export async function downloadWindchillContent({ const response = await secureFetchWithValidation( url, { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: createBasicAuthHeader(params.username, params.password) }, stripAuthOnRedirect: true, diff --git a/apps/sim/lib/internal/zoho-desk/operations.ts b/apps/sim/lib/internal/zoho-desk/operations.ts index 21279980c70..8cb3f45a30e 100644 --- a/apps/sim/lib/internal/zoho-desk/operations.ts +++ b/apps/sim/lib/internal/zoho-desk/operations.ts @@ -37,6 +37,7 @@ export async function getZohoDeskAttachment( } const response = await secureFetchWithValidation(downloadUrl.toString(), { + profile: 'contentFetch', method: 'GET', headers: buildZohoDeskHeaders({ accessToken: input.accessToken, orgId: input.orgId }), timeout: 30_000, diff --git a/apps/sim/lib/internal/zoom/operations.ts b/apps/sim/lib/internal/zoom/operations.ts index 9bf64c3e8ff..1068f38c779 100644 --- a/apps/sim/lib/internal/zoom/operations.ts +++ b/apps/sim/lib/internal/zoom/operations.ts @@ -67,13 +67,14 @@ export async function getZoomMeetingRecordings( if (input.ttl) query.set('ttl', String(input.ttl)) const baseUrl = `https://api.zoom.us/v2/meetings/${encodeURIComponent(input.meetingId)}/recordings` const apiUrl = query.size > 0 ? `${baseUrl}?${query}` : baseUrl - const validation = await validateUrlWithDNS(apiUrl, 'apiUrl') + const validation = await validateUrlWithDNS(apiUrl, 'apiUrl', 'configuredEndpoint') context.signal?.throwIfAborted() if (!validation.isValid || !validation.resolvedIP) { throw new ZoomOperationError(validation.error || 'Invalid Zoom API URL', 400) } const response = await secureFetchWithPinnedIP(apiUrl, validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'GET', headers: { 'Content-Type': 'application/json', @@ -101,12 +102,17 @@ export async function getZoomMeetingRecordings( 413 ) } - const fileValidation = await validateUrlWithDNS(file.download_url, 'downloadUrl') + const fileValidation = await validateUrlWithDNS( + file.download_url, + 'downloadUrl', + 'contentFetch' + ) if (!fileValidation.isValid || !fileValidation.resolvedIP) continue const downloadResponse = await secureFetchWithPinnedIP( file.download_url, fileValidation.resolvedIP, { + profile: 'contentFetch', method: 'GET', headers: { Authorization: `Bearer ${input.accessToken}` }, maxResponseBytes: remainingBytes, diff --git a/apps/sim/lib/internal/zoominfo/client.ts b/apps/sim/lib/internal/zoominfo/client.ts index b908cfce267..d98e19ea868 100644 --- a/apps/sim/lib/internal/zoominfo/client.ts +++ b/apps/sim/lib/internal/zoominfo/client.ts @@ -74,6 +74,7 @@ async function fetchAccessToken( const response = await secureFetchWithValidation( tokenUrl, { + profile: 'configuredEndpoint', method: 'POST', headers: { Authorization: `Basic ${basic}`, @@ -163,6 +164,7 @@ async function invokeZoomInfo( const response = await secureFetchWithValidation( url, { + profile: 'configuredEndpoint', method: input.method, headers, body: hasBody diff --git a/apps/sim/lib/knowledge/documents/secure-fetch.server.ts b/apps/sim/lib/knowledge/documents/secure-fetch.server.ts index 818174760af..24cd4814dc8 100644 --- a/apps/sim/lib/knowledge/documents/secure-fetch.server.ts +++ b/apps/sim/lib/knowledge/documents/secure-fetch.server.ts @@ -11,7 +11,6 @@ import { } from '@/lib/knowledge/documents/utils' export interface SecureFetchRetryOptions extends RetryOptions { - allowHttp?: boolean timeout?: number maxResponseBytes?: number } @@ -28,16 +27,15 @@ export interface SecureFetchRetryOptions extends RetryOptions { */ export async function secureFetchWithRetry( url: string, - options: SecureFetchOptions = {}, + options: SecureFetchOptions, retryOptions: SecureFetchRetryOptions = {} ): Promise { - const { allowHttp, timeout, maxResponseBytes, ...retry } = retryOptions + const { timeout, maxResponseBytes, ...retry } = retryOptions return retryWithExponentialBackoff(async () => { const response = await secureFetchWithValidation( url, { ...options, - ...(allowHttp !== undefined ? { allowHttp } : {}), ...(timeout !== undefined ? { timeout } : {}), ...(maxResponseBytes !== undefined ? { maxResponseBytes } : {}), }, diff --git a/apps/sim/lib/knowledge/documents/utils.test.ts b/apps/sim/lib/knowledge/documents/utils.test.ts index 03cf245c8c1..eda1f582b90 100644 --- a/apps/sim/lib/knowledge/documents/utils.test.ts +++ b/apps/sim/lib/knowledge/documents/utils.test.ts @@ -836,17 +836,21 @@ describe('secureFetchWithRetry', () => { expect(mockSecureFetchWithValidation).toHaveBeenCalledTimes(1) }) - it('forwards allowHttp / timeout / maxResponseBytes to the pinned fetch', async () => { + it('forwards the egress profile, timeout and maxResponseBytes to the pinned fetch', async () => { mockSecureFetchWithValidation.mockResolvedValue(fakeResponse(200)) await secureFetchWithRetry( 'http://localhost:9000', - { method: 'GET' }, - { allowHttp: true, timeout: 5000, maxResponseBytes: 1024, ...FAST_RETRY } + { method: 'GET', profile: 'configuredEndpoint' }, + { timeout: 5000, maxResponseBytes: 1024, ...FAST_RETRY } ) const [, options] = mockSecureFetchWithValidation.mock.calls[0] - expect(options).toMatchObject({ allowHttp: true, timeout: 5000, maxResponseBytes: 1024 }) + expect(options).toMatchObject({ + profile: 'configuredEndpoint', + timeout: 5000, + maxResponseBytes: 1024, + }) }) /** diff --git a/apps/sim/lib/media/falai.ts b/apps/sim/lib/media/falai.ts index 21a36dfbad9..8ab2f35077c 100644 --- a/apps/sim/lib/media/falai.ts +++ b/apps/sim/lib/media/falai.ts @@ -202,12 +202,13 @@ export async function downloadFalMedia( return { contentType: match[1], buffer } } - const validation = await validateUrlWithDNS(url, 'mediaUrl') + const validation = await validateUrlWithDNS(url, 'mediaUrl', 'contentFetch') if (!validation.isValid || !validation.resolvedIP) { throw new Error(validation.error || 'Generated media URL failed validation') } const response = await secureFetchWithPinnedIP(url, validation.resolvedIP, { + profile: 'contentFetch', method: 'GET', maxResponseBytes: MAX_MEDIA_BYTES, }) diff --git a/apps/sim/lib/uploads/contexts/workspace/fetch-external-url.ts b/apps/sim/lib/uploads/contexts/workspace/fetch-external-url.ts index 3a0c495f59d..f411b8f684c 100644 --- a/apps/sim/lib/uploads/contexts/workspace/fetch-external-url.ts +++ b/apps/sim/lib/uploads/contexts/workspace/fetch-external-url.ts @@ -87,7 +87,7 @@ export async function fetchExternalUrlToWorkspace( timeoutMs = DEFAULT_TIMEOUT_MS, } = options - const urlValidation = await validateUrlWithDNS(url, 'fileUrl') + const urlValidation = await validateUrlWithDNS(url, 'fileUrl', 'contentFetch') if (!urlValidation.isValid || !urlValidation.resolvedIP) { throw new ExternalUrlValidationError(urlValidation.error || 'Invalid external URL') } @@ -96,6 +96,7 @@ export async function fetchExternalUrlToWorkspace( const extension = path.extname(filename).toLowerCase().substring(1) const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP, { + profile: 'contentFetch', timeout: timeoutMs, maxResponseBytes: maxDownloadBytes, signal, diff --git a/apps/sim/lib/uploads/utils/file-utils.server.ts b/apps/sim/lib/uploads/utils/file-utils.server.ts index faa41bf5bfb..47a1489dc7f 100644 --- a/apps/sim/lib/uploads/utils/file-utils.server.ts +++ b/apps/sim/lib/uploads/utils/file-utils.server.ts @@ -199,7 +199,7 @@ export async function resolveFileInputToUrl( }, } } else { - const urlValidation = await validateUrlWithDNS(fileUrl, 'filePath') + const urlValidation = await validateUrlWithDNS(fileUrl, 'filePath', 'contentFetch') if (!urlValidation.isValid) { return { error: { status: 400, message: urlValidation.error || 'Invalid URL' } } } @@ -276,12 +276,13 @@ export async function downloadFileFromUrl( return downloadFile({ key, context, maxBytes, signal }) } - const urlValidation = await validateUrlWithDNS(fileUrl, 'fileUrl') + const urlValidation = await validateUrlWithDNS(fileUrl, 'fileUrl', 'contentFetch') if (!urlValidation.isValid) { throw new Error(`Invalid file URL: ${urlValidation.error}`) } const response = await secureFetchWithPinnedIP(fileUrl, urlValidation.resolvedIP!, { + profile: 'contentFetch', timeout: timeoutMs, maxResponseBytes: maxBytes, signal, diff --git a/apps/sim/lib/webhooks/polling/rss.ts b/apps/sim/lib/webhooks/polling/rss.ts index 662eaf6e9a9..fada0deddf3 100644 --- a/apps/sim/lib/webhooks/polling/rss.ts +++ b/apps/sim/lib/webhooks/polling/rss.ts @@ -199,7 +199,7 @@ async function fetchNewRssItems( logger: Logger ): Promise<{ feed: RssFeed; items: RssItem[]; etag?: string; lastModified?: string }> { try { - const urlValidation = await validateUrlWithDNS(config.feedUrl, 'feedUrl') + const urlValidation = await validateUrlWithDNS(config.feedUrl, 'feedUrl', 'requestTarget') if (!urlValidation.isValid) { logger.error(`[${requestId}] Invalid RSS feed URL: ${urlValidation.error}`) throw new Error(`Invalid RSS feed URL: ${urlValidation.error}`) @@ -217,6 +217,7 @@ async function fetchNewRssItems( } const response = await secureFetchWithPinnedIP(config.feedUrl, urlValidation.resolvedIP!, { + profile: 'requestTarget', headers, timeout: 30000, maxResponseBytes: MAX_RSS_FEED_BYTES, diff --git a/apps/sim/lib/webhooks/providers/emailbison.ts b/apps/sim/lib/webhooks/providers/emailbison.ts index ee4d1ba1b31..d77dc680ebb 100644 --- a/apps/sim/lib/webhooks/providers/emailbison.ts +++ b/apps/sim/lib/webhooks/providers/emailbison.ts @@ -153,13 +153,14 @@ export const emailBisonHandler: WebhookProviderHandler = { }) const targetUrl = emailBisonUrl('/api/webhook-url', {}, apiBaseUrl) - const urlValidation = await validateUrlWithDNS(targetUrl, 'apiBaseUrl') + const urlValidation = await validateUrlWithDNS(targetUrl, 'apiBaseUrl', 'configuredEndpoint') if (!urlValidation.isValid) { logger.warn(`[${requestId}] Invalid Email Bison Instance URL: ${urlValidation.error}`) throw new Error('Email Bison Instance URL could not be validated.') } const response = await secureFetchWithPinnedIP(targetUrl, urlValidation.resolvedIP!, { + profile: 'configuredEndpoint', method: 'POST', headers: emailBisonHeaders({ apiKey, apiBaseUrl }), body: JSON.stringify({ @@ -229,7 +230,7 @@ export const emailBisonHandler: WebhookProviderHandler = { {}, apiBaseUrl ) - const urlValidation = await validateUrlWithDNS(targetUrl, 'apiBaseUrl') + const urlValidation = await validateUrlWithDNS(targetUrl, 'apiBaseUrl', 'configuredEndpoint') if (!urlValidation.isValid) { logger.warn(`[${requestId}] Invalid Email Bison Instance URL: ${urlValidation.error}`, { webhookId: webhook.id, @@ -240,6 +241,7 @@ export const emailBisonHandler: WebhookProviderHandler = { } const response = await secureFetchWithPinnedIP(targetUrl, urlValidation.resolvedIP!, { + profile: 'configuredEndpoint', method: 'DELETE', headers: emailBisonHeaders({ apiKey, apiBaseUrl }), }) diff --git a/apps/sim/lib/webhooks/providers/gitlab.ts b/apps/sim/lib/webhooks/providers/gitlab.ts index 6b5f4cd8d1b..41d71d758e1 100644 --- a/apps/sim/lib/webhooks/providers/gitlab.ts +++ b/apps/sim/lib/webhooks/providers/gitlab.ts @@ -34,6 +34,7 @@ async function cleanupGitLabHookByUrl( host: unknown ): Promise { const res = await secureFetchWithValidation(gitlabProjectHooksUrl(projectId, host), { + profile: 'configuredEndpoint', headers: { 'PRIVATE-TOKEN': accessToken }, }).catch(() => null) if (!res || !res.ok) return @@ -46,6 +47,7 @@ async function cleanupGitLabHookByUrl( .filter((hook) => hook.url === url && hook.id != null) .map((hook) => secureFetchWithValidation(`${gitlabProjectHooksUrl(projectId, host)}/${hook.id}`, { + profile: 'configuredEndpoint', method: 'DELETE', headers: { 'PRIVATE-TOKEN': accessToken }, }).catch(() => null) @@ -198,6 +200,7 @@ export const gitlabHandler: WebhookProviderHandler = { const { getGitLabEventFlags } = await import('@/triggers/gitlab/utils') const secretToken = generateId() const res = await secureFetchWithValidation(gitlabProjectHooksUrl(projectId, host), { + profile: 'configuredEndpoint', method: 'POST', headers: { 'PRIVATE-TOKEN': accessToken, 'Content-Type': 'application/json' }, body: JSON.stringify({ @@ -269,10 +272,7 @@ export const gitlabHandler: WebhookProviderHandler = { const res = await secureFetchWithValidation( `${gitlabProjectHooksUrl(projectId, host)}/${externalId}`, - { - method: 'DELETE', - headers: { 'PRIVATE-TOKEN': accessToken }, - } + { profile: 'configuredEndpoint', method: 'DELETE', headers: { 'PRIVATE-TOKEN': accessToken } } ) if (!res.ok && res.status !== 404) { diff --git a/apps/sim/lib/webhooks/providers/microsoft-teams.ts b/apps/sim/lib/webhooks/providers/microsoft-teams.ts index 4e35dba8467..47d5bd9cf5d 100644 --- a/apps/sim/lib/webhooks/providers/microsoft-teams.ts +++ b/apps/sim/lib/webhooks/providers/microsoft-teams.ts @@ -94,7 +94,7 @@ async function fetchWithDNSPinning( requestId: string ): Promise { try { - const urlValidation = await validateUrlWithDNS(url, 'contentUrl') + const urlValidation = await validateUrlWithDNS(url, 'contentUrl', 'contentFetch') if (!urlValidation.isValid) { logger.warn(`[${requestId}] Invalid content URL: ${urlValidation.error}`, { url }) return null @@ -103,7 +103,10 @@ async function fetchWithDNSPinning( if (accessToken) { headers.Authorization = `Bearer ${accessToken}` } - const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP!, { headers }) + const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP!, { + profile: 'contentFetch', + headers, + }) return response } catch (error) { logger.error(`[${requestId}] Error fetching URL with DNS pinning`, { diff --git a/apps/sim/lib/webhooks/providers/slack.ts b/apps/sim/lib/webhooks/providers/slack.ts index ba9e20b93ef..a320602c6db 100644 --- a/apps/sim/lib/webhooks/providers/slack.ts +++ b/apps/sim/lib/webhooks/providers/slack.ts @@ -341,7 +341,7 @@ async function downloadSlackFiles( } try { - const urlValidation = await validateUrlWithDNS(urlPrivate, 'url_private') + const urlValidation = await validateUrlWithDNS(urlPrivate, 'url_private', 'contentFetch') if (!urlValidation.isValid) { logger.warn('Slack file url_private failed DNS validation, skipping', { fileId: f.id, @@ -351,6 +351,7 @@ async function downloadSlackFiles( } const response = await secureFetchWithPinnedIP(urlPrivate, urlValidation.resolvedIP!, { + profile: 'contentFetch', headers: { Authorization: `Bearer ${botToken}` }, }) diff --git a/apps/sim/providers/azure-anthropic/index.test.ts b/apps/sim/providers/azure-anthropic/index.test.ts index 6ca390ca2b9..46d9122a9d7 100644 --- a/apps/sim/providers/azure-anthropic/index.test.ts +++ b/apps/sim/providers/azure-anthropic/index.test.ts @@ -77,7 +77,11 @@ describe('azureAnthropicProvider — SSRF pinning', () => { request({ azureEndpoint: 'https://rebind.attacker.tld' }) ) - expect(mockValidate).toHaveBeenCalledWith('https://rebind.attacker.tld', 'azureEndpoint') + expect(mockValidate).toHaveBeenCalledWith( + 'https://rebind.attacker.tld', + 'azureEndpoint', + 'configuredEndpoint' + ) expect(mockCreatePinnedFetch).toHaveBeenCalledWith('203.0.113.10') expect(buildClientOptions()).toMatchObject({ fetch: sentinelFetch }) }) diff --git a/apps/sim/providers/azure-anthropic/index.ts b/apps/sim/providers/azure-anthropic/index.ts index fe7881755db..78ded084fc0 100644 --- a/apps/sim/providers/azure-anthropic/index.ts +++ b/apps/sim/providers/azure-anthropic/index.ts @@ -32,7 +32,11 @@ export const azureAnthropicProvider: ProviderConfig = { let pinnedFetch: typeof fetch | undefined let pinnedIP: string | undefined if (userProvidedEndpoint) { - const validation = await validateUrlWithDNS(userProvidedEndpoint, 'azureEndpoint') + const validation = await validateUrlWithDNS( + userProvidedEndpoint, + 'azureEndpoint', + 'configuredEndpoint' + ) if (!validation.isValid) { logger.warn('Blocked SSRF attempt via azureEndpoint', { endpoint: userProvidedEndpoint, diff --git a/apps/sim/providers/azure-openai/index.test.ts b/apps/sim/providers/azure-openai/index.test.ts index 5d82e8d6ed5..98e4099b0a3 100644 --- a/apps/sim/providers/azure-openai/index.test.ts +++ b/apps/sim/providers/azure-openai/index.test.ts @@ -142,7 +142,11 @@ describe('azureOpenAIProvider — SSRF pinning', () => { request({ azureEndpoint: 'https://rebind.attacker.tld' }) ) - expect(mockValidate).toHaveBeenCalledWith('https://rebind.attacker.tld', 'azureEndpoint') + expect(mockValidate).toHaveBeenCalledWith( + 'https://rebind.attacker.tld', + 'azureEndpoint', + 'configuredEndpoint' + ) expect(mockCreatePinnedFetch).toHaveBeenCalledWith('203.0.113.10') expect(responsesConfig().fetch).toBe(sentinelFetch) }) diff --git a/apps/sim/providers/azure-openai/index.ts b/apps/sim/providers/azure-openai/index.ts index cdef124d348..1428c62815e 100644 --- a/apps/sim/providers/azure-openai/index.ts +++ b/apps/sim/providers/azure-openai/index.ts @@ -674,7 +674,11 @@ export const azureOpenAIProvider: ProviderConfig = { let pinnedFetch: typeof fetch | undefined if (userProvidedEndpoint) { - const validation = await validateUrlWithDNS(userProvidedEndpoint, 'azureEndpoint') + const validation = await validateUrlWithDNS( + userProvidedEndpoint, + 'azureEndpoint', + 'configuredEndpoint' + ) if (!validation.isValid) { logger.warn('Blocked SSRF attempt via azureEndpoint', { endpoint: userProvidedEndpoint, diff --git a/apps/sim/providers/vllm/index.test.ts b/apps/sim/providers/vllm/index.test.ts index 07162d5e335..9c6da6fc379 100644 --- a/apps/sim/providers/vllm/index.test.ts +++ b/apps/sim/providers/vllm/index.test.ts @@ -189,7 +189,7 @@ describe('vllmProvider', () => { expect(mockValidateUrlWithDNS).toHaveBeenCalledWith( 'https://my-vllm.example.com', 'vLLM endpoint', - { allowHttp: true } + 'configuredEndpoint' ) expect(mockCreatePinnedFetch).toHaveBeenCalledWith('203.0.113.10') expect(openAIArgs[0].baseURL).toBe('https://my-vllm.example.com/v1') @@ -208,7 +208,7 @@ describe('vllmProvider', () => { expect(mockValidateUrlWithDNS).toHaveBeenCalledWith( 'https://my-vllm.example.com/v1', 'vLLM endpoint', - { allowHttp: true } + 'configuredEndpoint' ) expect(openAIArgs[0].baseURL).toBe('https://my-vllm.example.com/v1') expect(openAIArgs[0].fetch).toBe(pinnedFetchFn) diff --git a/apps/sim/providers/vllm/index.ts b/apps/sim/providers/vllm/index.ts index e2fb433403a..db5241fa95c 100644 --- a/apps/sim/providers/vllm/index.ts +++ b/apps/sim/providers/vllm/index.ts @@ -115,20 +115,23 @@ export const vllmProvider: ProviderConfig = { /** * A user-supplied endpoint is attacker-controlled: validate it against the - * central SSRF guard and pin the connection to the resolved IP to defeat DNS + * egress guard and pin the connection to the resolved IP to defeat DNS * rebinding. The operator-configured `VLLM_BASE_URL` is trusted and left * unvalidated, mirroring the Azure providers. * - * `allowHttp` is enabled because self-hosted vLLM is frequently served over - * plain HTTP; this only relaxes the protocol requirement — the private/reserved - * IP blocklist and blocked-port checks still apply, so SSRF protection is intact. + * The `configuredEndpoint` profile is what makes a self-hosted vLLM reachable + * at all: over plain HTTP, which these deployments usually are, and at a + * private address once the operator names it in the egress allowlist. + * Anything they have not named stays blocked. */ let pinnedFetch: typeof fetch | undefined let pinnedIP: string | undefined if (userProvidedEndpoint) { - const validation = await validateUrlWithDNS(userProvidedEndpoint, 'vLLM endpoint', { - allowHttp: true, - }) + const validation = await validateUrlWithDNS( + userProvidedEndpoint, + 'vLLM endpoint', + 'configuredEndpoint' + ) if (!validation.isValid) { logger.warn('Blocked SSRF attempt via vLLM endpoint', { endpoint: userProvidedEndpoint, diff --git a/apps/sim/tools/bitbucket/utils.server.ts b/apps/sim/tools/bitbucket/utils.server.ts index d29b43d804d..afed0ef03a5 100644 --- a/apps/sim/tools/bitbucket/utils.server.ts +++ b/apps/sim/tools/bitbucket/utils.server.ts @@ -75,7 +75,7 @@ export async function secureBitbucketRead( signal?: AbortSignal } = {} ): Promise { - const validation = await validateUrlWithDNS(url, 'bitbucketUrl') + const validation = await validateUrlWithDNS(url, 'bitbucketUrl', 'configuredEndpoint') if (!validation.isValid || !validation.resolvedIP) { throw new Error(`Invalid Bitbucket URL: ${validation.error ?? 'DNS resolution failed'}`) } @@ -83,6 +83,7 @@ export async function secureBitbucketRead( for (let attempt = 1; attempt <= BITBUCKET_READ_MAX_ATTEMPTS; attempt++) { try { const response = await secureFetchWithPinnedIP(url, validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'GET', headers, maxResponseBytes, @@ -137,7 +138,11 @@ export async function resolveBitbucketPullRequestRedirect( targetQuery?: Record } = {} ): Promise { - const initialValidation = await validateUrlWithDNS(initialUrl, 'bitbucketPullRequestUrl') + const initialValidation = await validateUrlWithDNS( + initialUrl, + 'bitbucketPullRequestUrl', + 'configuredEndpoint' + ) if (!initialValidation.isValid || !initialValidation.resolvedIP) { throw new Error( `Invalid Bitbucket pull request URL: ${initialValidation.error ?? 'DNS resolution failed'}` diff --git a/apps/sim/tools/convex/utils.ts b/apps/sim/tools/convex/utils.ts index 324b0f5f836..01450202475 100644 --- a/apps/sim/tools/convex/utils.ts +++ b/apps/sim/tools/convex/utils.ts @@ -15,7 +15,7 @@ import type { */ export function convexApiUrl(deploymentUrl: string, path: string): string { const trimmed = deploymentUrl.trim().replace(/\/+$/, '') - const validation = validateExternalUrl(trimmed, 'Deployment URL') + const validation = validateExternalUrl(trimmed, 'Deployment URL', 'configuredEndpoint') if (!validation.isValid) { throw new Error(`${validation.error} (e.g., https://your-deployment.convex.cloud)`) } diff --git a/apps/sim/tools/github/utils.server.test.ts b/apps/sim/tools/github/utils.server.test.ts index 5499a987616..6223a66a47e 100644 --- a/apps/sim/tools/github/utils.server.test.ts +++ b/apps/sim/tools/github/utils.server.test.ts @@ -16,7 +16,8 @@ vi.mock('@sim/security/dns', () => ({ vi.mock('@/lib/core/config/env-flags', () => ({ isHosted: false, - isPrivateDatabaseHostsAllowed: false, + egressAllowedHosts: undefined, + egressAllowedIpRanges: undefined, getProxyUrl: () => undefined, })) diff --git a/apps/sim/tools/github/utils.server.ts b/apps/sim/tools/github/utils.server.ts index ad2c250ac0d..1ef56eaffec 100644 --- a/apps/sim/tools/github/utils.server.ts +++ b/apps/sim/tools/github/utils.server.ts @@ -63,12 +63,13 @@ export async function secureGitHubRequest( url: string, options: SecureGitHubRequestOptions ): Promise { - const validation = await validateUrlWithDNS(url, 'githubUrl') + const validation = await validateUrlWithDNS(url, 'githubUrl', 'configuredEndpoint') if (!validation.isValid || !validation.resolvedIP) { throw new Error(`Invalid GitHub URL: ${validation.error ?? 'DNS resolution failed'}`) } const response = await secureFetchWithPinnedIP(url, validation.resolvedIP, { + profile: 'configuredEndpoint', method: options.method ?? 'GET', headers: withUserAgent(options.headers), body: options.body, diff --git a/apps/sim/tools/index.test.ts b/apps/sim/tools/index.test.ts index ec4d03ee860..3833b391ab5 100644 --- a/apps/sim/tools/index.test.ts +++ b/apps/sim/tools/index.test.ts @@ -2783,7 +2783,8 @@ describe('Internal Route Trust', () => { expect(result.success).toBe(true) expect(mockValidateUrlWithDNS).toHaveBeenCalledWith( 'http://127.0.0.2:3000/api/v1/workflows/test', - 'toolUrl' + 'toolUrl', + 'requestTarget' ) expect(mockSecureFetchWithPinnedIP).toHaveBeenCalledWith( 'http://127.0.0.2:3000/api/v1/workflows/test', @@ -2874,7 +2875,8 @@ describe('Internal Route Trust', () => { expect(result.success).toBe(true) expect(mockValidateUrlWithDNS).toHaveBeenCalledWith( 'http://127.0.0.1:4000/api/provider', - 'toolUrl' + 'toolUrl', + 'requestTarget' ) expect(mockSecureFetchWithPinnedIP).toHaveBeenCalled() } finally { diff --git a/apps/sim/tools/index.ts b/apps/sim/tools/index.ts index 30acd56bcd3..6a7f47cbecb 100644 --- a/apps/sim/tools/index.ts +++ b/apps/sim/tools/index.ts @@ -2614,7 +2614,7 @@ async function executeToolRequest( const isLastAttempt = attempt === maxAttempts - 1 try { - const urlValidation = await validateUrlWithDNS(fullUrl, 'toolUrl') + const urlValidation = await validateUrlWithDNS(fullUrl, 'toolUrl', 'requestTarget') if (!urlValidation.isValid) { throw new Error(`Invalid tool URL: ${urlValidation.error}`) } @@ -2629,6 +2629,7 @@ async function executeToolRequest( } const secureResponse = await secureFetchWithPinnedIP(fullUrl, urlValidation.resolvedIP!, { + profile: 'requestTarget', method: requestParams.method, headers: headersRecord, body: requestParams.body ?? undefined, diff --git a/apps/sim/tools/posthog/utils.ts b/apps/sim/tools/posthog/utils.ts index 0296fe41911..fde423d8699 100644 --- a/apps/sim/tools/posthog/utils.ts +++ b/apps/sim/tools/posthog/utils.ts @@ -32,7 +32,7 @@ export function getPostHogIngestBaseUrl(region?: 'us' | 'eu', host?: string): st function normalizeHost(host: string): string { const trimmed = host.trim().replace(/\/+$/, '') const withProtocol = /^https?:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}` - const validation = validateExternalUrl(withProtocol, 'Self-hosted host') + const validation = validateExternalUrl(withProtocol, 'Self-hosted host', 'configuredEndpoint') if (!validation.isValid) { throw new Error(`${validation.error} (e.g., posthog.mycompany.com)`) } diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 560b9db9b1c..04b3a520c1a 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -29,6 +29,11 @@ services: - OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434} - SOCKET_SERVER_URL=${SOCKET_SERVER_URL:-http://realtime:3002} - NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-} + # Lets a workflow reach a service on the Docker host. Reaching it also + # requires naming it in EGRESS_ALLOWED_HOSTS; this only makes the name + # resolve, which it does not on Linux by default. + extra_hosts: + - 'host.docker.internal:host-gateway' depends_on: db: condition: service_healthy diff --git a/docker-compose.ollama.yml b/docker-compose.ollama.yml index 2f595b92ead..39104560346 100644 --- a/docker-compose.ollama.yml +++ b/docker-compose.ollama.yml @@ -27,6 +27,11 @@ services: - SIM_AGENT_API_URL=${SIM_AGENT_API_URL} - OLLAMA_URL=http://ollama:11434 - NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-} + # Lets a workflow reach a service on the Docker host. Reaching it also + # requires naming it in EGRESS_ALLOWED_HOSTS; this only makes the name + # resolve, which it does not on Linux by default. + extra_hosts: + - 'host.docker.internal:host-gateway' depends_on: db: condition: service_healthy diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 17a32481749..2cea451fee6 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -50,6 +50,11 @@ services: # different host:port (e.g. wss://socket.example.com). - NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-} - ADMISSION_GATE_MAX_INFLIGHT=${ADMISSION_GATE_MAX_INFLIGHT:-500} + # Lets a workflow reach a service on the Docker host. Reaching it also + # requires naming it in EGRESS_ALLOWED_HOSTS; this only makes the name + # resolve, which it does not on Linux by default. + extra_hosts: + - 'host.docker.internal:host-gateway' depends_on: db: condition: service_healthy diff --git a/package.json b/package.json index 115b8854347..7937b83cb66 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "lint:helm": "helm lint ./helm/sim --strict --values ./helm/sim/test/values-lint.yaml", "lint:all": "turbo run lint && bun run lint:helm", "check": "turbo run format:check", + "check:egress-boundary": "bun run scripts/check-egress-boundary.ts", "check:boundaries": "bun run scripts/check-monorepo-boundaries.ts", "check:api-validation": "bun run scripts/check-api-validation-contracts.ts --check", "check:api-contract-routes": "bun run scripts/check-api-contract-routes.ts", diff --git a/packages/security/package.json b/packages/security/package.json index 782ecd26c5d..7f774918811 100644 --- a/packages/security/package.json +++ b/packages/security/package.json @@ -18,6 +18,10 @@ "types": "./src/dns.ts", "default": "./src/dns.ts" }, + "./egress": { + "types": "./src/egress.ts", + "default": "./src/egress.ts" + }, "./encryption": { "types": "./src/encryption.ts", "default": "./src/encryption.ts" diff --git a/packages/security/src/egress.test.ts b/packages/security/src/egress.test.ts new file mode 100644 index 00000000000..ef1dfa3fbd4 --- /dev/null +++ b/packages/security/src/egress.test.ts @@ -0,0 +1,262 @@ +import { describe, expect, it } from 'vitest' +import { + createEgressPolicy, + type EgressPolicy, + evaluateAddress, + evaluateUrl, + STRICT_EGRESS_POLICY, +} from './egress' + +/** The hosted posture: vouches for nothing, no matter what an operator wrote. */ +const hosted = STRICT_EGRESS_POLICY + +/** A self-hosted posture with a typical Docker/K8s allowlist. */ +const selfHosted = createEgressPolicy({ + allowedHosts: 'host.docker.internal,*.svc.cluster.local', + allowedRanges: '10.0.0.0/8,192.168.65.254/32', + insecureHttp: 'whenVouched', +}) + +function decide(policy: EgressPolicy, href: string, address?: string) { + const url = new URL(href) + return address === undefined ? evaluateUrl(url, policy) : evaluateAddress(url, address, policy) +} + +function reason(policy: EgressPolicy, href: string, address?: string) { + const decision = decide(policy, href, address) + return decision.allowed ? null : decision.reason +} + +describe('evaluateUrl — scheme shape', () => { + it.each([ + ['file:///etc/passwd', 'local file'], + ['gopher://example.com/', 'gopher smuggling'], + ['ftp://example.com/', 'ftp'], + ['data:text/plain,hi', 'data URI'], + ])('rejects %s — %s', (href) => { + expect(reason(hosted, href)).toBe('scheme-not-permitted') + }) + + it('rejects plain http to an unvouched host', () => { + expect(reason(hosted, 'http://example.com/')).toBe('insecure-scheme') + }) + + it('allows https to a public host', () => { + expect(decide(hosted, 'https://example.com/', '93.184.216.34').allowed).toBe(true) + }) +}) + +describe('evaluateUrl — IP-literal hosts resolve without DNS', () => { + it.each([ + ['https://127.0.0.1/', 'address-loopback', 'IPv4 loopback'], + ['https://[::1]/', 'address-loopback', 'IPv6 loopback'], + ['https://127.0.0.5/', 'address-loopback', 'the whole 127/8 range, not just .1'], + ['https://10.1.2.3/', 'address-blocked', 'RFC1918 10/8'], + ['https://192.168.1.1/', 'address-blocked', 'RFC1918 192.168/16'], + ['https://172.16.0.1/', 'address-blocked', 'RFC1918 172.16/12'], + ['https://169.254.1.1/', 'address-blocked', 'link-local'], + ['https://0177.0.0.1/', 'address-loopback', 'octal IPv4 encoding'], + ['https://[::ffff:127.0.0.1]/', 'address-loopback', 'IPv4-mapped IPv6'], + ])('rejects %s as %s — %s', (href, expected) => { + expect(reason(hosted, href)).toBe(expected) + }) + + it('allows a public IP literal', () => { + expect(decide(hosted, 'https://93.184.216.34/').allowed).toBe(true) + }) +}) + +describe('cloud metadata is never reachable', () => { + const metadata = [ + ['169.254.169.254', 'AWS/Azure/GCP IMDS'], + ['169.254.170.2', 'AWS ECS task metadata'], + ['100.100.100.200', 'Alibaba Cloud'], + ['192.0.0.192', 'Oracle Cloud'], + ] as const + + it.each(metadata)('blocks %s on the hosted posture — %s', (ip) => { + expect(reason(hosted, `https://${ip}/`)).toBe('address-metadata') + }) + + it.each(metadata)('blocks %s even when an operator allowlists it outright — %s', (ip) => { + const permissive = createEgressPolicy({ + allowedRanges: `${ip}/32`, + insecureHttp: 'whenVouched', + }) + expect(reason(permissive, `https://${ip}/`)).toBe('address-metadata') + }) + + it('blocks metadata behind a broad operator range allowlist', () => { + const permissive = createEgressPolicy({ allowedRanges: '169.254.0.0/16' }) + expect(reason(permissive, 'https://169.254.169.254/')).toBe('address-metadata') + // ...while the rest of the allowlisted range still works. + expect(decide(permissive, 'https://169.254.1.1/').allowed).toBe(true) + }) + + it('blocks metadata reached through an allowlisted hostname', () => { + const permissive = createEgressPolicy({ allowedHosts: 'metadata.internal' }) + expect(reason(permissive, 'https://metadata.internal/', '169.254.169.254')).toBe( + 'address-metadata' + ) + }) + + it('blocks the AWS IPv6 metadata address', () => { + expect(reason(hosted, 'https://[fd00:ec2::254]/')).toBe('address-metadata') + }) +}) + +describe('operator allowlist — the self-hosted posture', () => { + it('permits plain http to an allowlisted hostname (issue #7200)', () => { + expect( + decide(selfHosted, 'http://host.docker.internal:7274/v1/x', '192.168.65.254').allowed + ).toBe(true) + }) + + it('permits an allowlisted hostname regardless of the private address it resolves to', () => { + expect(decide(selfHosted, 'http://host.docker.internal/', '172.17.0.1').allowed).toBe(true) + }) + + it('permits a wildcard hostname match', () => { + expect(decide(selfHosted, 'http://api.svc.cluster.local/', '10.4.5.6').allowed).toBe(true) + }) + + it('permits an address inside an allowlisted range even for an unlisted hostname', () => { + expect(decide(selfHosted, 'http://build-box.corp/', '10.9.9.9').allowed).toBe(true) + }) + + it('still refuses a private address outside every allowlist entry', () => { + expect(reason(selfHosted, 'https://other.corp/', '172.16.4.4')).toBe('address-blocked') + }) + + it('still refuses plain http to a host it does not vouch for', () => { + expect(reason(selfHosted, 'http://example.com/', '93.184.216.34')).toBe('insecure-scheme') + }) + + it('does not let a wildcard match a bare suffix or a different domain', () => { + // Addresses here sit outside the allowlisted 10/8, so only a hostname match + // could permit them — which is exactly what is being asserted absent. + expect(reason(selfHosted, 'https://svc.cluster.local/', '172.16.1.1')).toBe('address-blocked') + expect(reason(selfHosted, 'https://evil-svc.cluster.local.attacker.com/', '172.16.1.1')).toBe( + 'address-blocked' + ) + }) + + it('matches hostnames case-insensitively', () => { + expect(decide(selfHosted, 'http://HOST.DOCKER.INTERNAL/', '10.0.0.1').allowed).toBe(true) + }) +}) + +describe('the same operator config is inert on the hosted posture', () => { + it.each([ + ['http://host.docker.internal/', '192.168.65.254'], + ['https://api.svc.cluster.local/', '10.4.5.6'], + ['https://build-box.corp/', '10.9.9.9'], + ])('refuses %s', (href, address) => { + // `hosted` is built without the operator lists — the app layer drops them. + expect(decide(hosted, href, address).allowed).toBe(false) + }) +}) + +describe('denied ports', () => { + it.each([ + ['22', 'SSH'], + ['3306', 'MySQL'], + ['5432', 'PostgreSQL'], + ['6379', 'Redis'], + ['27017', 'MongoDB'], + ])('refuses port %s on an unvouched host — %s', (port) => { + expect(reason(hosted, `https://example.com:${port}/`)).toBe('port-denied') + }) + + it('lifts the port denylist for a vouched destination', () => { + expect(decide(selfHosted, 'http://host.docker.internal:9200/', '10.0.0.5').allowed).toBe(true) + }) + + it('leaves ordinary ports alone', () => { + expect(decide(hosted, 'https://example.com:8443/', '93.184.216.34').allowed).toBe(true) + }) +}) + +describe('evaluateAddress is authoritative for DNS names', () => { + it('refuses a public hostname that resolves into private space', () => { + expect(reason(hosted, 'https://rebind.example.com/', '10.0.0.1')).toBe('address-blocked') + }) + + it('refuses a public hostname that resolves to loopback', () => { + expect(reason(hosted, 'https://localtest.me/', '127.0.0.1')).toBe('address-loopback') + }) + + it('accepts a public hostname resolving to a public address', () => { + expect(decide(hosted, 'https://example.com/', '93.184.216.34').allowed).toBe(true) + }) + + it('lets evaluateUrl pass a DNS name it cannot yet classify', () => { + expect(decide(hosted, 'https://rebind.example.com/').allowed).toBe(true) + }) +}) + +describe('invalid input fails closed', () => { + it.each([['not-an-ip'], [''], ['999.999.999.999'], ['::gg']])( + 'refuses the unparseable address %s', + (address) => { + expect(decide(hosted, 'https://example.com/', address).allowed).toBe(false) + } + ) +}) + +describe('createEgressPolicy rejects malformed operator config', () => { + it('names the offending setting in the error', () => { + expect(() => + createEgressPolicy({ + allowedRanges: 'not-a-cidr', + sourceNames: { hosts: 'EGRESS_ALLOWED_HOSTS', ranges: 'EGRESS_ALLOWED_IP_RANGES' }, + }) + ).toThrow(/EGRESS_ALLOWED_IP_RANGES entry "not-a-cidr"/) + }) + + it('refuses a catch-all network', () => { + expect(() => createEgressPolicy({ allowedRanges: '0.0.0.0/0' })).toThrow(/catch-all/) + }) + + it('refuses a bare-suffix wildcard', () => { + expect(() => createEgressPolicy({ allowedHosts: '*.com' })).toThrow(/at least two labels/) + }) + + it('refuses a non-leading wildcard', () => { + expect(() => createEgressPolicy({ allowedHosts: 'api.*.example.com' })).toThrow(/leading/) + }) + + it('refuses a URL where a hostname is expected', () => { + expect(() => createEgressPolicy({ allowedHosts: 'https://example.com/x' })).toThrow( + /expected a hostname/ + ) + }) + + it('tolerates whitespace and empty entries in a list', () => { + const policy = createEgressPolicy({ allowedHosts: ' a.example.com , , b.example.com ' }) + expect(decide(policy, 'https://a.example.com/', '10.0.0.1').allowed).toBe(true) + expect(decide(policy, 'https://b.example.com/', '10.0.0.1').allowed).toBe(true) + }) + + it('accepts an array as well as a comma-separated string', () => { + const policy = createEgressPolicy({ allowedRanges: ['10.0.0.0/8', '192.168.0.0/16'] }) + expect(decide(policy, 'https://x.corp/', '192.168.4.4').allowed).toBe(true) + }) +}) + +describe('must not over-block', () => { + it.each([ + ['https://93.184.216.34/', 'public IPv4 literal'], + ['https://[2606:2800:220:1:248:1893:25c8:1946]/', 'public IPv6 literal'], + ['https://example.com:8080/', 'non-standard but permitted port'], + ['https://example.com/path?q=1#frag', 'query and fragment'], + ])('allows %s — %s', (href) => { + expect(decide(hosted, href).allowed).toBe(true) + }) + + it('does not treat a hostname containing a metadata-looking label as metadata', () => { + expect(decide(hosted, 'https://169.254.169.254.example.com/', '93.184.216.34').allowed).toBe( + true + ) + }) +}) diff --git a/packages/security/src/egress.ts b/packages/security/src/egress.ts new file mode 100644 index 00000000000..9803f5d92d3 --- /dev/null +++ b/packages/security/src/egress.ts @@ -0,0 +1,377 @@ +/** + * Outbound-request (egress) policy: the single decision layer for "may this + * deployment connect to this destination?". + * + * Pure by construction — no DNS, no environment, no I/O, no deployment-posture + * global. A policy is a value built once from operator configuration, and every + * decision is a function of `(destination, policy)`. That is what lets a caller + * evaluate the hosted and self-hosted postures side by side in one test with no + * module mocking, and what lets the same policy object be re-applied unchanged + * to every redirect hop of a request. + * + * Two checks, in order: + * + * 1. {@link evaluateUrl} — pre-DNS. Scheme shape and port. Resolves fully when + * the host is an IP literal, since no lookup is needed to classify one. + * 2. {@link evaluateAddress} — authoritative, once per resolved address. This is + * the check that must gate the actual connect, because only a resolved + * address can be classified (a DNS name says nothing about where it points). + * + * The allowlist is an operator statement of "I vouch for this destination", so a + * match lifts the private-address block, the plain-HTTP restriction, and the + * port denylist together — those three are one question about one destination, + * not three independent switches. Cloud metadata endpoints are the deliberate + * exception and can never be lifted; see {@link METADATA_ADDRESSES}. + */ + +import * as ipaddr from 'ipaddr.js' +import { isLoopbackHostname, unwrapIpv6Brackets } from './hostnames' +import { isIpLiteral, isLoopbackIp, isPrivateIp } from './ssrf' + +type IpAddress = ipaddr.IPv4 | ipaddr.IPv6 + +/** Schemes that may ever carry an outbound request. */ +export type EgressScheme = 'http:' | 'https:' + +/** When a policy tolerates plain HTTP. */ +export type InsecureHttpPolicy = 'never' | 'whenVouched' | 'always' + +/** + * Why a destination was refused. Each value maps to exactly one operator-facing + * remedy, so the set is deliberately no finer than the underlying classifier can + * actually distinguish. + */ +export type EgressDenyReason = + /** Not `http:`/`https:` at all — `file:`, `gopher:`, and friends. */ + | 'scheme-not-permitted' + /** Plain HTTP to a destination the policy does not vouch for. */ + | 'insecure-scheme' + /** A port associated with a non-HTTP service, on an unvouched destination. */ + | 'port-denied' + /** Loopback — called out separately because it is the most common mistake. */ + | 'address-loopback' + /** Private, reserved, link-local, multicast, or otherwise not publicly routable. */ + | 'address-blocked' + /** A cloud metadata endpoint. Never liftable by an allowlist. */ + | 'address-metadata' + +export type EgressDecision = + | { readonly allowed: true } + | { readonly allowed: false; readonly reason: EgressDenyReason; readonly detail: string } + +const ALLOWED: EgressDecision = { allowed: true } + +function deny(reason: EgressDenyReason, detail: string): EgressDecision { + return { allowed: false, reason, detail } +} + +/** + * Cloud instance-metadata endpoints, which hand out credentials to anything that + * can reach them and are therefore the highest-value SSRF target on any managed + * host. Blocked unconditionally: an operator who allowlists a broad range such + * as `169.254.0.0/16` must not silently re-expose these. + */ +const METADATA_ADDRESSES: readonly string[] = [ + '169.254.169.254', // AWS IMDS, Azure IMDS, GCP, DigitalOcean, Oracle + '169.254.170.2', // AWS ECS task metadata + '100.100.100.200', // Alibaba Cloud + '192.0.0.192', // Oracle Cloud (legacy) + 'fd00:ec2::254', // AWS IMDS over IPv6 +] + +/** + * Ports that speak a non-HTTP protocol on a conventional deployment. Refusing + * them blunts protocol-smuggling through a URL the caller does not control. + * Lifted for an allowlisted destination, so an operator can reach their own + * internal Elasticsearch on 9200 after naming it. + */ +const DENIED_PORTS: ReadonlySet = new Set([ + 22, // SSH + 23, // Telnet + 25, // SMTP + 3306, // MySQL + 5432, // PostgreSQL + 6379, // Redis + 27017, // MongoDB + 9200, // Elasticsearch +]) + +interface CidrRange { + readonly address: IpAddress + readonly prefixLength: number +} + +interface HostPattern { + /** Lowercased host, or the lowercased suffix (including the leading dot) for a wildcard. */ + readonly value: string + readonly wildcard: boolean +} + +/** + * A resolved, immutable egress policy. Build one with {@link createEgressPolicy}; + * the shape is internal so the matching rules can change without every caller + * re-deriving them. + */ +export interface EgressPolicy { + /** + * When plain HTTP is acceptable. `whenVouched` covers operator-run internal + * services, which frequently have no TLS; `always` is for a destination whose + * scheme is fixed by protocol rather than by trust, such as an HTTP proxy. + */ + readonly insecureHttp: InsecureHttpPolicy + /** + * Whether loopback counts as vouched. Separate from the operator allowlist + * because a service on the same machine needs no naming to be intentional — + * a single-tenant deployment talking to its own `localhost` is the ordinary + * case, not a privilege. + */ + readonly allowLoopback: boolean + readonly allowedHosts: readonly HostPattern[] + readonly allowedRanges: readonly CidrRange[] +} + +export interface EgressPolicySpec { + /** + * Operator-supplied host allowlist. Entries are exact hostnames or a single + * leading-wildcard label (`*.svc.cluster.local`). Empty or omitted means the + * policy vouches for nothing. + */ + readonly allowedHosts?: string | readonly string[] + /** Operator-supplied CIDR/IP allowlist. */ + readonly allowedRanges?: string | readonly string[] + /** When plain HTTP is acceptable. Defaults to `never`. */ + readonly insecureHttp?: InsecureHttpPolicy + /** Whether loopback destinations are vouched for without being allowlisted. */ + readonly allowLoopback?: boolean + /** + * Names of the settings these lists came from, used verbatim in the error a + * malformed entry throws so the operator knows which value to fix. + */ + readonly sourceNames?: { readonly hosts: string; readonly ranges: string } +} + +const DEFAULT_SOURCE_NAMES = { hosts: 'allowedHosts', ranges: 'allowedRanges' } as const + +function splitEntries(value: string | readonly string[] | undefined): string[] { + if (value === undefined) return [] + const parts = typeof value === 'string' ? value.split(',') : value + return parts.map((entry) => entry.trim()).filter((entry) => entry.length > 0) +} + +function parseHostPattern(entry: string, sourceName: string): HostPattern { + const value = unwrapIpv6Brackets(entry.toLowerCase()) + if (value.startsWith('*.')) { + const suffix = value.slice(1) + if (suffix.length < 2 || !suffix.includes('.', 1)) { + throw new Error( + `Invalid ${sourceName} entry "${entry}": a wildcard must cover at least two labels, e.g. "*.example.com"` + ) + } + return { value: suffix, wildcard: true } + } + if (value.includes('*')) { + throw new Error( + `Invalid ${sourceName} entry "${entry}": a wildcard is only supported as a leading "*." label` + ) + } + if (value.includes('/') || /\s/.test(value)) { + throw new Error( + `Invalid ${sourceName} entry "${entry}": expected a hostname, not a URL or CIDR` + ) + } + return { value, wildcard: false } +} + +function parseCidrRange(entry: string, sourceName: string): CidrRange { + if (entry.includes('/')) { + if (!ipaddr.isValidCIDR(entry)) { + throw new Error(`Invalid ${sourceName} entry "${entry}"`) + } + const [address, prefixLength] = ipaddr.parseCIDR(entry) + if (prefixLength === 0) { + throw new Error(`Invalid ${sourceName} entry "${entry}": catch-all networks are unsafe`) + } + return { address, prefixLength } + } + + const value = unwrapIpv6Brackets(entry) + if (!ipaddr.isValid(value)) { + throw new Error(`Invalid ${sourceName} entry "${entry}"`) + } + const address = ipaddr.parse(value) + return { address, prefixLength: address.kind() === 'ipv4' ? 32 : 128 } +} + +/** + * Builds a policy from operator configuration. Throws on any malformed entry + * rather than silently dropping it — a typo in an allowlist would otherwise + * present as an unexplained connection failure long after startup. + */ +export function createEgressPolicy(spec: EgressPolicySpec = {}): EgressPolicy { + const sourceNames = spec.sourceNames ?? DEFAULT_SOURCE_NAMES + return { + insecureHttp: spec.insecureHttp ?? 'never', + allowLoopback: spec.allowLoopback ?? false, + allowedHosts: splitEntries(spec.allowedHosts).map((entry) => + parseHostPattern(entry, sourceNames.hosts) + ), + allowedRanges: splitEntries(spec.allowedRanges).map((entry) => + parseCidrRange(entry, sourceNames.ranges) + ), + } +} + +/** A policy that vouches for nothing — public HTTPS destinations only. */ +export const STRICT_EGRESS_POLICY: EgressPolicy = createEgressPolicy() + +function matchesHostAllowlist(host: string, policy: EgressPolicy): boolean { + if (policy.allowedHosts.length === 0) return false + const clean = unwrapIpv6Brackets(host.toLowerCase()) + return policy.allowedHosts.some((pattern) => + pattern.wildcard ? clean.endsWith(pattern.value) : clean === pattern.value + ) +} + +function matchesRangeAllowlist(address: string, policy: EgressPolicy): boolean { + if (policy.allowedRanges.length === 0) return false + const clean = unwrapIpv6Brackets(address) + if (!ipaddr.isValid(clean)) return false + const parsed = ipaddr.process(clean) + return policy.allowedRanges.some( + (range) => + range.address.kind() === parsed.kind() && parsed.match(range.address, range.prefixLength) + ) +} + +function isMetadataAddress(address: string): boolean { + const clean = unwrapIpv6Brackets(address) + if (!ipaddr.isValid(clean)) return false + const normalized = ipaddr.process(clean).toString() + return METADATA_ADDRESSES.some((candidate) => ipaddr.process(candidate).toString() === normalized) +} + +/** + * Whether the policy vouches for this destination. A hostname match alone is + * enough — the operator named that host, so wherever it points is their call. + * Otherwise a resolved address inside an allowlisted range vouches for it, which + * is why this cannot be decided before DNS for a hostname destination. + */ +function isVouched(url: URL, address: string | undefined, policy: EgressPolicy): boolean { + if (matchesHostAllowlist(url.hostname, policy)) return true + // `localhost` is loopback by name, so a loopback-permitting policy can vouch + // for it before DNS — which is what lets the synchronous check accept a local + // dev server without pretending to know where an arbitrary hostname points. + if (policy.allowLoopback && isLoopbackHostname(url.hostname)) return true + if (address === undefined) return false + if (policy.allowLoopback && isLoopbackIp(unwrapIpv6Brackets(address))) return true + return matchesRangeAllowlist(address, policy) +} + +function checkSchemeAndPort(url: URL, vouched: boolean, policy: EgressPolicy): EgressDecision { + if ( + url.protocol === 'http:' && + policy.insecureHttp !== 'always' && + !(vouched && policy.insecureHttp === 'whenVouched') + ) { + return deny('insecure-scheme', `plain http to ${url.hostname}`) + } + + if (!vouched && url.port) { + const port = Number.parseInt(url.port, 10) + if (DENIED_PORTS.has(port)) { + return deny('port-denied', `port ${port}`) + } + } + + return ALLOWED +} + +/** Classifies one address, assuming the vouched decision has already been made. */ +function checkAddressClass(address: string, vouched: boolean): EgressDecision { + if (vouched) return ALLOWED + + const clean = unwrapIpv6Brackets(address) + if (isLoopbackIp(clean)) { + return deny('address-loopback', address) + } + if (isPrivateIp(clean)) { + return deny('address-blocked', address) + } + + return ALLOWED +} + +/** + * Pre-DNS gate. Decides completely when the host is an IP literal; for a + * hostname it judges the destination as unvouched, since where the name points + * is not known yet. + * + * Neither verdict is the last word on a hostname. A refusal may be liftable once + * the address is known ({@link policyCanVouch}, {@link isLiftableByVouching}), + * and an approval covers only what needs no lookup — {@link evaluateAddress} is + * authoritative and must run against every resolved address before connecting. + */ +export function evaluateUrl(url: URL, policy: EgressPolicy): EgressDecision { + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + return deny('scheme-not-permitted', `unsupported protocol ${url.protocol}`) + } + + const host = unwrapIpv6Brackets(url.hostname) + if (isIpLiteral(host)) { + return evaluateAddress(url, host, policy) + } + + // Judged as if unvouched, because a hostname's address is not known yet. A + // policy that could still vouch for it once resolved must not treat this + // verdict as final — see {@link policyCanVouch}. + return checkSchemeAndPort(url, isVouched(url, undefined, policy), policy) +} + +/** + * Whether this policy has any way to vouch for a destination it has not already + * accepted — an allowlist entry, or a loopback carve-out. + * + * A DNS-resolving caller uses this to decide whether a refusal from + * {@link evaluateUrl} on a hostname is final, or whether it must resolve and let + * {@link evaluateAddress} rule on the addresses. Without it the pre-DNS check + * would have to either refuse destinations the policy actually permits, or wave + * through ones it does not. + */ +export function policyCanVouch(policy: EgressPolicy): boolean { + return policy.allowedHosts.length > 0 || policy.allowedRanges.length > 0 || policy.allowLoopback +} + +/** + * Whether a refusal could be lifted by learning the destination's address. + * `scheme-not-permitted` and `address-metadata` never can be. + */ +export function isLiftableByVouching(reason: EgressDenyReason): boolean { + return reason !== 'scheme-not-permitted' && reason !== 'address-metadata' +} + +/** + * Authoritative gate for one resolved address. `url` is the destination the + * address was resolved for, so a hostname allowlist entry still applies. + * + * Call this for every address a host resolves to. A caller that connects to a + * different address than the one it evaluated has no protection against DNS + * rebinding, so the evaluated address must also be the pinned one. + */ +export function evaluateAddress(url: URL, address: string, policy: EgressPolicy): EgressDecision { + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + return deny('scheme-not-permitted', `unsupported protocol ${url.protocol}`) + } + + // Checked before the allowlist, and before the cheaper scheme/port rules, so + // that reaching for a metadata endpoint always reports why it is hopeless. + if (isMetadataAddress(address)) { + return deny('address-metadata', address) + } + + const vouched = isVouched(url, address, policy) + + const shape = checkSchemeAndPort(url, vouched, policy) + if (!shape.allowed) return shape + + return checkAddressClass(address, vouched) +} diff --git a/packages/sim-setup/src/steps.ts b/packages/sim-setup/src/steps.ts index 4febfd3aaa0..ed61a4d7df5 100644 --- a/packages/sim-setup/src/steps.ts +++ b/packages/sim-setup/src/steps.ts @@ -236,12 +236,17 @@ export async function promptSecurity(vars: Map): Promise(key: K): PropertyDescriptor { return { enumerable: true, get: () => envFlagsState[key], - set: (value: boolean) => { + set: (value: EnvFlagsMockState[K]) => { envFlagsState[key] = value }, } diff --git a/scripts/check-egress-boundary.ts b/scripts/check-egress-boundary.ts new file mode 100644 index 00000000000..ec428fef55c --- /dev/null +++ b/scripts/check-egress-boundary.ts @@ -0,0 +1,102 @@ +#!/usr/bin/env bun +/** + * Keeps outbound HTTP behind the egress guard. + * + * Every request Sim makes to a user- or model-influenced destination has to go + * through `lib/core/security/egress`, which resolves DNS, classifies each + * address against the deployment's policy, and pins the connection to the + * address it approved. A module that reaches for `node:http`, `node:https`, or + * `undici` directly gets none of that, and the omission is invisible — the code + * works, it just has no guard. + * + * This checks the import edge rather than the call, because that is the part + * that cannot be hidden behind a helper. + * + * Not checked: bare `fetch()`. It is used constantly for same-origin and + * server-action calls where the guard does not apply, so flagging it would be + * noise. The transports it can reach are covered by the import rule above. + * + * Usage: bun run scripts/check-egress-boundary.ts + */ +import { readdirSync, readFileSync } from 'node:fs' +import path from 'node:path' + +const ROOT = path.resolve(import.meta.dir, '..') + +const SCAN_DIRS = ['apps/sim/app', 'apps/sim/lib', 'apps/sim/tools', 'apps/sim/connectors'] + +const SKIP_DIRS = new Set(['node_modules', 'dist', '.next', '.turbo', 'coverage']) + +/** Raw HTTP transports. Reaching one directly bypasses DNS pinning. */ +const TRANSPORT_IMPORT = + /^\s*import\s[^'"]*from\s+['"](?:node:)?(http|https|undici|http-proxy-agent|https-proxy-agent)['"]/ + +/** + * Modules allowed to hold a transport import, each because it *is* part of the + * guard or predates it for a documented reason. + */ +const ALLOWED = new Set([ + // The guard itself: resolves, classifies, pins, and follows redirects. + 'apps/sim/lib/core/security/input-validation.server.ts', + // Streaming MCP transport, built on the guard's pinned dispatcher. + 'apps/sim/lib/mcp/pinned-fetch.ts', + // Builds a dispatcher to carry a caller's deadline; issues no request itself. + 'apps/sim/lib/core/utils/fetch-deadline.ts', +]) + +function walk(dir: string, out: string[] = []): string[] { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (SKIP_DIRS.has(entry.name)) continue + const full = path.join(dir, entry.name) + if (entry.isDirectory()) walk(full, out) + else if (/\.(ts|tsx)$/.test(entry.name) && !/\.test\.tsx?$/.test(entry.name)) out.push(full) + } + return out +} + +interface Violation { + file: string + line: number + snippet: string +} + +function main() { + const violations: Violation[] = [] + let scanned = 0 + + for (const scanDir of SCAN_DIRS) { + const abs = path.join(ROOT, scanDir) + for (const file of walk(abs)) { + const rel = path.relative(ROOT, file).split(path.sep).join('/') + if (ALLOWED.has(rel)) continue + scanned++ + const lines = readFileSync(file, 'utf8').split('\n') + for (let i = 0; i < lines.length; i++) { + if (TRANSPORT_IMPORT.test(lines[i])) { + violations.push({ file: rel, line: i + 1, snippet: lines[i].trim() }) + } + } + } + } + + if (violations.length === 0) { + console.log(`✓ check-egress-boundary: ${scanned} files, no unguarded HTTP transports`) + process.exit(0) + } + + console.error('✗ check-egress-boundary: raw HTTP transport outside the egress guard\n') + for (const violation of violations) { + console.error(` ${violation.file}:${violation.line}`) + console.error(` ${violation.snippet}`) + } + console.error( + '\n These modules can open a socket without resolving and classifying the\n' + + ' destination first, so a user- or model-supplied URL reaches the network\n' + + ' unchecked. Use secureFetchWithValidation (or secureFetchWithPinnedIP with\n' + + ' a validated address) from @/lib/core/security/input-validation.server and\n' + + ' pass the egress profile describing where the URL came from.\n' + ) + process.exit(1) +} + +main() From 6a4ec2d64de747b217f1c271befed59d0963538f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 13:59:23 -0700 Subject: [PATCH 02/20] fix(egress): close regressions found reviewing the profile migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nine issues from an adversarial pass over the previous commit. Plain HTTP was the sharpest. `allowHttp: true` permitted http to any host; folding those sites into `configuredEndpoint` made it conditional on the destination being vouched, so on the hosted platform — where the allowlist is always empty — an `http://` vLLM or Jupyter endpoint became unreachable with no remedy. Adds `selfHostedService` for the four integrations that passed the old flag, where plain HTTP is expected by protocol rather than granted by trust. Loopback was the sharpest security one. The old guard decided loopback from the hostname; the rewrite decided it from the resolved address, so on any self-hosted deployment an attacker-chosen name resolving to `127.0.0.1` inherited the carve-out and reached the deployment's own loopback services. Now keyed on the hostname again, with tests for `localtest.me` and `nip.io`. The deprecated alias was over-broad in one direction and too narrow in the other. It fed the shared allowlist, which `requestTarget` also honors, so an upgrading deployment that set it for Postgres silently handed every workflow author a route into its internal network; it is now scoped to `databaseHost`, which is all the flag ever governed. And it expanded to a hand-written range list that omitted CGNAT — where Tailscale lives — so a self-hoster reaching Postgres over Tailscale would have broken on upgrade. It now maps to a policy that vouches for any private address, which is what the flag actually did. Also: - `env-flags` exports are functions, satisfying the CI rule that every export there be `is`/`get`-prefixed, and incidentally fixing a stale-cache bug: the policy cache keyed only on the allowlist, so a changed posture was not picked up. - No allowlist remedy is offered on the hosted platform, where the variables are ignored and the advice would send the reader nowhere. - `validateDatabricksWorkspaceHost` keeps its suffix-only matching; the extracted helper had widened it to accept the bare apex. - The boundary check scans `executor`, `providers` and `triggers` too. - Drops a comment referencing `validateHostname`, which no longer exists. - Connector tests were passing options without a profile, exercising the fail-closed fallback instead of asserting a real one. --- .../docs/platform/self-hosting/security.mdx | 3 +- apps/sim/lib/core/config/env-flags.ts | 61 ++++++------ apps/sim/lib/core/security/egress/profiles.ts | 92 +++++++++++++------ .../security/input-validation.server.test.ts | 5 +- .../core/security/input-validation.server.ts | 7 +- .../core/security/input-validation.test.ts | 32 +++++-- .../sim/lib/core/security/input-validation.ts | 14 ++- .../pinned-redirect-replay.server.test.ts | 5 +- .../secure-fetch-response-cap.server.test.ts | 5 +- .../lib/internal/clickhouse/client.test.ts | 4 +- apps/sim/lib/internal/clickhouse/client.ts | 2 +- apps/sim/lib/internal/jupyter/client.test.ts | 4 +- apps/sim/lib/internal/jupyter/client.ts | 4 +- .../lib/internal/onepassword/client.test.ts | 2 +- apps/sim/lib/internal/onepassword/client.ts | 2 +- .../sim/lib/knowledge/documents/utils.test.ts | 18 ++-- apps/sim/providers/vllm/index.test.ts | 4 +- apps/sim/providers/vllm/index.ts | 2 +- apps/sim/tools/github/utils.server.test.ts | 5 +- packages/security/src/egress.test.ts | 54 +++++++++++ packages/security/src/egress.ts | 37 ++++++-- packages/testing/src/mocks/env-flags.mock.ts | 23 +++++ scripts/check-egress-boundary.ts | 10 +- 23 files changed, 284 insertions(+), 111 deletions(-) diff --git a/apps/docs/content/docs/platform/self-hosting/security.mdx b/apps/docs/content/docs/platform/self-hosting/security.mdx index 61768cf0adb..36f7e938e3c 100644 --- a/apps/docs/content/docs/platform/self-hosting/security.mdx +++ b/apps/docs/content/docs/platform/self-hosting/security.mdx @@ -138,7 +138,8 @@ Sim blocks outbound requests to private, reserved, and loopback addresses. This | Provenance | Examples | Reaches allowlisted private destinations | |---|---|---| -| Configured endpoint | A self-hosted vLLM, Jupyter, GitHub Enterprise, Grafana, ClickHouse, an MCP server, a connector's host | Yes | +| Configured endpoint | GitHub Enterprise, Grafana, an MCP server, a data-drain destination, a connector's host | Yes | +| Self-hosted service | vLLM, Jupyter, 1Password Connect, ClickHouse — software usually run on-prem without TLS, so plain HTTP is expected | Yes | | Request target | The HTTP block's URL, an A2A agent, an RSS feed, a Function block's `fetch` | Yes | | Database host | A database, cache, or mail connector's host | Yes | | Content fetch | An image URL, a file imported by URL, a link from a third-party API response | **No** | diff --git a/apps/sim/lib/core/config/env-flags.ts b/apps/sim/lib/core/config/env-flags.ts index cf1f28b25f6..0c4265f2eea 100644 --- a/apps/sim/lib/core/config/env-flags.ts +++ b/apps/sim/lib/core/config/env-flags.ts @@ -134,44 +134,41 @@ if (isTruthy(env.DISABLE_AUTH)) { }) } -const legacyPrivateHostsAllowed = isTruthy(env.ALLOW_PRIVATE_DATABASE_HOSTS) - /** - * Ranges the deprecated `ALLOW_PRIVATE_DATABASE_HOSTS` stood for: it was a - * blanket "reach anything private", so it maps to the private and loopback - * space in full. Appended to whatever the operator listed explicitly, which is - * why an unset flag contributes nothing. Cloud metadata stays unreachable - * regardless — that block is not lifted by any allowlist. + * Destinations on a private network that outbound requests may reach, as raw + * operator config. Empty on the hosted platform regardless of what is set, so a + * tenant can never pivot into Sim's own network — mirroring {@link isAuthDisabled}. + * + * Read through a function rather than captured at module load so a changed value + * is picked up, and parsed in `@sim/security/egress` rather than here: this file + * is loaded by `next.config.ts` before the `@/` alias exists and must stay + * dependency-light. */ -const LEGACY_PRIVATE_RANGES = - '10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,127.0.0.0/8,169.254.0.0/16,::1/128,fc00::/7,fe80::/10' +export function getEgressAllowedHosts(): string | undefined { + return isHosted ? undefined : env.EGRESS_ALLOWED_HOSTS +} -function joinConfig(...parts: Array): string | undefined { - const joined = parts.filter((part) => part && part.trim().length > 0).join(',') - return joined.length > 0 ? joined : undefined +export function getEgressAllowedIpRanges(): string | undefined { + return isHosted ? undefined : env.EGRESS_ALLOWED_IP_RANGES } /** - * Destinations on a private network that outbound requests may reach, as raw - * operator config. Empty on the hosted platform regardless of what is set, so a - * tenant can never pivot into Sim's own network — mirroring {@link isAuthDisabled}. + * Whether the deprecated `ALLOW_PRIVATE_DATABASE_HOSTS` is set. + * + * It stood for a blanket "database and connector tools may reach anything + * private", so it maps to a policy that vouches for every private address rather + * than to a range list — a list would silently drop the ranges it never + * enumerated, CGNAT (`100.64.0.0/10`, where Tailscale lives) among them. * - * Parsing and enforcement live in `@sim/security/egress`; these are passed - * through unparsed because `env-flags` is loaded by `next.config.ts` before the - * `@/` alias exists and must stay dependency-light. - */ -export const egressAllowedHosts = isHosted - ? undefined - : joinConfig(env.EGRESS_ALLOWED_HOSTS, legacyPrivateHostsAllowed ? 'localhost' : undefined) - -export const egressAllowedIpRanges = isHosted - ? undefined - : joinConfig( - env.EGRESS_ALLOWED_IP_RANGES, - legacyPrivateHostsAllowed ? LEGACY_PRIVATE_RANGES : undefined - ) - -if (legacyPrivateHostsAllowed) { + * Scoped to database hosts, which is all the flag ever governed. Widening it to + * HTTP destinations would hand every workflow author on an upgrading deployment + * a route into the internal network they never granted. + */ +export function isLegacyPrivateDatabaseAccessAllowed(): boolean { + return !isHosted && isTruthy(env.ALLOW_PRIVATE_DATABASE_HOSTS) +} + +if (isTruthy(env.ALLOW_PRIVATE_DATABASE_HOSTS)) { import('@sim/logger') .then(({ createLogger }) => { const logger = createLogger('EnvFlags') @@ -181,7 +178,7 @@ if (legacyPrivateHostsAllowed) { ) } else { logger.warn( - 'ALLOW_PRIVATE_DATABASE_HOSTS is deprecated and opens the whole private address space. Replace it with EGRESS_ALLOWED_HOSTS / EGRESS_ALLOWED_IP_RANGES naming only the destinations you need.' + 'ALLOW_PRIVATE_DATABASE_HOSTS is deprecated. It opens the whole private address space to database and connector tools. Replace it with EGRESS_ALLOWED_HOSTS / EGRESS_ALLOWED_IP_RANGES naming only the destinations you need.' ) } }) diff --git a/apps/sim/lib/core/security/egress/profiles.ts b/apps/sim/lib/core/security/egress/profiles.ts index 2da4618714b..868f1d06557 100644 --- a/apps/sim/lib/core/security/egress/profiles.ts +++ b/apps/sim/lib/core/security/egress/profiles.ts @@ -18,7 +18,12 @@ import { type EgressPolicy, type InsecureHttpPolicy, } from '@sim/security/egress' -import { egressAllowedHosts, egressAllowedIpRanges, isHosted } from '@/lib/core/config/env-flags' +import { + getEgressAllowedHosts, + getEgressAllowedIpRanges, + isHosted, + isLegacyPrivateDatabaseAccessAllowed, +} from '@/lib/core/config/env-flags' /** * Where the URL for an outbound request came from. @@ -35,12 +40,17 @@ import { egressAllowedHosts, egressAllowedIpRanges, isHosted } from '@/lib/core/ * Configured like the first, but without its loopback carve-out: loopback is * exactly where Sim's own database and Redis listen, so reaching them has to * be named rather than assumed. + * - `selfHostedService` — a configured endpoint for software normally run + * on-prem without TLS: vLLM, Jupyter, 1Password Connect, ClickHouse. Same + * reachability as `configuredEndpoint`, but plain HTTP is expected rather than + * conditional, which is what these integrations relied on before. * - `proxy` — the egress proxy itself. Held to the strictest rule of all, * because it is the component that decides where everything else may go: plain * HTTP by protocol, but public destinations only, and no allowlist. */ export type EgressProfile = | 'configuredEndpoint' + | 'selfHostedService' | 'requestTarget' | 'contentFetch' | 'databaseHost' @@ -65,6 +75,12 @@ interface ProfileSpec { * never true on the hosted platform, where `localhost` is Sim's own process. */ readonly allowLoopback: boolean + /** + * Whether the deprecated `ALLOW_PRIVATE_DATABASE_HOSTS` applies. Only + * `databaseHost` sets this, because that is the only thing the flag ever + * governed. + */ + readonly honorsLegacyPrivateFlag?: boolean } const PROFILE_SPECS: Record = { @@ -73,9 +89,15 @@ const PROFILE_SPECS: Record = { insecureHttp: 'whenVouched', allowLoopback: !isHosted, }, + selfHostedService: { honorsAllowlist: true, insecureHttp: 'always', allowLoopback: !isHosted }, requestTarget: { honorsAllowlist: true, insecureHttp: 'whenVouched', allowLoopback: !isHosted }, contentFetch: { honorsAllowlist: false, insecureHttp: 'never', allowLoopback: false }, - databaseHost: { honorsAllowlist: true, insecureHttp: 'whenVouched', allowLoopback: false }, + databaseHost: { + honorsAllowlist: true, + insecureHttp: 'whenVouched', + allowLoopback: false, + honorsLegacyPrivateFlag: true, + }, proxy: { honorsAllowlist: false, insecureHttp: 'always', allowLoopback: false }, } @@ -84,17 +106,32 @@ const SOURCE_NAMES = { ranges: 'EGRESS_ALLOWED_IP_RANGES', } as const -function buildPolicies(hosts: string | undefined, ranges: string | undefined) { +interface DeploymentConfig { + readonly hosts: string | undefined + readonly ranges: string | undefined + readonly legacyPrivate: boolean +} + +function readDeploymentConfig(): DeploymentConfig { + return { + hosts: getEgressAllowedHosts(), + ranges: getEgressAllowedIpRanges(), + legacyPrivate: isLegacyPrivateDatabaseAccessAllowed(), + } +} + +function buildPolicies(config: DeploymentConfig): Record { return Object.fromEntries( (Object.keys(PROFILE_SPECS) as EgressProfile[]).map((profile) => { const spec = PROFILE_SPECS[profile] return [ profile, createEgressPolicy({ - allowedHosts: spec.honorsAllowlist ? hosts : undefined, - allowedRanges: spec.honorsAllowlist ? ranges : undefined, + allowedHosts: spec.honorsAllowlist ? config.hosts : undefined, + allowedRanges: spec.honorsAllowlist ? config.ranges : undefined, insecureHttp: spec.insecureHttp, allowLoopback: spec.allowLoopback, + allowPrivate: Boolean(spec.honorsLegacyPrivateFlag && config.legacyPrivate), sourceNames: SOURCE_NAMES, }), ] @@ -102,19 +139,22 @@ function buildPolicies(hosts: string | undefined, ranges: string | undefined) { ) as Record } +function sameConfig(a: DeploymentConfig, b: DeploymentConfig): boolean { + return a.hosts === b.hosts && a.ranges === b.ranges && a.legacyPrivate === b.legacyPrivate +} + /** * Policies are built eagerly so a malformed allowlist entry throws at startup * rather than at whichever request first happens to touch it, and cached against * the configuration they were built from so that changing it rebuilds rather * than silently serving a stale policy. Caching on the value rather than "built - * once" is what keeps the allowlist reachable from a test without a module-level - * reset hook. + * once" is what keeps the configuration reachable from a test without a + * module-level reset hook. */ -let cache = { - hosts: egressAllowedHosts, - ranges: egressAllowedIpRanges, - policies: buildPolicies(egressAllowedHosts, egressAllowedIpRanges), -} +let cache = (() => { + const config = readDeploymentConfig() + return { config, policies: buildPolicies(config) } +})() /** * The policy governing requests of the given provenance on this deployment. @@ -125,21 +165,13 @@ let cache = { * cause is no longer visible. */ export function resolveEgressPolicy(profile: EgressProfile): EgressPolicy { - if (cache.hosts !== egressAllowedHosts || cache.ranges !== egressAllowedIpRanges) { - cache = { - hosts: egressAllowedHosts, - ranges: egressAllowedIpRanges, - policies: buildPolicies(egressAllowedHosts, egressAllowedIpRanges), - } + const config = readDeploymentConfig() + if (!sameConfig(cache.config, config)) { + cache = { config, policies: buildPolicies(config) } } return cache.policies[profile] ?? cache.policies.contentFetch } -/** True when this deployment has any private-network allowlist configured. */ -function hasAllowlist(): boolean { - return Boolean(egressAllowedHosts || egressAllowedIpRanges) -} - /** * Turns a refusal into a message the person who hit it can act on. * @@ -157,11 +189,15 @@ export function describeEgressDenial( // strictest spec, so a bad profile can never advertise a remedy that does not // apply to the policy that actually refused the request. const spec = PROFILE_SPECS[profile] ?? PROFILE_SPECS.contentFetch - const remedy = spec.honorsAllowlist - ? hasAllowlist() - ? ` It is not covered by ${SOURCE_NAMES.hosts} or ${SOURCE_NAMES.ranges}.` - : ` Self-hosted deployments can permit specific destinations with ${SOURCE_NAMES.hosts} or ${SOURCE_NAMES.ranges}.` - : '' + const config = readDeploymentConfig() + // No remedy is offered on the hosted platform, where the allowlist variables + // are ignored and pointing at them would send the reader somewhere useless. + const remedy = + !spec.honorsAllowlist || isHosted + ? '' + : config.hosts || config.ranges + ? ` It is not covered by ${SOURCE_NAMES.hosts} or ${SOURCE_NAMES.ranges}.` + : ` Self-hosted deployments can permit specific destinations with ${SOURCE_NAMES.hosts} or ${SOURCE_NAMES.ranges}.` switch (decision.reason) { case 'scheme-not-permitted': diff --git a/apps/sim/lib/core/security/input-validation.server.test.ts b/apps/sim/lib/core/security/input-validation.server.test.ts index 1475a0c6b6d..dbc75ea1799 100644 --- a/apps/sim/lib/core/security/input-validation.server.test.ts +++ b/apps/sim/lib/core/security/input-validation.server.test.ts @@ -13,8 +13,9 @@ vi.mock('@sim/security/dns', () => ({ vi.mock('@/lib/core/config/env-flags', () => ({ isHosted: false, - egressAllowedHosts: undefined, - egressAllowedIpRanges: undefined, + getEgressAllowedHosts: () => undefined, + getEgressAllowedIpRanges: () => undefined, + isLegacyPrivateDatabaseAccessAllowed: () => false, getProxyUrl: () => undefined, })) diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index c44d31aa5bf..a5db1456dbc 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -125,10 +125,9 @@ export async function validateAndPinProxyUrl( * Validates a database hostname by resolving DNS and checking the resolved IP * against private/reserved ranges to prevent SSRF via database connections. * - * Unlike validateHostname (which enforces strict RFC hostname format), this - * function is permissive about hostname format to avoid breaking legitimate - * database hostnames (e.g. underscores in Docker/K8s service names). It only - * blocks localhost and private/reserved IPs. + * Permissive about hostname format, so a legitimate database host is not + * rejected on shape alone — Docker and K8s service names carry underscores that + * a strict RFC check would refuse. Only the address is judged. * * Self-hosted operators reach a database on their private network (e.g. a * Docker/Swarm service name that resolves to an internal IP) by naming it in the diff --git a/apps/sim/lib/core/security/input-validation.test.ts b/apps/sim/lib/core/security/input-validation.test.ts index 3fd24267239..3cc21cbb4c5 100644 --- a/apps/sim/lib/core/security/input-validation.test.ts +++ b/apps/sim/lib/core/security/input-validation.test.ts @@ -575,24 +575,36 @@ describe('validateDatabaseHost', () => { envFlagsMock.egressAllowedIpRanges = undefined }) - it('keeps working for a deployment that still sets only the old flag', async () => { - // env-flags expands the flag into the full private space, so the alias - // reproduces the behavior those deployments have today. - envFlagsMock.egressAllowedHosts = 'localhost' - envFlagsMock.egressAllowedIpRanges = - '10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,127.0.0.0/8,169.254.0.0/16,::1/128,fc00::/7,fe80::/10' + afterEach(() => { + envFlagsMock.legacyPrivateDatabaseAccess = false + }) - expect((await validateDatabaseHost('localhost')).isValid).toBe(true) - expect((await validateDatabaseHost('10.0.0.5')).isValid).toBe(true) - expect((await validateDatabaseHost('127.0.0.1')).isValid).toBe(true) + it.each([ + ['localhost', 'loopback by name'], + ['127.0.0.1', 'loopback literal'], + ['10.0.0.5', 'RFC1918'], + ['100.64.0.1', 'CGNAT, where a Tailscale host lives'], + ])('keeps %s reachable for a deployment still on the old flag — %s', async (host) => { + envFlagsMock.legacyPrivateDatabaseAccess = true + expect((await validateDatabaseHost(host)).isValid).toBe(true) }) it('still cannot reach cloud metadata through the alias', async () => { - envFlagsMock.egressAllowedIpRanges = '169.254.0.0/16' + envFlagsMock.legacyPrivateDatabaseAccess = true const result = await validateDatabaseHost('169.254.169.254') expect(result.isValid).toBe(false) expect(result.error).toContain('cloud metadata endpoint') }) + + it('does not widen HTTP destinations, which the flag never governed', async () => { + envFlagsMock.legacyPrivateDatabaseAccess = true + expect( + (await validateUrlWithDNS('https://10.0.0.5/api', 'url', 'requestTarget')).isValid + ).toBe(false) + expect( + (await validateUrlWithDNS('https://10.0.0.5/api', 'url', 'configuredEndpoint')).isValid + ).toBe(false) + }) }) describe('self-host opt-in (EGRESS_ALLOWED_HOSTS / EGRESS_ALLOWED_IP_RANGES)', () => { diff --git a/apps/sim/lib/core/security/input-validation.ts b/apps/sim/lib/core/security/input-validation.ts index e96f2e542b1..3602d0d233a 100644 --- a/apps/sim/lib/core/security/input-validation.ts +++ b/apps/sim/lib/core/security/input-validation.ts @@ -1094,6 +1094,7 @@ const SERVICENOW_ALLOWED_HOST_SUFFIXES = [ * @param options.paramName - Name of the parameter for error messages * @param options.assumeHttps - Accept a bare host by prepending `https://` * @param options.sanitize - What to return as `sanitized`: the input, or the parsed origin + * @param options.allowBareSuffix - Also accept the suffix itself as a hostname */ function validateVendorHostedUrl( url: string | null | undefined, @@ -1103,9 +1104,17 @@ function validateVendorHostedUrl( paramName: string assumeHttps?: boolean sanitize?: 'input' | 'origin' + allowBareSuffix?: boolean } ): ValidationResult { - const { suffixes, vendor, paramName, assumeHttps = false, sanitize = 'input' } = options + const { + suffixes, + vendor, + paramName, + assumeHttps = false, + sanitize = 'input', + allowBareSuffix = true, + } = options const raw = typeof url === 'string' ? url.trim() : '' if (!raw) { @@ -1120,7 +1129,7 @@ function validateVendorHostedUrl( const parsed = new URL(candidate) const hostname = parsed.hostname.toLowerCase() const allowed = suffixes.some( - (suffix) => hostname === suffix.slice(1) || hostname.endsWith(suffix) + (suffix) => (allowBareSuffix && hostname === suffix.slice(1)) || hostname.endsWith(suffix) ) if (!allowed) { @@ -1278,6 +1287,7 @@ export function validateDatabricksWorkspaceHost( paramName, assumeHttps: true, sanitize: 'origin', + allowBareSuffix: false, }) } diff --git a/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts b/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts index e05746c9d24..58046fec89d 100644 --- a/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts +++ b/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts @@ -15,8 +15,9 @@ vi.mock('@sim/security/dns', () => ({ vi.mock('@/lib/core/config/env-flags', () => ({ isHosted: false, - egressAllowedHosts: undefined, - egressAllowedIpRanges: undefined, + getEgressAllowedHosts: () => undefined, + getEgressAllowedIpRanges: () => undefined, + isLegacyPrivateDatabaseAccessAllowed: () => false, getProxyUrl: () => undefined, })) diff --git a/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts b/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts index 7a362e1c5ed..8d57159c27e 100644 --- a/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts +++ b/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts @@ -12,8 +12,9 @@ vi.mock('@sim/security/dns', () => ({ vi.mock('@/lib/core/config/env-flags', () => ({ isHosted: false, - egressAllowedHosts: undefined, - egressAllowedIpRanges: undefined, + getEgressAllowedHosts: () => undefined, + getEgressAllowedIpRanges: () => undefined, + isLegacyPrivateDatabaseAccessAllowed: () => false, getProxyUrl: () => undefined, })) diff --git a/apps/sim/lib/internal/clickhouse/client.test.ts b/apps/sim/lib/internal/clickhouse/client.test.ts index 04a0b34464c..7a89177687c 100644 --- a/apps/sim/lib/internal/clickhouse/client.test.ts +++ b/apps/sim/lib/internal/clickhouse/client.test.ts @@ -95,7 +95,7 @@ describe('clickhouseRequest DNS pinning', () => { const [url, , options] = mockSecureFetchWithPinnedIP.mock.calls[0] expect(url).toMatch(/^https:\/\//) - expect(options.profile).toBe('configuredEndpoint') + expect(options.profile).toBe('selfHostedService') }) it('allows http for the initial request when secure is false', async () => { @@ -103,7 +103,7 @@ describe('clickhouseRequest DNS pinning', () => { const [url, , options] = mockSecureFetchWithPinnedIP.mock.calls[0] expect(url).toMatch(/^http:\/\//) - expect(options.profile).toBe('configuredEndpoint') + expect(options.profile).toBe('selfHostedService') }) it('brackets an unbracketed IPv6 literal when constructing the request URL', async () => { diff --git a/apps/sim/lib/internal/clickhouse/client.ts b/apps/sim/lib/internal/clickhouse/client.ts index 46494c9a5be..b7c4201c37b 100644 --- a/apps/sim/lib/internal/clickhouse/client.ts +++ b/apps/sim/lib/internal/clickhouse/client.ts @@ -74,7 +74,7 @@ export async function requestClickHouse( }, body: statement, timeout: REQUEST_TIMEOUT_MS, - profile: 'configuredEndpoint', + profile: 'selfHostedService', maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, redirectPolicy: { mode: 'standard', diff --git a/apps/sim/lib/internal/jupyter/client.test.ts b/apps/sim/lib/internal/jupyter/client.test.ts index 15e70dc7c29..9f2256bf54e 100644 --- a/apps/sim/lib/internal/jupyter/client.test.ts +++ b/apps/sim/lib/internal/jupyter/client.test.ts @@ -43,7 +43,7 @@ describe('Jupyter client', () => { expect(securityMocks.validateUrlWithDNS).toHaveBeenCalledWith( 'http://jupyter.example.com:8888/base/api/kernels', 'serverUrl', - 'configuredEndpoint' + 'selfHostedService' ) expect(securityMocks.secureFetchWithPinnedIP).toHaveBeenCalledWith( 'http://jupyter.example.com:8888/base/api/kernels', @@ -55,7 +55,7 @@ describe('Jupyter client', () => { 'Content-Type': 'application/json', }, body: JSON.stringify({ name: 'python3' }), - profile: 'configuredEndpoint', + profile: 'selfHostedService', maxRedirects: 0, maxResponseBytes: 10 * 1024 * 1024, signal: controller.signal, diff --git a/apps/sim/lib/internal/jupyter/client.ts b/apps/sim/lib/internal/jupyter/client.ts index 40c988b4686..e1a10b0cb8d 100644 --- a/apps/sim/lib/internal/jupyter/client.ts +++ b/apps/sim/lib/internal/jupyter/client.ts @@ -43,7 +43,7 @@ export async function requestJupyterApi( } const url = `${base}/api/${input.path}` - const urlValidation = await validateUrlWithDNS(url, 'serverUrl', 'configuredEndpoint') + const urlValidation = await validateUrlWithDNS(url, 'serverUrl', 'selfHostedService') signal?.throwIfAborted() if (!urlValidation.isValid || !urlValidation.resolvedIP) { throw new InvalidJupyterTargetError(`Invalid Jupyter serverUrl: ${urlValidation.error}`) @@ -57,7 +57,7 @@ export async function requestJupyterApi( ...(hasBody ? { 'Content-Type': 'application/json' } : {}), }, body: hasBody ? JSON.stringify(input.body) : undefined, - profile: 'configuredEndpoint', + profile: 'selfHostedService', maxRedirects: 0, maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, signal, diff --git a/apps/sim/lib/internal/onepassword/client.test.ts b/apps/sim/lib/internal/onepassword/client.test.ts index 784b1357c34..fba97de50e8 100644 --- a/apps/sim/lib/internal/onepassword/client.test.ts +++ b/apps/sim/lib/internal/onepassword/client.test.ts @@ -151,7 +151,7 @@ describe('connectRequest', () => { 'Content-Type': 'application/json', }, body: '{"title":"Example"}', - profile: 'configuredEndpoint', + profile: 'selfHostedService', maxResponseBytes: 10 * 1024 * 1024, signal: controller.signal, }) diff --git a/apps/sim/lib/internal/onepassword/client.ts b/apps/sim/lib/internal/onepassword/client.ts index aed98c5b806..aae6c1f5a8b 100644 --- a/apps/sim/lib/internal/onepassword/client.ts +++ b/apps/sim/lib/internal/onepassword/client.ts @@ -379,7 +379,7 @@ export async function connectRequest(options: { method: options.method, headers, body: options.body ? JSON.stringify(options.body) : undefined, - profile: 'configuredEndpoint', + profile: 'selfHostedService', maxResponseBytes: options.maxResponseBytes ?? MAX_JSON_API_RESPONSE_BYTES, signal: options.signal, }) diff --git a/apps/sim/lib/knowledge/documents/utils.test.ts b/apps/sim/lib/knowledge/documents/utils.test.ts index eda1f582b90..d6a116e0674 100644 --- a/apps/sim/lib/knowledge/documents/utils.test.ts +++ b/apps/sim/lib/knowledge/documents/utils.test.ts @@ -802,7 +802,11 @@ describe('secureFetchWithRetry', () => { ) await expect( - secureFetchWithRetry('https://attacker.test', { method: 'GET' }, FAST_RETRY) + secureFetchWithRetry( + 'https://attacker.test', + { method: 'GET', profile: 'configuredEndpoint' }, + FAST_RETRY + ) ).rejects.toThrow('blocked IP address') expect(mockSecureFetchWithValidation).toHaveBeenCalledTimes(1) @@ -815,7 +819,7 @@ describe('secureFetchWithRetry', () => { const response = await secureFetchWithRetry( 'https://example.com/api', - { method: 'GET' }, + { method: 'GET', profile: 'configuredEndpoint' }, FAST_RETRY ) @@ -828,7 +832,7 @@ describe('secureFetchWithRetry', () => { const response = await secureFetchWithRetry( 'https://example.com/api', - { method: 'GET' }, + { method: 'GET', profile: 'configuredEndpoint' }, FAST_RETRY ) @@ -872,7 +876,7 @@ describe('secureFetchWithRetry', () => { const response = await secureFetchWithRetry( 'https://api.github.com/repos', - { method: 'GET' }, + { method: 'GET', profile: 'configuredEndpoint' }, FAST_RETRY ) @@ -887,7 +891,7 @@ describe('secureFetchWithRetry', () => { const response = await secureFetchWithRetry( 'https://api.github.com/repos', - { method: 'GET' }, + { method: 'GET', profile: 'configuredEndpoint' }, FAST_RETRY ) @@ -902,7 +906,7 @@ describe('secureFetchWithRetry', () => { const response = await secureFetchWithRetry( 'https://example.com/api', - { method: 'GET' }, + { method: 'GET', profile: 'configuredEndpoint' }, FAST_RETRY ) @@ -918,7 +922,7 @@ describe('secureFetchWithRetry', () => { const error = await secureFetchWithRetry( 'https://gitlab.example.com/api/v4/projects', - { method: 'GET' }, + { method: 'GET', profile: 'configuredEndpoint' }, { ...FAST_RETRY, maxRetries: 0 } ).then( () => undefined, diff --git a/apps/sim/providers/vllm/index.test.ts b/apps/sim/providers/vllm/index.test.ts index 9c6da6fc379..b520e90a4a2 100644 --- a/apps/sim/providers/vllm/index.test.ts +++ b/apps/sim/providers/vllm/index.test.ts @@ -189,7 +189,7 @@ describe('vllmProvider', () => { expect(mockValidateUrlWithDNS).toHaveBeenCalledWith( 'https://my-vllm.example.com', 'vLLM endpoint', - 'configuredEndpoint' + 'selfHostedService' ) expect(mockCreatePinnedFetch).toHaveBeenCalledWith('203.0.113.10') expect(openAIArgs[0].baseURL).toBe('https://my-vllm.example.com/v1') @@ -208,7 +208,7 @@ describe('vllmProvider', () => { expect(mockValidateUrlWithDNS).toHaveBeenCalledWith( 'https://my-vllm.example.com/v1', 'vLLM endpoint', - 'configuredEndpoint' + 'selfHostedService' ) expect(openAIArgs[0].baseURL).toBe('https://my-vllm.example.com/v1') expect(openAIArgs[0].fetch).toBe(pinnedFetchFn) diff --git a/apps/sim/providers/vllm/index.ts b/apps/sim/providers/vllm/index.ts index db5241fa95c..24c9c5135bb 100644 --- a/apps/sim/providers/vllm/index.ts +++ b/apps/sim/providers/vllm/index.ts @@ -130,7 +130,7 @@ export const vllmProvider: ProviderConfig = { const validation = await validateUrlWithDNS( userProvidedEndpoint, 'vLLM endpoint', - 'configuredEndpoint' + 'selfHostedService' ) if (!validation.isValid) { logger.warn('Blocked SSRF attempt via vLLM endpoint', { diff --git a/apps/sim/tools/github/utils.server.test.ts b/apps/sim/tools/github/utils.server.test.ts index 6223a66a47e..c4f77ba2682 100644 --- a/apps/sim/tools/github/utils.server.test.ts +++ b/apps/sim/tools/github/utils.server.test.ts @@ -16,8 +16,9 @@ vi.mock('@sim/security/dns', () => ({ vi.mock('@/lib/core/config/env-flags', () => ({ isHosted: false, - egressAllowedHosts: undefined, - egressAllowedIpRanges: undefined, + getEgressAllowedHosts: () => undefined, + getEgressAllowedIpRanges: () => undefined, + isLegacyPrivateDatabaseAccessAllowed: () => false, getProxyUrl: () => undefined, })) diff --git a/packages/security/src/egress.test.ts b/packages/security/src/egress.test.ts index ef1dfa3fbd4..b5e13d38cd5 100644 --- a/packages/security/src/egress.test.ts +++ b/packages/security/src/egress.test.ts @@ -157,6 +157,60 @@ describe('the same operator config is inert on the hosted posture', () => { }) }) +describe('loopback is vouched by name, never by resolved address', () => { + const selfHostedLoopback = createEgressPolicy({ + insecureHttp: 'whenVouched', + allowLoopback: true, + }) + + it.each([ + ['http://localhost:11434/api', '127.0.0.1', 'a local Ollama'], + ['http://127.0.0.1:8888/tree', '127.0.0.1', 'a local Jupyter'], + ['http://[::1]:8080/', '::1', 'IPv6 loopback'], + ['http://127.0.0.5:8080/', '127.0.0.5', 'the rest of 127/8'], + ])('permits %s — %s', (href, address) => { + expect(decide(selfHostedLoopback, href, address).allowed).toBe(true) + }) + + it('refuses a public hostname that merely resolves to loopback', () => { + // The carve-out keys off the hostname. Keying it off the resolved address + // would turn any attacker-controlled DNS name into a route to loopback. + expect(reason(selfHostedLoopback, 'https://localtest.me/', '127.0.0.1')).toBe( + 'address-loopback' + ) + expect(reason(selfHostedLoopback, 'https://127.0.0.1.nip.io/', '127.0.0.1')).toBe( + 'address-loopback' + ) + }) + + it('does not extend the carve-out past loopback', () => { + expect(reason(selfHostedLoopback, 'https://svc.internal/', '10.0.0.5')).toBe('address-blocked') + }) + + it('is absent when the policy does not permit loopback', () => { + expect(reason(hosted, 'http://localhost:11434/api', '127.0.0.1')).toBe('insecure-scheme') + }) +}) + +describe('allowPrivate — the deprecated blanket flag', () => { + const legacy = createEgressPolicy({ insecureHttp: 'whenVouched', allowPrivate: true }) + + it.each([ + ['10.0.0.5', 'RFC1918'], + ['192.168.1.9', 'RFC1918'], + ['172.16.0.1', 'RFC1918'], + ['100.64.0.1', 'CGNAT — where Tailscale lives'], + ['127.0.0.1', 'loopback'], + ['198.18.0.1', 'benchmarking'], + ])('vouches for %s — %s', (address) => { + expect(decide(legacy, 'https://db.internal/', address).allowed).toBe(true) + }) + + it('still cannot reach cloud metadata', () => { + expect(reason(legacy, 'https://metadata/', '169.254.169.254')).toBe('address-metadata') + }) +}) + describe('denied ports', () => { it.each([ ['22', 'SSH'], diff --git a/packages/security/src/egress.ts b/packages/security/src/egress.ts index 9803f5d92d3..5ec8d83c37d 100644 --- a/packages/security/src/egress.ts +++ b/packages/security/src/egress.ts @@ -126,6 +126,12 @@ export interface EgressPolicy { * case, not a privilege. */ readonly allowLoopback: boolean + /** + * Vouches for every private address without naming one. Exists only to + * reproduce the deprecated `ALLOW_PRIVATE_DATABASE_HOSTS`, which bypassed the + * address check outright; a named allowlist is the supported form. + */ + readonly allowPrivate: boolean readonly allowedHosts: readonly HostPattern[] readonly allowedRanges: readonly CidrRange[] } @@ -143,6 +149,8 @@ export interface EgressPolicySpec { readonly insecureHttp?: InsecureHttpPolicy /** Whether loopback destinations are vouched for without being allowlisted. */ readonly allowLoopback?: boolean + /** Whether every private address is vouched for. See {@link EgressPolicy.allowPrivate}. */ + readonly allowPrivate?: boolean /** * Names of the settings these lists came from, used verbatim in the error a * malformed entry throws so the operator knows which value to fix. @@ -212,6 +220,7 @@ export function createEgressPolicy(spec: EgressPolicySpec = {}): EgressPolicy { return { insecureHttp: spec.insecureHttp ?? 'never', allowLoopback: spec.allowLoopback ?? false, + allowPrivate: spec.allowPrivate ?? false, allowedHosts: splitEntries(spec.allowedHosts).map((entry) => parseHostPattern(entry, sourceNames.hosts) ), @@ -256,14 +265,25 @@ function isMetadataAddress(address: string): boolean { * Otherwise a resolved address inside an allowlisted range vouches for it, which * is why this cannot be decided before DNS for a hostname destination. */ +/** + * Whether the destination names itself as loopback — `localhost`, or a loopback + * IP literal. + * + * The loopback carve-out keys off this rather than off the resolved address, so + * a public hostname that happens to resolve to `127.0.0.1` does not inherit it. + * Letting the address decide would make every DNS name an attacker controls a + * route to the deployment's own loopback services. + */ +function isLoopbackDestination(host: string): boolean { + const clean = unwrapIpv6Brackets(host) + return isLoopbackHostname(clean) || isLoopbackIp(clean) +} + function isVouched(url: URL, address: string | undefined, policy: EgressPolicy): boolean { if (matchesHostAllowlist(url.hostname, policy)) return true - // `localhost` is loopback by name, so a loopback-permitting policy can vouch - // for it before DNS — which is what lets the synchronous check accept a local - // dev server without pretending to know where an arbitrary hostname points. - if (policy.allowLoopback && isLoopbackHostname(url.hostname)) return true + if (policy.allowLoopback && isLoopbackDestination(url.hostname)) return true if (address === undefined) return false - if (policy.allowLoopback && isLoopbackIp(unwrapIpv6Brackets(address))) return true + if (policy.allowPrivate && isPrivateIp(unwrapIpv6Brackets(address))) return true return matchesRangeAllowlist(address, policy) } @@ -338,7 +358,12 @@ export function evaluateUrl(url: URL, policy: EgressPolicy): EgressDecision { * through ones it does not. */ export function policyCanVouch(policy: EgressPolicy): boolean { - return policy.allowedHosts.length > 0 || policy.allowedRanges.length > 0 || policy.allowLoopback + return ( + policy.allowedHosts.length > 0 || + policy.allowedRanges.length > 0 || + policy.allowLoopback || + policy.allowPrivate + ) } /** diff --git a/packages/testing/src/mocks/env-flags.mock.ts b/packages/testing/src/mocks/env-flags.mock.ts index c13bcc9c19c..24868e1af2b 100644 --- a/packages/testing/src/mocks/env-flags.mock.ts +++ b/packages/testing/src/mocks/env-flags.mock.ts @@ -20,6 +20,7 @@ export interface EnvFlagsMockState { isAuthDisabled: boolean egressAllowedHosts: string | undefined egressAllowedIpRanges: string | undefined + legacyPrivateDatabaseAccess: boolean isRegistrationDisabled: boolean isEmailPasswordEnabled: boolean isSignupMxValidationEnabled: boolean @@ -70,6 +71,7 @@ const defaultEnvFlagsState: EnvFlagsMockState = { isAuthDisabled: false, egressAllowedHosts: undefined, egressAllowedIpRanges: undefined, + legacyPrivateDatabaseAccess: false, isRegistrationDisabled: false, isEmailPasswordEnabled: true, isSignupMxValidationEnabled: false, @@ -119,6 +121,18 @@ const envFlagsState: EnvFlagsMockState = { ...defaultEnvFlagsState } * {@link resetEnvFlagsMock} restores the default implementations. */ export const envFlagsMockFns = { + /** + * Egress config is exposed as functions by the real module, but held as + * mutable state here so a test can still write + * `envFlagsMock.egressAllowedHosts = '...'` and have the read observe it. + */ + getEgressAllowedHosts: vi.fn<() => string | undefined>(() => envFlagsState.egressAllowedHosts), + getEgressAllowedIpRanges: vi.fn<() => string | undefined>( + () => envFlagsState.egressAllowedIpRanges + ), + isLegacyPrivateDatabaseAccessAllowed: vi.fn<() => boolean>( + () => envFlagsState.legacyPrivateDatabaseAccess + ), getAllowedIntegrationsFromEnv: vi.fn<() => string[] | null>(() => null), getPreviewBlocksFromEnv: vi.fn<() => string[]>(() => []), getBlacklistedProvidersFromEnv: vi.fn<() => string[]>(() => []), @@ -153,6 +167,15 @@ export function resetEnvFlagsMock(): void { envFlagsMockFns.getBlacklistedProvidersFromEnv.mockReset().mockImplementation(() => []) envFlagsMockFns.getAllowedMcpDomainsFromEnv.mockReset().mockImplementation(() => null) envFlagsMockFns.getCostMultiplier.mockReset().mockImplementation(() => 1) + envFlagsMockFns.getEgressAllowedHosts + .mockReset() + .mockImplementation(() => envFlagsState.egressAllowedHosts) + envFlagsMockFns.getEgressAllowedIpRanges + .mockReset() + .mockImplementation(() => envFlagsState.egressAllowedIpRanges) + envFlagsMockFns.isLegacyPrivateDatabaseAccessAllowed + .mockReset() + .mockImplementation(() => envFlagsState.legacyPrivateDatabaseAccess) } /** diff --git a/scripts/check-egress-boundary.ts b/scripts/check-egress-boundary.ts index ec428fef55c..5e6ac0da112 100644 --- a/scripts/check-egress-boundary.ts +++ b/scripts/check-egress-boundary.ts @@ -23,7 +23,15 @@ import path from 'node:path' const ROOT = path.resolve(import.meta.dir, '..') -const SCAN_DIRS = ['apps/sim/app', 'apps/sim/lib', 'apps/sim/tools', 'apps/sim/connectors'] +const SCAN_DIRS = [ + 'apps/sim/app', + 'apps/sim/lib', + 'apps/sim/tools', + 'apps/sim/connectors', + 'apps/sim/executor', + 'apps/sim/providers', + 'apps/sim/triggers', +] const SKIP_DIRS = new Set(['node_modules', 'dist', '.next', '.turbo', 'coverage']) From 92ab5a61459201c62b3778a37929be868029dfc5 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 14:05:48 -0700 Subject: [PATCH 03/20] fix(egress): merge duplicate afterEach in the alias suite --- apps/sim/lib/core/security/input-validation.test.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/apps/sim/lib/core/security/input-validation.test.ts b/apps/sim/lib/core/security/input-validation.test.ts index 3cc21cbb4c5..8b2bcef3797 100644 --- a/apps/sim/lib/core/security/input-validation.test.ts +++ b/apps/sim/lib/core/security/input-validation.test.ts @@ -570,11 +570,6 @@ describe('validateDatabaseHost', () => { }) describe('deprecated ALLOW_PRIVATE_DATABASE_HOSTS alias', () => { - afterEach(() => { - envFlagsMock.egressAllowedHosts = undefined - envFlagsMock.egressAllowedIpRanges = undefined - }) - afterEach(() => { envFlagsMock.legacyPrivateDatabaseAccess = false }) From ee83f71def8e80e6b74cc4c6839c8f98b57a8ff8 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 14:17:30 -0700 Subject: [PATCH 04/20] fix(egress): drop credentials on cross-origin redirects by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stripping was gated on a `redirectPolicy` being supplied at all, so the many callers that pass none — the data-drain webhook and Slack's `url_private` upload among them — handed their Authorization header to whatever host a redirect named. Two reviewers flagged the same thing at two different call sites, which is the tell that the default was wrong rather than those sites. Keeping credentials is now the explicit opt-in it reads as, and `host` always goes, since it describes the origin being left. --- .../core/security/input-validation.server.ts | 32 ++++++++++--------- .../pinned-redirect-replay.server.test.ts | 30 +++++++++++++++-- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index a5db1456dbc..d5f42087c22 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -1054,21 +1054,23 @@ export async function secureFetchWithPinnedIP( if (redirectHeaders && hop.dropBody) { redirectHeaders = stripHeaders(redirectHeaders, ENTITY_HEADERS) } - if ( - redirectHeaders && - redirectPolicy && - isCrossOrigin && - (redirectPolicy.mode === 'standard' || - !redirectPolicy.sendCredentialsOnCrossOriginRedirect) - ) { - const sensitiveHeaders = redirectPolicy.sendCredentialsOnCrossOriginRedirect - ? ['host'] - : [ - 'host', - ...CROSS_ORIGIN_CREDENTIAL_HEADERS, - ...(redirectPolicy.sensitiveHeaders ?? []), - ] - redirectHeaders = stripHeaders(redirectHeaders, sensitiveHeaders) + // Credentials are dropped on a cross-origin hop unless a policy + // explicitly asks to keep them. Gating this on a policy being + // supplied at all, as it once was, meant the many callers that pass + // none handed their Authorization header to whatever host the + // redirect named. `host` always goes: it describes the old origin. + if (redirectHeaders && isCrossOrigin) { + const keepCredentials = redirectPolicy?.sendCredentialsOnCrossOriginRedirect === true + redirectHeaders = stripHeaders( + redirectHeaders, + keepCredentials + ? ['host'] + : [ + 'host', + ...CROSS_ORIGIN_CREDENTIAL_HEADERS, + ...(redirectPolicy?.sensitiveHeaders ?? []), + ] + ) } if (redirectHeaders && options.stripAuthOnRedirect) { redirectHeaders = stripHeaders(redirectHeaders, ['authorization']) diff --git a/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts b/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts index 58046fec89d..3b1473d162c 100644 --- a/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts +++ b/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts @@ -81,7 +81,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { expect(hops).toEqual([]) }) - it('preserves historical replay when no redirect policy is present', async () => { + it('replays method and body without a policy, but not credentials cross-origin', async () => { const hops: RecordedHop[] = [] const target = await startRecordingServer(hops) const origin = await startServer((req, res) => { @@ -103,10 +103,34 @@ describe('secureFetchWithPinnedIP redirect replay', () => { expect(response.status).toBe(200) expect(hops).toHaveLength(1) + // Legacy replay semantics: the method and body survive a 303. expect(hops[0].method).toBe('POST') expect(hops[0].body).toBe('{"message":"legacy"}') - expect(hops[0].headers.authorization).toBe('Bearer legacy-token') - expect(hops[0].headers.host).toBe('legacy.example') + // ...but the credentials do not travel to another origin just because this + // caller passed no redirect policy. + expect(hops[0].headers.authorization).toBeUndefined() + expect(hops[0].headers.host).not.toBe('legacy.example') + }) + + it('keeps credentials cross-origin only when a policy explicitly opts in', async () => { + const hops: RecordedHop[] = [] + const target = await startRecordingServer(hops) + const origin = await startServer((req, res) => { + req.resume() + res.writeHead(303, { location: `${target}/after` }) + res.end() + }) + + await secureFetchWithPinnedIP(origin, '127.0.0.1', { + method: 'POST', + body: '{"message":"opt-in"}', + headers: { Authorization: 'Bearer keep-me', 'Content-Type': 'application/json' }, + redirectPolicy: { mode: 'legacy', sendCredentialsOnCrossOriginRedirect: true }, + profile: 'configuredEndpoint', + }) + + expect(hops).toHaveLength(1) + expect(hops[0].headers.authorization).toBe('Bearer keep-me') }) it('lets a legacy block withhold credentials without changing its replay semantics', async () => { From 938337cf677c689c1af09a31c4ce8dc8de43fdc5 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 14:24:22 -0700 Subject: [PATCH 05/20] fix(egress): close review findings on metadata folding, loopback and provenance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two of these are bugs in the new guard itself. `::a9fe:a9fe` is what the WHATWG URL parser normalizes `::169.254.169.254` to, and `ipaddr.process` folds the IPv4-mapped form to IPv4 but not the deprecated IPv4-compatible one — so the metadata comparison missed it. `isPrivateIp` still caught it for an ordinary destination, but an allowlisted one skipped straight past the class check. Addresses are now folded to a canonical IPv4 before the comparison. The loopback carve-out vouched on the hostname alone, so a resolver answering `localhost` with a routable address kept the exemption and got pinned to it. The address has to land on loopback too, which is what the guard did before. Also from review: - The boundary script matched line by line, so an import list broken across lines walked past it. It reads the whole source now. - Windchill's `ReplicaUrl` arrives in a Stage 1 response and the link-preview URL is harvested from rendered content; both are `contentFetch`, not configured destinations. - Textract fetches two different things through one helper: a caller-supplied document URL, and a presigned URL Sim minted against its own storage, which on a self-hosted deployment legitimately points at a private MinIO. Labelling both `contentFetch` would have broken internal documents there, so the provenance is threaded through instead. --- apps/sim/app/api/link-preview/route.ts | 4 +- .../lib/internal/textract/document-input.ts | 18 +++++-- apps/sim/lib/internal/windchill/client.ts | 4 +- packages/security/src/egress.test.ts | 17 +++++++ packages/security/src/egress.ts | 47 +++++++++++++++++-- scripts/check-egress-boundary.ts | 23 ++++++--- 6 files changed, 96 insertions(+), 17 deletions(-) diff --git a/apps/sim/app/api/link-preview/route.ts b/apps/sim/app/api/link-preview/route.ts index 75fb85bae54..8aaba14c0db 100644 --- a/apps/sim/app/api/link-preview/route.ts +++ b/apps/sim/app/api/link-preview/route.ts @@ -53,7 +53,9 @@ function parsePreview(html: string): LinkPreview { async function fetchPreview(url: string): Promise { const response = await secureFetchWithValidation(url, { - profile: 'requestTarget', + // The URL is harvested from a rendered link rather than authored as a + // destination, so it gets no reach into a private network. + profile: 'contentFetch', timeout: FETCH_TIMEOUT_MS, maxRedirects: MAX_REDIRECTS, maxResponseBytes: MAX_RESPONSE_BYTES, diff --git a/apps/sim/lib/internal/textract/document-input.ts b/apps/sim/lib/internal/textract/document-input.ts index f683f142dd1..29f97878c9f 100644 --- a/apps/sim/lib/internal/textract/document-input.ts +++ b/apps/sim/lib/internal/textract/document-input.ts @@ -1,6 +1,7 @@ import type { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { NextResponse } from 'next/server' +import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { validateS3BucketName } from '@/lib/core/security/input-validation' import { secureFetchWithPinnedIP, @@ -36,18 +37,25 @@ export type ResolveDocumentResult = | { ok: true; document: ResolvedDocument } | { ok: false; response: NextResponse } +/** + * `profile` distinguishes the two kinds of URL that reach here: a document URL + * the caller supplied, and a presigned URL Sim minted against its own configured + * object storage — which on a self-hosted deployment legitimately points at a + * private or loopback MinIO. + */ async function fetchDocumentBytes( url: string, + profile: EgressProfile, signal?: AbortSignal ): Promise<{ bytes: Buffer; contentType: string }> { signal?.throwIfAborted() - const urlValidation = await validateUrlWithDNS(url, 'Document URL', 'contentFetch') + const urlValidation = await validateUrlWithDNS(url, 'Document URL', profile) if (!urlValidation.isValid) { throw new TextractOperationError(urlValidation.error || 'Invalid document URL', 400) } const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP!, { - profile: 'contentFetch', + profile, method: 'GET', signal, }) @@ -177,7 +185,11 @@ export async function resolveDocumentInput( } } - const fetched = await fetchDocumentBytes(fileUrl, signal) + const fetched = await fetchDocumentBytes( + fileUrl, + isInternalFilePath ? 'configuredEndpoint' : 'contentFetch', + signal + ) return { ok: true, document: { diff --git a/apps/sim/lib/internal/windchill/client.ts b/apps/sim/lib/internal/windchill/client.ts index cfa9adaf3fc..2b0059c37b7 100644 --- a/apps/sim/lib/internal/windchill/client.ts +++ b/apps/sim/lib/internal/windchill/client.ts @@ -320,7 +320,9 @@ export async function uploadWindchillContent({ const stageTwoResponse = await secureFetchWithValidation( descriptor.replicaUrl, { - profile: 'configuredEndpoint', + // Windchill hands this URL back in the Stage 1 response, so it is + // response-derived rather than configured. + profile: 'contentFetch', method: 'POST', headers: { 'Content-Type': multipart.contentType, Accept: 'application/json' }, body: multipart.body, diff --git a/packages/security/src/egress.test.ts b/packages/security/src/egress.test.ts index b5e13d38cd5..a58ac6dc3b2 100644 --- a/packages/security/src/egress.test.ts +++ b/packages/security/src/egress.test.ts @@ -103,6 +103,15 @@ describe('cloud metadata is never reachable', () => { it('blocks the AWS IPv6 metadata address', () => { expect(reason(hosted, 'https://[fd00:ec2::254]/')).toBe('address-metadata') }) + + it.each([ + ['::a9fe:a9fe', 'the IPv4-compatible form the URL parser normalizes to'], + ['::ffff:169.254.169.254', 'the IPv4-mapped form'], + ['::169.254.169.254', 'written long-hand'], + ])('blocks %s — %s', (address) => { + const permissive = createEgressPolicy({ allowedHosts: 'metadata.internal' }) + expect(reason(permissive, 'https://metadata.internal/', address)).toBe('address-metadata') + }) }) describe('operator allowlist — the self-hosted posture', () => { @@ -190,6 +199,14 @@ describe('loopback is vouched by name, never by resolved address', () => { it('is absent when the policy does not permit loopback', () => { expect(reason(hosted, 'http://localhost:11434/api', '127.0.0.1')).toBe('insecure-scheme') }) + + it('refuses when a resolver answers localhost with a routable address', () => { + // The carve-out is for the loopback interface, not for whatever a resolver + // decides `localhost` means today. + expect(reason(selfHostedLoopback, 'https://localhost/api', '93.184.216.34')).toBe(null) + expect(reason(selfHostedLoopback, 'https://localhost/api', '10.0.0.7')).toBe('address-blocked') + expect(reason(selfHostedLoopback, 'http://localhost/api', '10.0.0.7')).toBe('insecure-scheme') + }) }) describe('allowPrivate — the deprecated blanket flag', () => { diff --git a/packages/security/src/egress.ts b/packages/security/src/egress.ts index 5ec8d83c37d..7e8da577134 100644 --- a/packages/security/src/egress.ts +++ b/packages/security/src/egress.ts @@ -252,11 +252,41 @@ function matchesRangeAllowlist(address: string, policy: EgressPolicy): boolean { ) } -function isMetadataAddress(address: string): boolean { +/** + * Canonical form of an address, folding every IPv4-in-IPv6 spelling down to the + * IPv4 it carries. `ipaddr.process` handles the IPv4-mapped form (`::ffff:x`) + * but leaves the deprecated IPv4-compatible one (`::a.b.c.d`, which the WHATWG + * URL parser normalizes to `::a9fe:a9fe`), so comparing without this misses the + * metadata endpoint written that way. + */ +function canonicalAddress(address: string): string | null { const clean = unwrapIpv6Brackets(address) - if (!ipaddr.isValid(clean)) return false - const normalized = ipaddr.process(clean).toString() - return METADATA_ADDRESSES.some((candidate) => ipaddr.process(candidate).toString() === normalized) + if (!ipaddr.isValid(clean)) return null + + const parsed = ipaddr.process(clean) + if (parsed.kind() === 'ipv6') { + const parts = (parsed as ipaddr.IPv6).parts + if (parts.slice(0, 6).every((part) => part === 0)) { + return ipaddr + .fromByteArray([ + (parts[6] >> 8) & 0xff, + parts[6] & 0xff, + (parts[7] >> 8) & 0xff, + parts[7] & 0xff, + ]) + .toString() + } + } + return parsed.toString() +} + +const CANONICAL_METADATA_ADDRESSES: ReadonlySet = new Set( + METADATA_ADDRESSES.map((address) => canonicalAddress(address) ?? address) +) + +function isMetadataAddress(address: string): boolean { + const canonical = canonicalAddress(address) + return canonical !== null && CANONICAL_METADATA_ADDRESSES.has(canonical) } /** @@ -281,7 +311,14 @@ function isLoopbackDestination(host: string): boolean { function isVouched(url: URL, address: string | undefined, policy: EgressPolicy): boolean { if (matchesHostAllowlist(url.hostname, policy)) return true - if (policy.allowLoopback && isLoopbackDestination(url.hostname)) return true + + if (policy.allowLoopback && isLoopbackDestination(url.hostname)) { + // The address must land on loopback too, so a resolver answering + // `localhost` with a routable address cannot borrow the carve-out. Before + // DNS there is no address to judge, and evaluateAddress rules later. + return address === undefined || isLoopbackIp(unwrapIpv6Brackets(address)) + } + if (address === undefined) return false if (policy.allowPrivate && isPrivateIp(unwrapIpv6Brackets(address))) return true return matchesRangeAllowlist(address, policy) diff --git a/scripts/check-egress-boundary.ts b/scripts/check-egress-boundary.ts index 5e6ac0da112..93de4c99a47 100644 --- a/scripts/check-egress-boundary.ts +++ b/scripts/check-egress-boundary.ts @@ -35,9 +35,14 @@ const SCAN_DIRS = [ const SKIP_DIRS = new Set(['node_modules', 'dist', '.next', '.turbo', 'coverage']) -/** Raw HTTP transports. Reaching one directly bypasses DNS pinning. */ +/** + * Raw HTTP transports. Reaching one directly bypasses DNS pinning. + * + * Matched against the whole source rather than line by line, because an import + * list broken across lines would otherwise slip past. + */ const TRANSPORT_IMPORT = - /^\s*import\s[^'"]*from\s+['"](?:node:)?(http|https|undici|http-proxy-agent|https-proxy-agent)['"]/ + /^[ \t]*import\b[\s\S]*?from\s*['"](?:node:)?(?:http|https|undici|http-proxy-agent|https-proxy-agent)['"]/gm /** * Modules allowed to hold a transport import, each because it *is* part of the @@ -78,11 +83,15 @@ function main() { const rel = path.relative(ROOT, file).split(path.sep).join('/') if (ALLOWED.has(rel)) continue scanned++ - const lines = readFileSync(file, 'utf8').split('\n') - for (let i = 0; i < lines.length; i++) { - if (TRANSPORT_IMPORT.test(lines[i])) { - violations.push({ file: rel, line: i + 1, snippet: lines[i].trim() }) - } + const source = readFileSync(file, 'utf8') + TRANSPORT_IMPORT.lastIndex = 0 + for (const match of source.matchAll(TRANSPORT_IMPORT)) { + const line = source.slice(0, match.index).split('\n').length + violations.push({ + file: rel, + line, + snippet: match[0].replace(/\s+/g, ' ').trim(), + }) } } } From 5100740757e07723073027c658dc2a7ade7c97e9 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 14:51:45 -0700 Subject: [PATCH 06/20] fix(egress): refuse to replay a request body across an origin boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The credential fix stripped headers. It did not touch the body, and a 307 or 308 preserves the body verbatim — so a server that redirects a credential-bearing POST across origins still handed the payload to whatever host it named. Agiloft's `EWLogin` form carries `$password` that way; 30 call sites send a body with redirects enabled, 28 of them with no policy at all. `followRedirectsGuarded` has always refused this on the undici path. The node path now does the same, with `allowCrossOriginBody` on `HttpRedirectPolicy` for a flow that genuinely needs it. Nothing opts in today. Refusing rather than dropping the body: a bodyless replay of a POST is a different request, and the caller has no way to tell it happened. The GitHub cross-origin test asserted only that the token was withheld; the request is now rejected outright, which subsumes it — the comment body no longer reaches the redirect target either. --- .../lib/core/security/http-redirect-policy.ts | 14 ++++ .../core/security/input-validation.server.ts | 19 ++++- .../pinned-redirect-replay.server.test.ts | 75 ++++++++++++++++--- apps/sim/tools/github/utils.server.test.ts | 23 +++--- 4 files changed, 110 insertions(+), 21 deletions(-) diff --git a/apps/sim/lib/core/security/http-redirect-policy.ts b/apps/sim/lib/core/security/http-redirect-policy.ts index 1ed54dcaf60..584c38e4e71 100644 --- a/apps/sim/lib/core/security/http-redirect-policy.ts +++ b/apps/sim/lib/core/security/http-redirect-policy.ts @@ -9,4 +9,18 @@ export interface HttpRedirectPolicy { mode: 'legacy' | 'standard' sendCredentialsOnCrossOriginRedirect: boolean sensitiveHeaders?: readonly string[] + /** + * Permits replaying the request body to a redirect target on another origin. + * + * Off by default, and rarely the right answer: a 307 or 308 preserves the body + * verbatim, so a server that redirects a credential-bearing POST — a login + * form, a signed webhook payload — hands it to whatever host it names. RFC 9110 + * allows the replay because it assumes the redirect comes from a server you + * already trust with the body, which is exactly the assumption a cross-origin + * hop breaks. + * + * Set this only where a provider documents a cross-origin redirect that needs + * the body, and where the body carries nothing the target should not see. + */ + allowCrossOriginBody?: boolean } diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index d5f42087c22..852e139b3a2 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -1075,10 +1075,27 @@ export async function secureFetchWithPinnedIP( if (redirectHeaders && options.stripAuthOnRedirect) { redirectHeaders = stripHeaders(redirectHeaders, ['authorization']) } + const redirectBody = hop.dropBody ? undefined : options.body + // Refusing rather than quietly dropping the body: a bodyless replay + // of a POST is a different request, and the caller cannot tell it + // happened. Matches followRedirectsGuarded, which has always refused. + if ( + isCrossOrigin && + redirectBody !== undefined && + redirectBody !== null && + redirectPolicy?.allowCrossOriginBody !== true + ) { + settledReject( + new Error( + 'Blocked by SSRF policy: cross-origin redirect would forward a request body' + ) + ) + return + } const redirectOptions: SecureFetchOptions = { ...options, method: hop.method, - body: hop.dropBody ? undefined : options.body, + body: redirectBody, headers: redirectHeaders, } return secureFetchWithPinnedIP( diff --git a/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts b/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts index 3b1473d162c..a1006d0b241 100644 --- a/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts +++ b/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts @@ -81,7 +81,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { expect(hops).toEqual([]) }) - it('replays method and body without a policy, but not credentials cross-origin', async () => { + it('strips credentials on a cross-origin hop when no policy is supplied', async () => { const hops: RecordedHop[] = [] const target = await startRecordingServer(hops) const origin = await startServer((req, res) => { @@ -91,11 +91,10 @@ describe('secureFetchWithPinnedIP redirect replay', () => { }) const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { - method: 'POST', - body: '{"message":"legacy"}', + method: 'GET', headers: { Authorization: 'Bearer legacy-token', - 'Content-Type': 'application/json', + 'X-Trace': 'keep-me', Host: 'legacy.example', }, profile: 'configuredEndpoint', @@ -103,13 +102,58 @@ describe('secureFetchWithPinnedIP redirect replay', () => { expect(response.status).toBe(200) expect(hops).toHaveLength(1) - // Legacy replay semantics: the method and body survive a 303. - expect(hops[0].method).toBe('POST') - expect(hops[0].body).toBe('{"message":"legacy"}') - // ...but the credentials do not travel to another origin just because this - // caller passed no redirect policy. expect(hops[0].headers.authorization).toBeUndefined() expect(hops[0].headers.host).not.toBe('legacy.example') + // Non-credential headers still travel. + expect(hops[0].headers['x-trace']).toBe('keep-me') + }) + + it('refuses to replay a body to another origin', async () => { + const hops: RecordedHop[] = [] + const target = await startRecordingServer(hops) + // 307 preserves the method and body verbatim, which is exactly the case + // that would hand an Agiloft-style `$password` form to the redirect target. + const origin = await startServer((req, res) => { + req.resume() + res.writeHead(307, { location: `${target}/after` }) + res.end() + }) + + await expect( + secureFetchWithPinnedIP(origin, '127.0.0.1', { + method: 'POST', + body: '$login=admin&$password=hunter2', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + profile: 'configuredEndpoint', + }) + ).rejects.toThrow('cross-origin redirect would forward a request body') + + expect(hops).toHaveLength(0) + }) + + it('replays a body to another origin only when a policy opts in', async () => { + const hops: RecordedHop[] = [] + const target = await startRecordingServer(hops) + const origin = await startServer((req, res) => { + req.resume() + res.writeHead(307, { location: `${target}/after` }) + res.end() + }) + + await secureFetchWithPinnedIP(origin, '127.0.0.1', { + method: 'POST', + body: '{"intentional":true}', + headers: { 'Content-Type': 'application/json' }, + redirectPolicy: { + mode: 'legacy', + sendCredentialsOnCrossOriginRedirect: false, + allowCrossOriginBody: true, + }, + profile: 'configuredEndpoint', + }) + + expect(hops).toHaveLength(1) + expect(hops[0].body).toBe('{"intentional":true}') }) it('keeps credentials cross-origin only when a policy explicitly opts in', async () => { @@ -125,7 +169,11 @@ describe('secureFetchWithPinnedIP redirect replay', () => { method: 'POST', body: '{"message":"opt-in"}', headers: { Authorization: 'Bearer keep-me', 'Content-Type': 'application/json' }, - redirectPolicy: { mode: 'legacy', sendCredentialsOnCrossOriginRedirect: true }, + redirectPolicy: { + mode: 'legacy', + sendCredentialsOnCrossOriginRedirect: true, + allowCrossOriginBody: true, + }, profile: 'configuredEndpoint', }) @@ -154,6 +202,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { redirectPolicy: { mode: 'legacy', sendCredentialsOnCrossOriginRedirect: false, + allowCrossOriginBody: true, sensitiveHeaders: ['x-api-key'], }, profile: 'configuredEndpoint', @@ -190,6 +239,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false, + allowCrossOriginBody: true, sensitiveHeaders: ['x-api-key'], }, profile: 'configuredEndpoint', @@ -227,6 +277,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false, + allowCrossOriginBody: true, }, profile: 'configuredEndpoint', }) @@ -252,6 +303,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false, + allowCrossOriginBody: true, }, profile: 'configuredEndpoint', }) @@ -280,6 +332,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false, + allowCrossOriginBody: true, }, profile: 'configuredEndpoint', }) @@ -308,6 +361,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: true, + allowCrossOriginBody: true, }, profile: 'configuredEndpoint', }) @@ -346,6 +400,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false, + allowCrossOriginBody: true, }, profile: 'configuredEndpoint', }) diff --git a/apps/sim/tools/github/utils.server.test.ts b/apps/sim/tools/github/utils.server.test.ts index c4f77ba2682..112037c5a30 100644 --- a/apps/sim/tools/github/utils.server.test.ts +++ b/apps/sim/tools/github/utils.server.test.ts @@ -85,7 +85,7 @@ describe('secureGitHubRequest redirects', () => { expect(hops[0].headers.authorization).toBeUndefined() }) - it('does not forward the GitHub token when a comment POST crosses an origin boundary', async () => { + it('refuses a comment POST that crosses an origin boundary', async () => { const hops: RecordedHop[] = [] const attacker = await startRecordingServer(hops) const origin = await startServer((req, res) => { @@ -94,15 +94,18 @@ describe('secureGitHubRequest redirects', () => { res.end() }) - await secureGitHubRequest(origin, { - method: 'POST', - headers: { ...GITHUB_HEADERS, 'Content-Type': 'application/json' }, - body: '{"body":"Looks good"}', - }) - - expect(hops).toHaveLength(1) - expect(hops[0].headers.authorization).toBeUndefined() - expect(hops[0].headers.cookie).toBeUndefined() + // Stronger than withholding the token: the comment body never reaches the + // redirect target either, so nothing is disclosed and nothing is written + // somewhere the caller did not address. + await expect( + secureGitHubRequest(origin, { + method: 'POST', + headers: { ...GITHUB_HEADERS, 'Content-Type': 'application/json' }, + body: '{"body":"Looks good"}', + }) + ).rejects.toThrow('cross-origin redirect would forward a request body') + + expect(hops).toHaveLength(0) }) it('replays a comment POST as a POST across a same-origin renamed-repository 301', async () => { From 1bfa6d890f115ec1a924180f1ac76ca97143e8e0 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 15:47:58 -0700 Subject: [PATCH 07/20] refactor(egress): retire the last three parallel SSRF implementations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes the duplicates left behind by the profile migration, so one policy governs every outbound request. `allowRedirectToIp` was a hand-threaded carve-out permitting exactly one pinned address across a redirect. The undici path now carries the request's egress profile instead, and `assertGuardedRedirectTarget` asks the policy — which can express "this range is permitted" where the carve-out could only ever express "this one address". An earlier commit claimed this was already deleted; it was not, and this is where it actually happens. `lib/internal/onepassword` resolved and classified addresses itself, permitting every private range on self-hosted with no way to configure it. `lib/mcp/ domain-check` did the same, and had two paths that returned `null` to mean "run this request with no guard at all" — one of them whenever `ALLOWED_MCP_DOMAINS` was configured, which left an allowlisted domain free to redirect anywhere, cloud metadata included. Domain governance and the address check are separate questions and both now apply. Both use `selfHostedService`, which grows a `denyServicePorts: false` dial: an operator-run service binds whatever port it likes, and refusing the eight non-HTTP service ports there would have been a silent narrowing. Fixes a bug the migration surfaced: `allowLoopback` was computed from `isHosted` when the profile table was first imported, so the hosted branch used the wrong policy and could not be exercised by a test. The posture is part of the config the policy cache is keyed on now. Breaking, for self-hosted deployments only, each with the same remedy — name the destination in EGRESS_ALLOWED_HOSTS / EGRESS_ALLOWED_IP_RANGES: - a 1Password Connect server on a private address other than loopback - an MCP server on a private address, or reached through a DNS name pointed at loopback (the carve-out keys off the hostname, not the address it resolves to) - an MCP server on a private address while ALLOWED_MCP_DOMAINS is configured Test suites that duplicated the guard's own address cases are collapsed with the implementations they were testing. Several used 203.0.113.x as a stand-in for a public address; it is TEST-NET-3, and only ever passed because the old carve-out permitted whatever address was pinned. --- .../mcp/servers/test-connection/route.test.ts | 1 + apps/sim/lib/core/security/egress/profiles.ts | 57 +++++--- .../guarded-request-fetch.server.test.ts | 20 +-- .../core/security/input-validation.server.ts | 65 +++++---- .../core/security/pinned-fetch.server.test.ts | 52 ++++--- .../lib/internal/onepassword/client.test.ts | 112 +++------------ apps/sim/lib/internal/onepassword/client.ts | 92 +++--------- apps/sim/lib/mcp/domain-check.test.ts | 67 +++++---- apps/sim/lib/mcp/domain-check.ts | 135 +++++------------- apps/sim/lib/mcp/oauth/probe.test.ts | 4 +- apps/sim/lib/mcp/oauth/probe.ts | 5 +- apps/sim/lib/mcp/oauth/revoke.test.ts | 1 + .../orchestration/server-lifecycle.test.ts | 1 + apps/sim/lib/mcp/pinned-fetch.test.ts | 5 +- apps/sim/lib/mcp/pinned-fetch.ts | 12 +- apps/sim/lib/mcp/service-pool.test.ts | 1 + apps/sim/lib/mcp/service.test.ts | 1 + .../providers/azure-anthropic/index.test.ts | 4 +- apps/sim/providers/azure-anthropic/index.ts | 2 +- apps/sim/providers/azure-openai/index.test.ts | 8 +- apps/sim/providers/azure-openai/index.ts | 2 +- apps/sim/providers/vllm/index.test.ts | 4 +- apps/sim/providers/vllm/index.ts | 2 +- apps/sim/tools/bitbucket/utils.server.ts | 2 +- packages/security/src/egress.ts | 10 +- 25 files changed, 279 insertions(+), 386 deletions(-) diff --git a/apps/sim/app/api/mcp/servers/test-connection/route.test.ts b/apps/sim/app/api/mcp/servers/test-connection/route.test.ts index fa95bb499c5..2b685bd6bc5 100644 --- a/apps/sim/app/api/mcp/servers/test-connection/route.test.ts +++ b/apps/sim/app/api/mcp/servers/test-connection/route.test.ts @@ -50,6 +50,7 @@ vi.mock('@/lib/mcp/client', () => ({ })) vi.mock('@/lib/mcp/domain-check', () => ({ + MCP_EGRESS_PROFILE: 'selfHostedService', McpDnsResolutionError: class extends Error {}, McpDomainNotAllowedError: class extends Error {}, McpSsrfError: MockMcpSsrfError, diff --git a/apps/sim/lib/core/security/egress/profiles.ts b/apps/sim/lib/core/security/egress/profiles.ts index 868f1d06557..8f62d904a32 100644 --- a/apps/sim/lib/core/security/egress/profiles.ts +++ b/apps/sim/lib/core/security/egress/profiles.ts @@ -41,9 +41,10 @@ import { * exactly where Sim's own database and Redis listen, so reaching them has to * be named rather than assumed. * - `selfHostedService` — a configured endpoint for software normally run - * on-prem without TLS: vLLM, Jupyter, 1Password Connect, ClickHouse. Same - * reachability as `configuredEndpoint`, but plain HTTP is expected rather than - * conditional, which is what these integrations relied on before. + * on-prem: vLLM, Jupyter, 1Password Connect, ClickHouse, an MCP server. Same + * reachability as `configuredEndpoint`, but plain HTTP and an arbitrary port + * are expected rather than conditional, because that is how these are + * ordinarily deployed inside a network. * - `proxy` — the egress proxy itself. Held to the strictest rule of all, * because it is the component that decides where everything else may go: plain * HTTP by protocol, but public destinations only, and no allowlist. @@ -67,14 +68,19 @@ interface ProfileSpec { readonly honorsAllowlist: boolean /** When plain HTTP is acceptable for this provenance. */ readonly insecureHttp: InsecureHttpPolicy + /** Whether the non-HTTP service ports are refused. Defaults to refusing them. */ + readonly denyServicePorts?: boolean /** - * Whether loopback is reachable without being allowlisted. True off the hosted - * platform for the two profiles whose URLs someone deliberately configured — + * Whether loopback is reachable without being allowlisted, off the hosted + * platform. True for the profiles whose URLs someone deliberately configured — * a single-tenant deployment pointing at its own `localhost` (Ollama, a local - * Jupyter, a sidecar) is the ordinary case. Never true for `contentFetch`, and - * never true on the hosted platform, where `localhost` is Sim's own process. + * Jupyter, a sidecar) is the ordinary case. Never for `contentFetch`, and never + * on the hosted platform, where `localhost` is Sim's own process. + * + * Combined with the posture at build time rather than captured here, so the + * hosted branch is reachable from a test. */ - readonly allowLoopback: boolean + readonly allowLoopbackOffHosted: boolean /** * Whether the deprecated `ALLOW_PRIVATE_DATABASE_HOSTS` applies. Only * `databaseHost` sets this, because that is the only thing the flag ever @@ -87,18 +93,27 @@ const PROFILE_SPECS: Record = { configuredEndpoint: { honorsAllowlist: true, insecureHttp: 'whenVouched', - allowLoopback: !isHosted, + allowLoopbackOffHosted: true, + }, + selfHostedService: { + honorsAllowlist: true, + insecureHttp: 'always', + allowLoopbackOffHosted: true, + denyServicePorts: false, + }, + requestTarget: { + honorsAllowlist: true, + insecureHttp: 'whenVouched', + allowLoopbackOffHosted: true, }, - selfHostedService: { honorsAllowlist: true, insecureHttp: 'always', allowLoopback: !isHosted }, - requestTarget: { honorsAllowlist: true, insecureHttp: 'whenVouched', allowLoopback: !isHosted }, - contentFetch: { honorsAllowlist: false, insecureHttp: 'never', allowLoopback: false }, + contentFetch: { honorsAllowlist: false, insecureHttp: 'never', allowLoopbackOffHosted: false }, databaseHost: { honorsAllowlist: true, insecureHttp: 'whenVouched', - allowLoopback: false, + allowLoopbackOffHosted: false, honorsLegacyPrivateFlag: true, }, - proxy: { honorsAllowlist: false, insecureHttp: 'always', allowLoopback: false }, + proxy: { honorsAllowlist: false, insecureHttp: 'always', allowLoopbackOffHosted: false }, } const SOURCE_NAMES = { @@ -110,6 +125,7 @@ interface DeploymentConfig { readonly hosts: string | undefined readonly ranges: string | undefined readonly legacyPrivate: boolean + readonly hosted: boolean } function readDeploymentConfig(): DeploymentConfig { @@ -117,6 +133,7 @@ function readDeploymentConfig(): DeploymentConfig { hosts: getEgressAllowedHosts(), ranges: getEgressAllowedIpRanges(), legacyPrivate: isLegacyPrivateDatabaseAccessAllowed(), + hosted: isHosted, } } @@ -130,8 +147,9 @@ function buildPolicies(config: DeploymentConfig): Record { byteStream('event: message\ndata: {"id":1}\n\n') ) ) - const { fetch } = createSsrfGuardedFetchWithDispatcher() + const { fetch } = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) const response = await fetch('https://mcp.example.com/serve', { method: 'POST', @@ -92,7 +92,7 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { undiciReply(302, { location: 'https://mcp.example.com/final' }, byteStream('redirect')) ) .mockResolvedValueOnce(undiciReply(200, {}, byteStream('final-body'))) - const { fetch } = createSsrfGuardedFetchWithDispatcher() + const { fetch } = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) const response = await fetch('https://mcp.example.com/start', { method: 'GET' }) @@ -110,7 +110,7 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { byteStream(JSON.stringify({ ok: true })) ) ) - const { fetch } = createSsrfGuardedFetchWithDispatcher() + const { fetch } = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) const response = await fetch('https://mcp.example.com/data', { method: 'GET' }) @@ -119,7 +119,7 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { it('normalizes a Headers instance and an ArrayBuffer body for undici.request', async () => { mockUndiciRequest.mockResolvedValueOnce(undiciReply(200, {}, byteStream('x'))) - const { fetch } = createSsrfGuardedFetchWithDispatcher() + const { fetch } = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) await fetch('https://mcp.example.com/x', { method: 'POST', @@ -135,7 +135,7 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { it('serializes a URLSearchParams body and defaults the form content-type (OAuth token exchange)', async () => { mockUndiciRequest.mockResolvedValueOnce(undiciReply(200, {}, byteStream('{}'))) - const { fetch } = createSsrfGuardedFetchWithDispatcher() + const { fetch } = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) await fetch('https://auth.example.com/token', { method: 'POST', @@ -149,7 +149,7 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { it('does not override an explicit content-type on a URLSearchParams body', async () => { mockUndiciRequest.mockResolvedValueOnce(undiciReply(200, {}, byteStream('{}'))) - const { fetch } = createSsrfGuardedFetchWithDispatcher() + const { fetch } = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) await fetch('https://auth.example.com/token', { method: 'POST', @@ -166,7 +166,7 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { it('copies each chunk so a recycled source buffer cannot corrupt queued data', async () => { const source = new Readable({ read() {} }) mockUndiciRequest.mockResolvedValueOnce(undiciReply(200, {}, source)) - const { fetch } = createSsrfGuardedFetchWithDispatcher() + const { fetch } = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) const response = await fetch('https://mcp.example.com/stream', { method: 'GET' }) const reader = response.body!.getReader() @@ -193,7 +193,7 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { source ) ) - const { fetch } = createSsrfGuardedFetchWithDispatcher() + const { fetch } = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) const response = await fetch('https://mcp.example.com/data', { method: 'GET' }) @@ -209,7 +209,7 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { mockUndiciRequest.mockResolvedValueOnce( undiciReply(200, { 'content-type': 'application/json', 'content-encoding': 'gzip' }, source) ) - const { fetch } = createSsrfGuardedFetchWithDispatcher() + const { fetch } = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) const response = await fetch('https://mcp.example.com/bad', { method: 'GET' }) @@ -220,7 +220,7 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { it('rejects the reader when the source is destroyed without an error (abort/reset)', async () => { const source = new Readable({ read() {} }) // stays open, never pushes mockUndiciRequest.mockResolvedValueOnce(undiciReply(200, {}, source)) - const { fetch } = createSsrfGuardedFetchWithDispatcher() + const { fetch } = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) const response = await fetch('https://mcp.example.com/hang', { method: 'GET' }) const reader = response.body!.getReader() diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index 852e139b3a2..de5e491927a 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -10,7 +10,6 @@ import { isIpLiteral, isPrivateIp, unwrapIpv6Brackets } from '@sim/security/ssrf import { toError } from '@sim/utils/errors' import { HttpProxyAgent } from 'http-proxy-agent' import { HttpsProxyAgent } from 'https-proxy-agent' -import * as ipaddr from 'ipaddr.js' import { Agent, type Dispatcher, @@ -484,23 +483,21 @@ const MAX_GUARDED_REDIRECTS = 5 * a 3xx to `http://169.254.169.254/` would otherwise connect directly. Hostname * targets are covered by {@link createSsrfGuardedLookup} at connect time. */ -function assertGuardedRedirectTarget(url: URL, allowedPinnedIp?: string): void { +function assertGuardedRedirectTarget(url: URL, profile: EgressProfile): void { if (url.protocol !== 'http:' && url.protocol !== 'https:') { throw new Error(`Blocked by SSRF policy: redirect to unsupported protocol ${url.protocol}`) } const host = unwrapIpv6Brackets(url.hostname) - if (ipaddr.isValid(host) && isPrivateIp(host)) { - // The pinned-private carve-out permits exactly its own validated IP as a target (a - // self-hosted MCP on a private IP, or a same-host redirect that stays on it) — but nothing - // else private (a redirect to e.g. the cloud metadata IP is still blocked). - if ( - allowedPinnedIp && - ipaddr.isValid(allowedPinnedIp) && - ipaddr.process(host).toString() === ipaddr.process(allowedPinnedIp).toString() - ) { - return - } - throw new Error('Blocked by SSRF policy: redirect to a private or reserved address') + if (!isIpLiteral(host)) return + + // The request's own policy decides, which is how a self-hosted server on a + // permitted private address stays reachable across a hop. It replaced a + // carve-out that permitted one pinned IP and could not express anything else. + const decision = checkResolvedEgress(url, host, profile) + if (!decision.allowed) { + throw new Error( + `Blocked by SSRF policy: ${describeEgressDenial(decision, 'redirect', profile)}` + ) } } @@ -566,14 +563,12 @@ export async function followRedirectsGuarded( rawFetch: (url: string, init: UndiciRequestInit) => Promise, input: string, init: UndiciRequestInit, - options?: { allowRedirectToIp?: string } + profile: EgressProfile ): Promise { let currentUrl = new URL(input) // The initial URL gets the same IP-literal check as redirect hops, so the exported guard is - // self-contained even when a caller skips its own up-front validation. `allowRedirectToIp` - // (the pinned-private MCP carve-out's validated IP) permits that one private target — both the - // initial URL and any hop that stays on it — while everything else private stays blocked. - assertGuardedRedirectTarget(currentUrl, options?.allowRedirectToIp) + // self-contained even when a caller skips its own up-front validation. + assertGuardedRedirectTarget(currentUrl, profile) let method = (init.method ?? 'GET').toUpperCase() let body = init.body let headers = init.headers @@ -601,7 +596,7 @@ export async function followRedirectsGuarded( throw new Error(`Blocked by SSRF policy: more than ${MAX_GUARDED_REDIRECTS} redirects`) } const nextUrl = new URL(location, currentUrl) - assertGuardedRedirectTarget(nextUrl, options?.allowRedirectToIp) + assertGuardedRedirectTarget(nextUrl, profile) const hopPolicy = resolveRedirectHop({ status, method, @@ -855,14 +850,17 @@ async function liftFetchArgs( * targets can't bypass the lookup and custom headers never cross origins. See * {@link createPinnedFetchWithDispatcher} for the `maxResponseSize` semantics. */ -export function createSsrfGuardedFetchWithDispatcher(options?: { maxResponseSize?: number }): { +export function createSsrfGuardedFetchWithDispatcher(options: { + profile: EgressProfile + maxResponseSize?: number +}): { fetch: typeof fetch dispatcher: Agent } { const dispatcher = new Agent({ allowH2: false, connect: { lookup: createSsrfGuardedLookup() }, - ...(options?.maxResponseSize !== undefined ? { maxResponseSize: options.maxResponseSize } : {}), + ...(options.maxResponseSize !== undefined ? { maxResponseSize: options.maxResponseSize } : {}), }) const rawFetch = (url: string, init: UndiciRequestInit): Promise => @@ -871,8 +869,13 @@ export function createSsrfGuardedFetchWithDispatcher(options?: { maxResponseSize const guarded = async (input: RequestInfo | URL, init?: RequestInit): Promise => { const { target, effectiveInit } = await liftFetchArgs(input, init) - // double-cast-allowed: DOM RequestInit and undici RequestInit are structurally compatible at runtime but the TS types differ - return followRedirectsGuarded(rawFetch, target, effectiveInit as unknown as UndiciRequestInit) + return followRedirectsGuarded( + rawFetch, + target, + // double-cast-allowed: DOM RequestInit and undici RequestInit are structurally compatible at runtime but the TS types differ + effectiveInit as unknown as UndiciRequestInit, + options.profile + ) } return { fetch: guarded, dispatcher } @@ -904,7 +907,7 @@ export function createSsrfGuardedFetchWithDispatcher(options?: { maxResponseSize */ export function createPinnedFetch( resolvedIP: string, - options?: { allowH2?: boolean } + options: { profile: EgressProfile; allowH2?: boolean } ): typeof fetch { return createPinnedFetchWithDispatcher(resolvedIP, options).fetch } @@ -922,12 +925,12 @@ export function createPinnedFetch( */ export function createPinnedFetchWithDispatcher( resolvedIP: string, - options?: { allowH2?: boolean; maxResponseSize?: number } + options: { profile: EgressProfile; allowH2?: boolean; maxResponseSize?: number } ): { fetch: typeof fetch; dispatcher: Agent } { const dispatcher = new Agent({ - allowH2: options?.allowH2 ?? false, + allowH2: options.allowH2 ?? false, connect: { lookup: createPinnedLookup(resolvedIP) }, - ...(options?.maxResponseSize !== undefined ? { maxResponseSize: options.maxResponseSize } : {}), + ...(options.maxResponseSize !== undefined ? { maxResponseSize: options.maxResponseSize } : {}), }) const rawFetch = (url: string, init: UndiciRequestInit): Promise => @@ -961,11 +964,7 @@ export function createPinnedFetchWithDispatcher( } return response } - // Permit this pinned IP as a redirect/initial target even when it's private (the - // self-hosted MCP carve-out on a private/loopback IP, and same-host redirects that stay on - // it) — otherwise the guarded policy would block a self-hosted server reaching itself. Any - // OTHER private target (e.g. a redirect to the cloud metadata IP) is still blocked. - return followRedirectsGuarded(rawFetch, target, undiciInit, { allowRedirectToIp: resolvedIP }) + return followRedirectsGuarded(rawFetch, target, undiciInit, options.profile) } return { fetch: pinned, dispatcher } diff --git a/apps/sim/lib/core/security/pinned-fetch.server.test.ts b/apps/sim/lib/core/security/pinned-fetch.server.test.ts index a1eca0eb09e..6953f3f6bee 100644 --- a/apps/sim/lib/core/security/pinned-fetch.server.test.ts +++ b/apps/sim/lib/core/security/pinned-fetch.server.test.ts @@ -2,7 +2,8 @@ * @vitest-environment node */ import { Readable } from 'node:stream' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { mockAgent, mockUndiciRequest, capturedAgentOptions } = vi.hoisted(() => { const capturedAgentOptions: unknown[] = [] @@ -47,6 +48,8 @@ function undiciReply(statusCode: number, headers: Record, body: return { statusCode, headers, body, trailers: {}, opaque: null, context: {} } } +afterEach(resetEnvFlagsMock) + describe('createPinnedFetch', () => { beforeEach(() => { vi.clearAllMocks() @@ -55,7 +58,7 @@ describe('createPinnedFetch', () => { }) it('builds an undici Agent whose pinned lookup always resolves to the validated IP', async () => { - createPinnedFetch('203.0.113.10') + createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) expect(capturedAgentOptions).toHaveLength(1) const { connect } = capturedAgentOptions[0] as { connect: { lookup: PinnedLookup } } @@ -66,23 +69,23 @@ describe('createPinnedFetch', () => { resolve({ address, family }) ) }) - expect(resolved).toEqual({ address: '203.0.113.10', family: 4 }) + expect(resolved).toEqual({ address: '93.184.216.34', family: 4 }) }) it('defaults allowH2 to false so existing consumers are unchanged', () => { - createPinnedFetch('203.0.113.10') + createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) const opts = capturedAgentOptions[0] as { allowH2?: boolean } expect(opts.allowH2).toBe(false) }) it('opts the Agent into HTTP/2 when allowH2 is requested', () => { - createPinnedFetch('203.0.113.10', { allowH2: true }) + createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint', allowH2: true }) const opts = capturedAgentOptions[0] as { allowH2?: boolean } expect(opts.allowH2).toBe(true) }) it('uses IPv6 family when the validated IP is IPv6', async () => { - createPinnedFetch('2606:4700:4700::1111') + createPinnedFetch('2606:4700:4700::1111', { profile: 'configuredEndpoint' }) const { connect } = capturedAgentOptions[0] as { connect: { lookup: PinnedLookup } } const resolved = await new Promise<{ address: string; family: number }>((resolve) => { connect.lookup('example.com', {}, (_err, address, family) => resolve({ address, family })) @@ -91,7 +94,7 @@ describe('createPinnedFetch', () => { }) it('dispatches through the pinned Agent, preserving init', async () => { - const pinned = createPinnedFetch('203.0.113.10') + const pinned = createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) const controller = new AbortController() await pinned('https://myresource.openai.azure.com/openai/v1/responses', { @@ -115,7 +118,7 @@ describe('createPinnedFetch', () => { mockUndiciRequest.mockResolvedValueOnce( undiciReply(302, { location: 'https://login.example.com/' }, byteStream('')) ) - const pinned = createPinnedFetch('203.0.113.10') + const pinned = createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) const response = await pinned('https://mcp.example.com/', { redirect: 'manual' }) @@ -128,7 +131,7 @@ describe('createPinnedFetch', () => { mockUndiciRequest.mockResolvedValueOnce( undiciReply(302, { location: 'https://login.example.com/' }, byteStream('')) ) - const pinned = createPinnedFetch('203.0.113.10') + const pinned = createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) const response = await pinned(new Request('https://mcp.example.com/', { redirect: 'manual' })) @@ -142,7 +145,7 @@ describe('createPinnedFetch', () => { undiciReply(307, { location: 'https://other-origin.example/final' }, byteStream('')) ) .mockResolvedValueOnce(undiciReply(200, {}, byteStream('done'))) - const pinned = createPinnedFetch('203.0.113.10') + const pinned = createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) const response = await pinned('https://azure.example.com/v1/responses', { method: 'GET', @@ -163,12 +166,13 @@ describe('createPinnedFetch', () => { expect(await response.text()).toBe('done') }) - it('does NOT block a private IP-literal URL (self-hosted-private MCP carve-out)', async () => { + it('reaches a private IP-literal URL the operator allowlisted', async () => { + setEnvFlags({ egressAllowedIpRanges: '10.0.0.0/8' }) mockUndiciRequest.mockResolvedValueOnce(undiciReply(200, {}, byteStream('mcp'))) - const pinned = createPinnedFetch('10.0.0.5') + const pinned = createPinnedFetch('10.0.0.5', { profile: 'configuredEndpoint' }) - // A self-hosted MCP configured with a private IP-literal URL must still connect — the old - // undici.fetch path never ran the SSRF initial-target check that would otherwise block it. + // A self-hosted MCP on a private address connects because the deployment + // named that range, not because the address happened to be the pinned one. const response = await pinned('http://10.0.0.5:3000/mcp', { method: 'POST', body: '{}' }) expect(mockUndiciRequest).toHaveBeenCalledTimes(1) @@ -176,13 +180,14 @@ describe('createPinnedFetch', () => { expect(await response.text()).toBe('mcp') }) - it('follows a redirect that stays on the pinned private IP (self-hosted MCP alias)', async () => { + it('follows a redirect that stays inside the allowlisted range', async () => { + setEnvFlags({ egressAllowedIpRanges: '10.0.0.0/8' }) mockUndiciRequest .mockResolvedValueOnce( undiciReply(301, { location: 'http://10.0.0.5:3000/mcp/' }, byteStream('')) ) .mockResolvedValueOnce(undiciReply(200, {}, byteStream('mcp'))) - const pinned = createPinnedFetch('10.0.0.5') + const pinned = createPinnedFetch('10.0.0.5', { profile: 'configuredEndpoint' }) const response = await pinned('http://10.0.0.5:3000/mcp', { method: 'GET' }) @@ -191,21 +196,22 @@ describe('createPinnedFetch', () => { expect(await response.text()).toBe('mcp') }) - it('STILL blocks a redirect to a different private IP (no metadata-IP escape)', async () => { + it('still blocks a redirect to a private IP outside the allowlist', async () => { + setEnvFlags({ egressAllowedIpRanges: '10.0.0.0/8' }) mockUndiciRequest.mockResolvedValueOnce( undiciReply(302, { location: 'http://169.254.169.254/latest/meta-data/' }, byteStream('')) ) - const pinned = createPinnedFetch('10.0.0.5') + const pinned = createPinnedFetch('10.0.0.5', { profile: 'configuredEndpoint' }) await expect(pinned('http://10.0.0.5:3000/mcp', { method: 'GET' })).rejects.toThrow( - /private or reserved/ + /cloud metadata endpoint/ ) // The initial request happened; the redirect to the metadata IP was refused. expect(mockUndiciRequest).toHaveBeenCalledTimes(1) }) it('reuses one dispatcher across all calls of a single instance', async () => { - const pinned = createPinnedFetch('203.0.113.10') + const pinned = createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) await pinned('https://example.com/a') await pinned('https://example.com/b') @@ -216,8 +222,8 @@ describe('createPinnedFetch', () => { }) it('creates an independent dispatcher per instance', async () => { - const a = createPinnedFetch('203.0.113.10') - const b = createPinnedFetch('203.0.113.10') + const a = createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) + const b = createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) await a('https://example.com/a') await b('https://example.com/b') @@ -229,7 +235,7 @@ describe('createPinnedFetch', () => { it('returns a streaming Response built from the undici.request body', async () => { mockUndiciRequest.mockResolvedValueOnce(undiciReply(201, {}, byteStream('pong'))) - const pinned = createPinnedFetch('203.0.113.10') + const pinned = createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) const response = await pinned('https://example.com') expect(response.status).toBe(201) expect(await response.text()).toBe('pong') diff --git a/apps/sim/lib/internal/onepassword/client.test.ts b/apps/sim/lib/internal/onepassword/client.test.ts index fba97de50e8..ef9f7805ee2 100644 --- a/apps/sim/lib/internal/onepassword/client.test.ts +++ b/apps/sim/lib/internal/onepassword/client.test.ts @@ -4,9 +4,10 @@ import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' -const { mockDnsLookup, mockSecureFetch } = vi.hoisted(() => ({ +const { mockDnsLookup, mockSecureFetch, mockValidateUrlWithDNS } = vi.hoisted(() => ({ mockDnsLookup: vi.fn(), mockSecureFetch: vi.fn(), + mockValidateUrlWithDNS: vi.fn(), })) vi.mock('dns/promises', () => ({ @@ -15,6 +16,7 @@ vi.mock('dns/promises', () => ({ vi.mock('@/lib/core/security/input-validation.server', () => ({ MAX_JSON_API_RESPONSE_BYTES: 10 * 1024 * 1024, secureFetchWithPinnedIP: mockSecureFetch, + validateUrlWithDNS: mockValidateUrlWithDNS, })) import { connectRequest, validateConnectServerUrl } from '@/lib/internal/onepassword/client' @@ -27,100 +29,31 @@ describe('validateConnectServerUrl', () => { setEnvFlags({ isHosted: false }) }) - it('rejects a non-URL string', async () => { - await expect(validateConnectServerUrl('not a url')).rejects.toThrow('is not a valid URL') - }) - - describe('hosted deployment', () => { - beforeEach(() => { - setEnvFlags({ isHosted: true }) - }) - - it.each([ - ['loopback', 'http://127.0.0.1:8080'], - ['RFC1918 10.x', 'http://10.0.0.5'], - ['RFC1918 192.168.x', 'http://192.168.1.1:8443'], - ['RFC1918 172.16.x', 'http://172.16.0.9'], - ['link-local metadata', 'http://169.254.169.254'], - ['IPv4-mapped IPv6 private', 'http://[::ffff:10.0.0.1]'], - ['IPv6 loopback', 'http://[::1]'], - ])('blocks %s', async (_label, url) => { - await expect(validateConnectServerUrl(url)).rejects.toThrow( - 'cannot point to a private or reserved IP address' - ) - }) - - it('allows a public IP literal', async () => { - await expect(validateConnectServerUrl('https://8.8.8.8')).resolves.toBe('8.8.8.8') - }) - - it('blocks a hostname that resolves to a private IP', async () => { - mockDnsLookup.mockResolvedValue([{ address: '10.1.2.3', family: 4 }]) - await expect(validateConnectServerUrl('https://connect.internal')).rejects.toThrow( - 'cannot point to a private or reserved IP address' - ) - }) + it('delegates to the egress guard as a self-hosted service', async () => { + mockValidateUrlWithDNS.mockResolvedValue({ isValid: true, resolvedIP: '10.0.0.9' }) - it('allows a hostname that resolves to a public IP', async () => { - mockDnsLookup.mockResolvedValue([{ address: '93.184.216.34', family: 4 }]) - await expect(validateConnectServerUrl('https://connect.example.com')).resolves.toBe( - '93.184.216.34' - ) - }) - - it('prefers the IPv4 address for a dual-stack host (avoids unreachable IPv6 pin)', async () => { - mockDnsLookup.mockResolvedValue([ - { address: '2606:4700::6810:85e5', family: 6 }, - { address: '93.184.216.34', family: 4 }, - ]) - await expect(validateConnectServerUrl('https://connect.example.com')).resolves.toBe( - '93.184.216.34' - ) - }) + await expect(validateConnectServerUrl('http://connect.internal:8080')).resolves.toBe('10.0.0.9') - it('pins the sole IPv6 address for an IPv6-only host', async () => { - mockDnsLookup.mockResolvedValue([{ address: '2606:4700::6810:85e5', family: 6 }]) - await expect(validateConnectServerUrl('https://connect.example.com')).resolves.toBe( - '2606:4700::6810:85e5' - ) - }) + // The profile is the whole policy decision: Connect is ordinarily deployed + // inside a network, on plain HTTP, on an arbitrary port. Which addresses that + // permits is the guard's contract, covered by its own tests rather than + // restated here — this file used to carry a copy of them alongside a copy of + // the implementation. + expect(mockValidateUrlWithDNS).toHaveBeenCalledWith( + 'http://connect.internal:8080', + '1Password server URL', + 'selfHostedService' + ) }) - describe('self-hosted deployment', () => { - beforeEach(() => { - setEnvFlags({ isHosted: false }) - }) - - it.each([ - ['loopback', 'http://127.0.0.1:8080', '127.0.0.1'], - ['RFC1918 10.x', 'http://10.0.0.5', '10.0.0.5'], - ['RFC1918 192.168.x', 'http://192.168.1.1:8443', '192.168.1.1'], - ])('allows %s (private Connect server)', async (_label, url, expected) => { - await expect(validateConnectServerUrl(url)).resolves.toBe(expected) - }) - - it('still blocks link-local metadata', async () => { - await expect(validateConnectServerUrl('http://169.254.169.254')).rejects.toThrow( - 'cannot point to a link-local address' - ) + it('surfaces the guard refusal verbatim', async () => { + mockValidateUrlWithDNS.mockResolvedValue({ + isValid: false, + error: '1Password server URL resolves to a private or reserved address (10.0.0.9).', }) - it('still blocks IPv6 link-local', async () => { - await expect(validateConnectServerUrl('http://[fe80::1]')).rejects.toThrow( - 'cannot point to a link-local address' - ) - }) - - it('allows a hostname that resolves to a private IP', async () => { - mockDnsLookup.mockResolvedValue([{ address: '10.1.2.3', family: 4 }]) - await expect(validateConnectServerUrl('https://connect.internal')).resolves.toBe('10.1.2.3') - }) - }) - - it('rejects when DNS resolution fails', async () => { - mockDnsLookup.mockRejectedValue(new Error('ENOTFOUND')) - await expect(validateConnectServerUrl('https://nope.invalid')).rejects.toThrow( - 'could not be resolved' + await expect(validateConnectServerUrl('http://connect.internal')).rejects.toThrow( + 'resolves to a private or reserved address (10.0.0.9)' ) }) }) @@ -129,6 +62,7 @@ describe('connectRequest', () => { beforeEach(() => { vi.clearAllMocks() setEnvFlags({ isHosted: false }) + mockValidateUrlWithDNS.mockResolvedValue({ isValid: true, resolvedIP: '8.8.8.8' }) mockSecureFetch.mockResolvedValue({ ok: true, status: 200 }) }) diff --git a/apps/sim/lib/internal/onepassword/client.ts b/apps/sim/lib/internal/onepassword/client.ts index aae6c1f5a8b..8a5041e589b 100644 --- a/apps/sim/lib/internal/onepassword/client.ts +++ b/apps/sim/lib/internal/onepassword/client.ts @@ -10,17 +10,12 @@ import type { VaultOverview, Website, } from '@1password/sdk' -import { createLogger } from '@sim/logger' -import { resolveHostAddresses } from '@sim/security/dns' -import { isPrivateIp, unwrapIpv6Brackets } from '@sim/security/ssrf' -import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' -import * as ipaddr from 'ipaddr.js' -import { isHosted } from '@/lib/core/config/env-flags' import { MAX_JSON_API_RESPONSE_BYTES, type SecureFetchResponse, secureFetchWithPinnedIP, + validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' /** Connect-format field type strings returned by normalization. */ @@ -262,85 +257,32 @@ export async function createOnePasswordClient(serviceAccountToken: string, signa return client } -const connectLogger = createLogger('OnePasswordConnect') - /** - * Enforces the SSRF policy for a resolved Connect server IP. + * Validates a Connect server URL against the deployment's egress policy and + * returns the resolved IP for DNS pinning. * - * On the hosted service, all private and reserved IPs are blocked — a tenant has - * no legitimate reason to point Connect at the platform's internal network. On - * self-hosted deployments only link-local (cloud metadata) is blocked, since the - * operator controls both the workflows and the network and Connect servers - * legitimately live on private (RFC1918) addresses. + * The `selfHostedService` profile matches how Connect is deployed: plain HTTP on + * an arbitrary port is ordinary, loopback is reachable off the hosted platform, + * and a Connect server on the rest of a private network is reachable once the + * operator names it in the egress allowlist. * - * @throws Error if the IP is not permitted under the active policy. - */ -function assertConnectIpAllowed(ip: string, hostname: string): void { - if (isHosted) { - if (isPrivateIp(ip)) { - connectLogger.warn('1Password Connect server URL resolves to a private or reserved IP', { - hostname, - resolvedIP: ip, - }) - throw new Error('1Password server URL cannot point to a private or reserved IP address') - } - return - } - - if (ipaddr.isValid(ip) && ipaddr.process(ip).range() === 'linkLocal') { - connectLogger.warn('1Password Connect server URL resolves to a link-local IP', { - hostname, - resolvedIP: ip, - }) - throw new Error('1Password server URL cannot point to a link-local address') - } -} - -/** - * Validates a Connect server URL against the SSRF policy and returns the resolved - * IP for DNS pinning to prevent TOCTOU rebinding. See {@link assertConnectIpAllowed} - * for the hosted vs. self-hosted policy. - * @throws Error if the URL is invalid, fails the IP policy, or DNS fails. + * @throws Error if the URL is invalid, refused by the policy, or unresolvable. */ export async function validateConnectServerUrl( serverUrl: string, signal?: AbortSignal ): Promise { signal?.throwIfAborted() - let hostname: string - try { - hostname = new URL(serverUrl).hostname - } catch { - throw new Error('1Password server URL is not a valid URL') - } - - const clean = unwrapIpv6Brackets(hostname) - - if (ipaddr.isValid(clean)) { - assertConnectIpAllowed(clean, clean) - return clean - } - - let addresses: string[] - let address: string - try { - const resolved = await resolveHostAddresses(clean) - signal?.throwIfAborted() - addresses = resolved.addresses - address = resolved.preferred - } catch (error) { - signal?.throwIfAborted() - connectLogger.warn('DNS lookup failed for 1Password Connect server URL', { - hostname: clean, - error: toError(error).message, - }) - throw new Error('1Password server URL hostname could not be resolved') - } - - for (const candidate of addresses) { - assertConnectIpAllowed(candidate, clean) + const validation = await validateUrlWithDNS( + serverUrl, + '1Password server URL', + 'selfHostedService' + ) + signal?.throwIfAborted() + if (!validation.isValid) { + throw new Error(validation.error ?? '1Password server URL is not reachable') } - return address + return validation.resolvedIP! } /** diff --git a/apps/sim/lib/mcp/domain-check.test.ts b/apps/sim/lib/mcp/domain-check.test.ts index aca421d7127..307330f2380 100644 --- a/apps/sim/lib/mcp/domain-check.test.ts +++ b/apps/sim/lib/mcp/domain-check.test.ts @@ -334,13 +334,13 @@ describe('validateMcpServerSsrf', () => { expect(mockDnsLookup).not.toHaveBeenCalled() }) - it('returns null for localhost URLs without DNS lookup', async () => { - await expect(validateMcpServerSsrf('http://localhost:3000/mcp')).resolves.toBeNull() - expect(mockDnsLookup).not.toHaveBeenCalled() + it('pins a localhost URL rather than leaving it unguarded', async () => { + mockDnsLookup.mockResolvedValue([{ address: '127.0.0.1', family: 4 }]) + await expect(validateMcpServerSsrf('http://localhost:3000/mcp')).resolves.toBe('127.0.0.1') }) - it('returns null for 127.0.0.1 literal without DNS lookup', async () => { - await expect(validateMcpServerSsrf('http://127.0.0.1:8080/mcp')).resolves.toBeNull() + it('pins a loopback literal without a DNS lookup', async () => { + await expect(validateMcpServerSsrf('http://127.0.0.1:8080/mcp')).resolves.toBe('127.0.0.1') expect(mockDnsLookup).not.toHaveBeenCalled() }) @@ -423,9 +423,22 @@ describe('validateMcpServerSsrf', () => { ) }) - it('returns resolved IP for URLs resolving to loopback on self-hosted (localhost alias)', async () => { + it('refuses a DNS alias that resolves to loopback unless it is allowlisted', async () => { + // The loopback carve-out keys off the hostname, so a name pointed at + // loopback is named in EGRESS_ALLOWED_HOSTS or it is not reachable. mockDnsLookup.mockResolvedValue([{ address: '127.0.0.1', family: 4 }]) - await expect(validateMcpServerSsrf('http://my-local-alias:3000/mcp')).resolves.toBe('127.0.0.1') + await expect(validateMcpServerSsrf('http://my-local-alias:3000/mcp')).rejects.toThrow( + McpSsrfError + ) + + setEnvFlags({ egressAllowedHosts: 'my-local-alias' }) + try { + await expect(validateMcpServerSsrf('http://my-local-alias:3000/mcp')).resolves.toBe( + '127.0.0.1' + ) + } finally { + setEnvFlags({ egressAllowedHosts: undefined }) + } }) it('throws for malformed URLs', async () => { @@ -466,9 +479,12 @@ describe('validateMcpServerSsrf', () => { expect(mockDnsLookup).not.toHaveBeenCalled() }) - it('skips loopback check on hosted when allowlist is configured', async () => { + it('still refuses loopback on hosted when a domain allowlist is configured', async () => { + // The domain allowlist governs which domains may be used. It is not a + // substitute for the address check, which it used to disable entirely. mockGetAllowedMcpDomainsFromEnv.mockReturnValue(['localhost']) - await expect(validateMcpServerSsrf('http://localhost:3000/mcp')).resolves.toBeNull() + mockDnsLookup.mockResolvedValue([{ address: '127.0.0.1', family: 4 }]) + await expect(validateMcpServerSsrf('http://localhost:3000/mcp')).rejects.toThrow(McpSsrfError) }) it('still blocks RFC-1918 IP literals on hosted (regression)', async () => { @@ -499,26 +515,29 @@ describe('validateMcpServerSsrf', () => { setEnvFlags({ isHosted: false }) }) - it('still allows localhost URLs (returns null, no pinning needed)', async () => { - await expect(validateMcpServerSsrf('http://localhost:3000/mcp')).resolves.toBeNull() - }) - - it('still allows 127.0.0.1 URLs (returns null, no pinning needed)', async () => { - await expect(validateMcpServerSsrf('http://127.0.0.1:8080/mcp')).resolves.toBeNull() + it('still reaches a local MCP server, now pinned rather than unguarded', async () => { + mockDnsLookup.mockResolvedValue([{ address: '127.0.0.1', family: 4 }]) + await expect(validateMcpServerSsrf('http://localhost:3000/mcp')).resolves.toBe('127.0.0.1') + await expect(validateMcpServerSsrf('http://127.0.0.1:8080/mcp')).resolves.toBe('127.0.0.1') }) - it('returns resolved loopback IP for DNS aliases (caller pins)', async () => { - mockDnsLookup.mockResolvedValue([{ address: '127.0.0.1', family: 4 }]) - await expect(validateMcpServerSsrf('http://my-local-alias/mcp')).resolves.toBe('127.0.0.1') + it('reaches a private MCP server once the operator allowlists it', async () => { + setEnvFlags({ egressAllowedIpRanges: '10.0.0.0/8' }) + try { + await expect(validateMcpServerSsrf('http://10.0.0.9:3000/mcp')).resolves.toBe('10.0.0.9') + } finally { + setEnvFlags({ egressAllowedIpRanges: undefined }) + } }) }) - it('skips all checks when ALLOWED_MCP_DOMAINS is configured', async () => { + it('applies the address check even when ALLOWED_MCP_DOMAINS is configured', async () => { + // Configuring the domain list used to disable this entirely, which left an + // allowlisted domain free to redirect at anything, metadata included. mockGetAllowedMcpDomainsFromEnv.mockReturnValue(['internal.corp']) - await expect(validateMcpServerSsrf('http://10.0.0.1/mcp')).resolves.toBeNull() - await expect( - validateMcpServerSsrf('http://169.254.169.254/latest/meta-data/') - ).resolves.toBeNull() - expect(mockDnsLookup).not.toHaveBeenCalled() + await expect(validateMcpServerSsrf('http://10.0.0.1/mcp')).rejects.toThrow(McpSsrfError) + await expect(validateMcpServerSsrf('http://169.254.169.254/latest/meta-data/')).rejects.toThrow( + McpSsrfError + ) }) }) diff --git a/apps/sim/lib/mcp/domain-check.ts b/apps/sim/lib/mcp/domain-check.ts index 6a39f0a8aa2..0283550b567 100644 --- a/apps/sim/lib/mcp/domain-check.ts +++ b/apps/sim/lib/mcp/domain-check.ts @@ -1,12 +1,18 @@ import { createLogger } from '@sim/logger' -import { resolveHostAddresses } from '@sim/security/dns' -import { isIpLiteral, isLoopbackIp, isPrivateIp, unwrapIpv6Brackets } from '@sim/security/ssrf' -import { toError } from '@sim/utils/errors' -import { getAllowedMcpDomainsFromEnv, isHosted } from '@/lib/core/config/env-flags' +import { getAllowedMcpDomainsFromEnv } from '@/lib/core/config/env-flags' +import type { EgressProfile } from '@/lib/core/security/egress/profiles' +import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import { createEnvVarPattern } from '@/executor/utils/reference-validation' const logger = createLogger('McpDomainCheck') +/** + * An MCP server URL is a configured endpoint for software commonly self-hosted: + * plain HTTP and an arbitrary port are ordinary, and reaching one on a private + * address is a matter of the operator naming it in the egress allowlist. + */ +export const MCP_EGRESS_PROFILE: EgressProfile = 'selfHostedService' + export class McpDomainNotAllowedError extends Error { constructor(domain: string) { super(`MCP server domain "${domain}" is not allowed by the server's ALLOWED_MCP_DOMAINS policy`) @@ -98,111 +104,40 @@ export function validateMcpDomain(url: string | undefined): void { } /** - * Returns true if the hostname is localhost or a loopback IP literal (full - * 127.0.0.0/8 range, or ::1). Expects IPv6 brackets to already be stripped. - */ -function isLocalhostHostname(hostname: string): boolean { - const clean = hostname.toLowerCase() - if (clean === 'localhost') return true - return isLoopbackIp(clean) -} - -/** - * Validates an MCP server URL against SSRF attacks by resolving DNS and - * rejecting private/reserved IP ranges (RFC-1918, link-local, cloud metadata). + * Validates an MCP server URL against the deployment's egress policy and returns + * the address to pin. * - * Only active when ALLOWED_MCP_DOMAINS is **not configured**. When an admin - * has set an explicit domain allowlist, they control which domains are - * reachable and private-network MCP servers are legitimate. Applying SSRF - * blocking on top of an admin-curated list would break self-hosted - * deployments where MCP servers run on internal networks. + * Domain governance (`ALLOWED_MCP_DOMAINS`) and this check are separate + * questions and both apply: an allowlisted domain still has to resolve somewhere + * the deployment permits. They used to be alternatives — configuring the domain + * list disabled this entirely — which left an allowlisted domain free to redirect + * anywhere, cloud metadata included. * - * Does NOT enforce protocol (HTTP is allowed) or block service ports — MCP - * servers legitimately run on HTTP and on arbitrary ports. + * Returns null only when the hostname still contains an unresolved env-var + * reference. That URL is checked again after resolution, at which point it takes + * the normal path. * - * Localhost/loopback is allowed for local dev MCP servers in self-hosted - * deployments, but blocked on the hosted environment (sim.ai) where users - * must not be able to reach the server's own loopback interface. - * URLs with env var references in the hostname are skipped — they will be - * validated after resolution at execution time. - * - * Returns the resolved IP (or the literal itself for IP-literal URLs) as a - * non-null **policy signal**: the SSRF guard is active for this server. A public - * resolution selects the validate-at-connect guarded fetch — DNS-rebinding TOCTOU - * and redirect escapes are prevented by re-validating every socket connect and - * following redirects under per-hop validation (see `createSsrfGuardedMcpFetch` / - * `followRedirectsGuarded`), NOT by pinning to this address. The value is literally - * pinned only for the self-hosted private/loopback carve-out (a policy-permitted - * DNS alias the guarded lookup would otherwise filter). Returns null when the guard - * is unnecessary or impossible: no URL, allowlist-only mode, env-var hostnames - * (validated later), and localhost on self-hosted (no rebinding risk against a - * fixed loopback). - * - * @throws McpSsrfError if the URL resolves to a blocked IP address + * @throws McpSsrfError when the policy refuses the destination + * @throws McpDnsResolutionError when the hostname cannot be resolved */ export async function validateMcpServerSsrf(url: string | undefined): Promise { if (!url) return null - if (getAllowedMcpDomainsFromEnv() !== null) return null if (hasEnvVarInHostname(url)) return null - let hostname: string - try { - hostname = new URL(url).hostname - } catch { - throw new McpSsrfError('MCP server URL is not a valid URL') - } - - const cleanHostname = unwrapIpv6Brackets(hostname) - - if (isLocalhostHostname(cleanHostname)) { - if (isHosted) { - throw new McpSsrfError('MCP server URL cannot point to a loopback address') - } - return null - } - - if (isIpLiteral(cleanHostname)) { - if (isPrivateIp(cleanHostname)) { - throw new McpSsrfError('MCP server URL cannot point to a private or reserved IP address') - } - // Public IP literal: pin to this exact address so the caller's pinned fetch - // (createPinnedFetch) keeps every redirect hop on it. Returning null here - // would fall back to the default fetch, which follows a 3xx redirect to a - // private/metadata host and escapes SSRF controls. - return cleanHostname - } - - let addresses: string[] - let address: string - try { - const resolved = await resolveHostAddresses(cleanHostname) - addresses = resolved.addresses - address = resolved.preferred - } catch (error) { - logger.warn('DNS lookup failed for MCP server URL', { - hostname, - error: toError(error).message, - }) - throw new McpDnsResolutionError(cleanHostname) - } + const validation = await validateUrlWithDNS(url, 'MCP server URL', MCP_EGRESS_PROFILE) + if (validation.isValid) return validation.resolvedIP! - for (const candidate of addresses) { - if (isLoopbackIp(candidate)) { - if (isHosted) { - logger.warn('MCP server URL resolves to loopback address', { - hostname, - resolvedIP: candidate, - }) - throw new McpSsrfError('MCP server URL resolves to a loopback address') - } - } else if (isPrivateIp(candidate)) { - logger.warn('MCP server URL resolves to blocked IP address', { - hostname, - resolvedIP: candidate, - }) - throw new McpSsrfError('MCP server URL resolves to a blocked IP address') + const error = validation.error ?? 'MCP server URL is not reachable' + if (error.includes('could not be resolved')) { + let hostname = url + try { + hostname = new URL(url).hostname + } catch { + // Fall back to the raw URL in the message. } + logger.warn('DNS lookup failed for MCP server URL', { hostname }) + throw new McpDnsResolutionError(hostname) } - - return address + logger.warn('MCP server URL refused by egress policy', { error }) + throw new McpSsrfError(error) } diff --git a/apps/sim/lib/mcp/oauth/probe.test.ts b/apps/sim/lib/mcp/oauth/probe.test.ts index 23551694cc2..3a6a2383c90 100644 --- a/apps/sim/lib/mcp/oauth/probe.test.ts +++ b/apps/sim/lib/mcp/oauth/probe.test.ts @@ -58,7 +58,9 @@ describe('detectMcpAuthType — connection pinning (SSRF / DNS-rebinding)', () = const authType = await detectMcpAuthType('https://rebind.example.com/mcp', '203.0.113.10') expect(authType).toBe('none') - expect(mockCreatePinnedFetchWithDispatcher).toHaveBeenCalledWith('203.0.113.10') + expect(mockCreatePinnedFetchWithDispatcher).toHaveBeenCalledWith('203.0.113.10', { + profile: 'selfHostedService', + }) expect(mockCreateSsrfGuardedMcpFetch).not.toHaveBeenCalled() expect(mockPinnedFetch).toHaveBeenCalledTimes(1) // The unpinned global fetch must never be used — that was the SSRF sink. diff --git a/apps/sim/lib/mcp/oauth/probe.ts b/apps/sim/lib/mcp/oauth/probe.ts index 72c6c593ced..da17123cc4f 100644 --- a/apps/sim/lib/mcp/oauth/probe.ts +++ b/apps/sim/lib/mcp/oauth/probe.ts @@ -3,6 +3,7 @@ import type { FetchLike } from '@modelcontextprotocol/sdk/shared/transport.js' import { createLogger } from '@sim/logger' import { isLoopbackHostname } from '@sim/security/hostnames' import { createPinnedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' +import { MCP_EGRESS_PROFILE } from '@/lib/mcp/domain-check' import { createSsrfGuardedMcpFetch } from '@/lib/mcp/pinned-fetch' import type { McpAuthType } from '@/lib/mcp/types' @@ -35,7 +36,9 @@ export async function detectMcpAuthType( // Pre-validated IP → pin directly (we own the Agent); otherwise the SSRF-guarded fetch // self-manages its per-request Agent teardown. - const pinned = resolvedIP ? createPinnedFetchWithDispatcher(resolvedIP) : undefined + const pinned = resolvedIP + ? createPinnedFetchWithDispatcher(resolvedIP, { profile: MCP_EGRESS_PROFILE }) + : undefined const probeFetch: FetchLike = pinned?.fetch ?? createSsrfGuardedMcpFetch() const controller = new AbortController() diff --git a/apps/sim/lib/mcp/oauth/revoke.test.ts b/apps/sim/lib/mcp/oauth/revoke.test.ts index ba91b2cad97..1f54008343a 100644 --- a/apps/sim/lib/mcp/oauth/revoke.test.ts +++ b/apps/sim/lib/mcp/oauth/revoke.test.ts @@ -44,6 +44,7 @@ vi.mock('@sim/security/ssrf', () => ({ isPrivateIp: (ip: string) => ip.startsWith('127.') || ip.startsWith('10.') || ip === '::1', })) vi.mock('@/lib/mcp/domain-check', () => ({ + MCP_EGRESS_PROFILE: 'selfHostedService', validateMcpServerSsrf: mockValidateMcpServerSsrf, })) vi.mock('@modelcontextprotocol/sdk/client/auth.js', () => ({ diff --git a/apps/sim/lib/mcp/orchestration/server-lifecycle.test.ts b/apps/sim/lib/mcp/orchestration/server-lifecycle.test.ts index b6df53b6b12..9d4f347010f 100644 --- a/apps/sim/lib/mcp/orchestration/server-lifecycle.test.ts +++ b/apps/sim/lib/mcp/orchestration/server-lifecycle.test.ts @@ -38,6 +38,7 @@ vi.mock('@sim/db/schema', () => ({ vi.mock('@sim/utils/id', () => ({ generateId: vi.fn() })) vi.mock('@/lib/core/security/encryption', () => encryptionMock) vi.mock('@/lib/mcp/domain-check', () => ({ + MCP_EGRESS_PROFILE: 'selfHostedService', McpDnsResolutionError: class extends Error {}, McpDomainNotAllowedError: class extends Error {}, McpSsrfError: class extends Error {}, diff --git a/apps/sim/lib/mcp/pinned-fetch.test.ts b/apps/sim/lib/mcp/pinned-fetch.test.ts index 3d9d8518b8b..f059750f648 100644 --- a/apps/sim/lib/mcp/pinned-fetch.test.ts +++ b/apps/sim/lib/mcp/pinned-fetch.test.ts @@ -31,6 +31,7 @@ vi.mock('@sim/security/ssrf', () => ({ isPrivateIp: (ip: string) => ip.startsWith('127.') || ip.startsWith('10.') || ip === '::1', })) vi.mock('@/lib/mcp/domain-check', () => ({ + MCP_EGRESS_PROFILE: 'selfHostedService', validateMcpServerSsrf: mockValidateMcpServerSsrf, })) @@ -55,7 +56,9 @@ describe('createGuardedMcpFetch', () => { // No dispatcher options: no `allowH2` opt-in (h1.1 default) and no Agent-level // maxResponseSize — the standalone GET SSE stream must stream unbounded (the body cap // is applied per-response to non-GET exchanges instead). - expect(mockCreateGuardedFetchWithDispatcher).toHaveBeenCalledWith() + expect(mockCreateGuardedFetchWithDispatcher).toHaveBeenCalledWith({ + profile: 'selfHostedService', + }) void close() expect(mockDestroy).toHaveBeenCalledTimes(1) diff --git a/apps/sim/lib/mcp/pinned-fetch.ts b/apps/sim/lib/mcp/pinned-fetch.ts index d5169a2b696..3d7a638f3be 100644 --- a/apps/sim/lib/mcp/pinned-fetch.ts +++ b/apps/sim/lib/mcp/pinned-fetch.ts @@ -6,7 +6,7 @@ import { createPinnedFetchWithDispatcher, createSsrfGuardedFetchWithDispatcher, } from '@/lib/core/security/input-validation.server' -import { validateMcpServerSsrf } from '@/lib/mcp/domain-check' +import { MCP_EGRESS_PROFILE, validateMcpServerSsrf } from '@/lib/mcp/domain-check' import { McpError } from '@/lib/mcp/types' const logger = createLogger('McpOauthFetch') @@ -95,7 +95,9 @@ function capResponseBody(response: Response, maxBytes: number): Response { * behavior and its security property for exactly this carve-out. */ export function createPinnedPrivateMcpFetch(resolvedIP: string): GuardedMcpFetch { - const { fetch: pinnedFetch, dispatcher } = createPinnedFetchWithDispatcher(resolvedIP) + const { fetch: pinnedFetch, dispatcher } = createPinnedFetchWithDispatcher(resolvedIP, { + profile: MCP_EGRESS_PROFILE, + }) const capped: typeof fetch = async (input, init) => { const method = init?.method ?? (input instanceof Request ? input.method : 'GET') const response = await pinnedFetch(input, init) @@ -107,7 +109,9 @@ export function createPinnedPrivateMcpFetch(resolvedIP: string): GuardedMcpFetch } export function createGuardedMcpFetch(): GuardedMcpFetch { - const { fetch: guardedFetch, dispatcher } = createSsrfGuardedFetchWithDispatcher() + const { fetch: guardedFetch, dispatcher } = createSsrfGuardedFetchWithDispatcher({ + profile: MCP_EGRESS_PROFILE, + }) // Per-request phase logging: a stalled transport request (e.g. a first `initialize` that hangs // to the client timeout) shows whether it stalls BEFORE response headers ("request" with no // "response headers" = connect/request stall) or AFTER ("response headers" then the SDK's @@ -297,12 +301,14 @@ export function createSsrfGuardedMcpFetch(timeoutMs: number = OAUTH_FETCH_TIMEOU // would filter the address, and an unguarded fallback would reopen rebinding — // keep the legacy pin to the validated address for exactly this case. const pinned = createPinnedFetchWithDispatcher(resolvedIP, { + profile: MCP_EGRESS_PROFILE, maxResponseSize: MAX_OAUTH_RESPONSE_BYTES, }) dispatcher = pinned.dispatcher response = await withDeadline(pinned.fetch(url, { ...init, signal }), signal) } else if (resolvedIP) { const guarded = createSsrfGuardedFetchWithDispatcher({ + profile: MCP_EGRESS_PROFILE, maxResponseSize: MAX_OAUTH_RESPONSE_BYTES, }) dispatcher = guarded.dispatcher diff --git a/apps/sim/lib/mcp/service-pool.test.ts b/apps/sim/lib/mcp/service-pool.test.ts index 2903558d561..2ea39812211 100644 --- a/apps/sim/lib/mcp/service-pool.test.ts +++ b/apps/sim/lib/mcp/service-pool.test.ts @@ -101,6 +101,7 @@ const SERVER_ROW = { } vi.mock('@/lib/mcp/domain-check', () => ({ + MCP_EGRESS_PROFILE: 'selfHostedService', isMcpDomainAllowed: () => true, validateMcpDomain: () => {}, validateMcpServerSsrf: async () => '203.0.113.10', diff --git a/apps/sim/lib/mcp/service.test.ts b/apps/sim/lib/mcp/service.test.ts index bf7706168a6..3abab84350a 100644 --- a/apps/sim/lib/mcp/service.test.ts +++ b/apps/sim/lib/mcp/service.test.ts @@ -99,6 +99,7 @@ vi.mock('@/lib/mcp/connection-manager', () => ({ })) vi.mock('@/lib/mcp/domain-check', () => ({ + MCP_EGRESS_PROFILE: 'selfHostedService', isMcpDomainAllowed: (...args: unknown[]) => mockIsDomainAllowed(...args), validateMcpDomain: (...args: unknown[]) => mockValidateDomain(...args), validateMcpServerSsrf: (...args: unknown[]) => mockValidateSsrf(...args), diff --git a/apps/sim/providers/azure-anthropic/index.test.ts b/apps/sim/providers/azure-anthropic/index.test.ts index 46d9122a9d7..df16b8a612c 100644 --- a/apps/sim/providers/azure-anthropic/index.test.ts +++ b/apps/sim/providers/azure-anthropic/index.test.ts @@ -82,7 +82,9 @@ describe('azureAnthropicProvider — SSRF pinning', () => { 'azureEndpoint', 'configuredEndpoint' ) - expect(mockCreatePinnedFetch).toHaveBeenCalledWith('203.0.113.10') + expect(mockCreatePinnedFetch).toHaveBeenCalledWith('203.0.113.10', { + profile: 'configuredEndpoint', + }) expect(buildClientOptions()).toMatchObject({ fetch: sentinelFetch }) }) diff --git a/apps/sim/providers/azure-anthropic/index.ts b/apps/sim/providers/azure-anthropic/index.ts index 78ded084fc0..0a39b6040c9 100644 --- a/apps/sim/providers/azure-anthropic/index.ts +++ b/apps/sim/providers/azure-anthropic/index.ts @@ -48,7 +48,7 @@ export const azureAnthropicProvider: ProviderConfig = { throw new Error('Invalid Azure Anthropic endpoint: could not resolve a pinnable IP address') } pinnedIP = validation.resolvedIP - pinnedFetch = createPinnedFetch(pinnedIP) + pinnedFetch = createPinnedFetch(pinnedIP, { profile: 'configuredEndpoint' }) } const apiKey = request.apiKey diff --git a/apps/sim/providers/azure-openai/index.test.ts b/apps/sim/providers/azure-openai/index.test.ts index 98e4099b0a3..0b31d9797c5 100644 --- a/apps/sim/providers/azure-openai/index.test.ts +++ b/apps/sim/providers/azure-openai/index.test.ts @@ -147,7 +147,9 @@ describe('azureOpenAIProvider — SSRF pinning', () => { 'azureEndpoint', 'configuredEndpoint' ) - expect(mockCreatePinnedFetch).toHaveBeenCalledWith('203.0.113.10') + expect(mockCreatePinnedFetch).toHaveBeenCalledWith('203.0.113.10', { + profile: 'configuredEndpoint', + }) expect(responsesConfig().fetch).toBe(sentinelFetch) }) @@ -203,7 +205,9 @@ describe('azureOpenAIProvider — SSRF pinning', () => { }) ) - expect(mockCreatePinnedFetch).toHaveBeenCalledWith('203.0.113.10') + expect(mockCreatePinnedFetch).toHaveBeenCalledWith('203.0.113.10', { + profile: 'configuredEndpoint', + }) expect(azureOpenAIArgs[0]).toMatchObject({ fetch: sentinelFetch }) }) diff --git a/apps/sim/providers/azure-openai/index.ts b/apps/sim/providers/azure-openai/index.ts index 1428c62815e..de3f3a612ec 100644 --- a/apps/sim/providers/azure-openai/index.ts +++ b/apps/sim/providers/azure-openai/index.ts @@ -689,7 +689,7 @@ export const azureOpenAIProvider: ProviderConfig = { if (!validation.resolvedIP) { throw new Error('Invalid Azure OpenAI endpoint: could not resolve a pinnable IP address') } - pinnedFetch = createPinnedFetch(validation.resolvedIP) + pinnedFetch = createPinnedFetch(validation.resolvedIP, { profile: 'configuredEndpoint' }) } const apiKey = request.apiKey diff --git a/apps/sim/providers/vllm/index.test.ts b/apps/sim/providers/vllm/index.test.ts index b520e90a4a2..3b6ecd20c2d 100644 --- a/apps/sim/providers/vllm/index.test.ts +++ b/apps/sim/providers/vllm/index.test.ts @@ -191,7 +191,9 @@ describe('vllmProvider', () => { 'vLLM endpoint', 'selfHostedService' ) - expect(mockCreatePinnedFetch).toHaveBeenCalledWith('203.0.113.10') + expect(mockCreatePinnedFetch).toHaveBeenCalledWith('203.0.113.10', { + profile: 'selfHostedService', + }) expect(openAIArgs[0].baseURL).toBe('https://my-vllm.example.com/v1') expect(openAIArgs[0].fetch).toBe(pinnedFetchFn) }) diff --git a/apps/sim/providers/vllm/index.ts b/apps/sim/providers/vllm/index.ts index 24c9c5135bb..5e15b24e64b 100644 --- a/apps/sim/providers/vllm/index.ts +++ b/apps/sim/providers/vllm/index.ts @@ -143,7 +143,7 @@ export const vllmProvider: ProviderConfig = { throw new Error('Invalid vLLM endpoint: could not resolve a pinnable IP address') } pinnedIP = validation.resolvedIP - pinnedFetch = createPinnedFetch(pinnedIP) + pinnedFetch = createPinnedFetch(pinnedIP, { profile: 'selfHostedService' }) } const apiKey = request.apiKey || env.VLLM_API_KEY || 'empty' diff --git a/apps/sim/tools/bitbucket/utils.server.ts b/apps/sim/tools/bitbucket/utils.server.ts index afed0ef03a5..2178bd84a08 100644 --- a/apps/sim/tools/bitbucket/utils.server.ts +++ b/apps/sim/tools/bitbucket/utils.server.ts @@ -151,7 +151,7 @@ export async function resolveBitbucketPullRequestRedirect( const { fetch: pinnedFetch, dispatcher } = createPinnedFetchWithDispatcher( initialValidation.resolvedIP, - { maxResponseSize: 64 * 1024 } + { profile: 'configuredEndpoint', maxResponseSize: 64 * 1024 } ) let initial: Response | null = null let initialStatus: number | null = null diff --git a/packages/security/src/egress.ts b/packages/security/src/egress.ts index 7e8da577134..321624f0ef4 100644 --- a/packages/security/src/egress.ts +++ b/packages/security/src/egress.ts @@ -132,6 +132,11 @@ export interface EgressPolicy { * address check outright; a named allowlist is the supported form. */ readonly allowPrivate: boolean + /** + * Whether the non-HTTP service ports are refused on an unvouched destination. + * False for operator-run services, which legitimately bind arbitrary ports. + */ + readonly denyServicePorts: boolean readonly allowedHosts: readonly HostPattern[] readonly allowedRanges: readonly CidrRange[] } @@ -151,6 +156,8 @@ export interface EgressPolicySpec { readonly allowLoopback?: boolean /** Whether every private address is vouched for. See {@link EgressPolicy.allowPrivate}. */ readonly allowPrivate?: boolean + /** Whether to refuse the non-HTTP service ports. Defaults to `true`. */ + readonly denyServicePorts?: boolean /** * Names of the settings these lists came from, used verbatim in the error a * malformed entry throws so the operator knows which value to fix. @@ -221,6 +228,7 @@ export function createEgressPolicy(spec: EgressPolicySpec = {}): EgressPolicy { insecureHttp: spec.insecureHttp ?? 'never', allowLoopback: spec.allowLoopback ?? false, allowPrivate: spec.allowPrivate ?? false, + denyServicePorts: spec.denyServicePorts ?? true, allowedHosts: splitEntries(spec.allowedHosts).map((entry) => parseHostPattern(entry, sourceNames.hosts) ), @@ -333,7 +341,7 @@ function checkSchemeAndPort(url: URL, vouched: boolean, policy: EgressPolicy): E return deny('insecure-scheme', `plain http to ${url.hostname}`) } - if (!vouched && url.port) { + if (policy.denyServicePorts && !vouched && url.port) { const port = Number.parseInt(url.port, 10) if (DENIED_PORTS.has(port)) { return deny('port-denied', `port ${port}`) From 58cea60dce5163ef53e4055d7409ad3b63ae881a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 18:00:26 -0700 Subject: [PATCH 08/20] docs(egress): document the allowlist in helm, troubleshooting and MCP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Helm chart was the gap. Kubernetes is where in-cluster service names matter most, and the chart neither documented nor schema-validated the setting that makes them reachable. Both variables now sit beside AUTH_TRUSTED_PROXIES in values.yaml and values.schema.json, and `helm template` carries them into the chart-managed Secret. The self-hosting troubleshooting page gains the failure as its own entry, next to the Ollama one that already explains why `localhost` is the container. Someone who hits this reads that page, not the security reference. The MCP page said the domain allowlist controls which servers are reachable. It governs which domains may be used; where those domains resolve is a separate check that now also applies. Adds an end-to-end test that runs the guard over a real socket against a real private interface — no mocked transport, no mocked DNS. It asserts the #7200 path directly: refused while unlisted, reached over plain HTTP once the range is named, still refused for a content-provenance URL. It skips on a host with no private interface rather than depending on outbound DNS. --- apps/docs/content/docs/agents/mcp.mdx | 2 + .../platform/self-hosting/troubleshooting.mdx | 18 ++++ .../security/egress-end-to-end.server.test.ts | 98 +++++++++++++++++++ helm/sim/values.schema.json | 8 ++ helm/sim/values.yaml | 8 ++ 5 files changed, 134 insertions(+) create mode 100644 apps/sim/lib/core/security/egress-end-to-end.server.test.ts diff --git a/apps/docs/content/docs/agents/mcp.mdx b/apps/docs/content/docs/agents/mcp.mdx index 6464ed8bb6c..f970a7abca6 100644 --- a/apps/docs/content/docs/agents/mcp.mdx +++ b/apps/docs/content/docs/agents/mcp.mdx @@ -85,6 +85,8 @@ Tool validation badges appear on servers with issues — for example, if a tool Self-hosted deployments can restrict which MCP server domains are allowed by setting the `ALLOWED_MCP_DOMAINS` environment variable (comma-separated list). When set, only servers on approved domains can be added. When unset, all domains are allowed. +This governs which domains may be used. It is separate from where those domains are allowed to resolve: an MCP server on a private address is reached by naming it in `EGRESS_ALLOWED_HOSTS` or `EGRESS_ALLOWED_IP_RANGES`, described in [Security](/platform/self-hosting/security#the-ssrf-boundary). Both checks apply. + ## Using MCP Tools in Agents Once MCP servers are configured, their tools become available within your agent blocks: diff --git a/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx b/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx index 6e00170b57f..e120504dc01 100644 --- a/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx +++ b/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx @@ -25,6 +25,24 @@ OLLAMA_URL=http://host.docker.internal:11434 # macOS/Windows OLLAMA_URL=http://192.168.1.x:11434 # Linux (use actual IP) ``` +## A Workflow Cannot Reach a Service on Your Network + +Outbound requests to private, reserved, and loopback addresses are blocked by default, so a workflow pointed at your Docker host, a LAN service, or a Kubernetes service name fails with a message naming the address it resolved to. + +Name the destination: + +```bash +EGRESS_ALLOWED_HOSTS=host.docker.internal,*.svc.cluster.local +EGRESS_ALLOWED_IP_RANGES=10.0.0.0/8 +``` + +Naming a destination also permits plain HTTP to it and lifts the blocked-port list for it. Cloud metadata endpoints (`169.254.169.254` and equivalents) stay blocked however broad the list is, and both variables are ignored on Sim Cloud. + +Two things this does not cover: + +- Inside a container `localhost` is the container itself, so it will never reach a service on your host. Use `host.docker.internal` (the Compose files map it) and name it above. +- URLs harvested from content or from a third-party API response — an image URL, a file imported by URL — never reach a private network, allowlist or not. + ## LM Studio Requests Route to Ollama Sim identifies dynamically discovered LM Studio and vLLM models by their `vllm/` prefix. If the endpoint is unavailable and you manually enter the raw LM Studio model identifier, Sim treats that unknown identifier as an Ollama model. diff --git a/apps/sim/lib/core/security/egress-end-to-end.server.test.ts b/apps/sim/lib/core/security/egress-end-to-end.server.test.ts new file mode 100644 index 00000000000..dbc1ed30c3c --- /dev/null +++ b/apps/sim/lib/core/security/egress-end-to-end.server.test.ts @@ -0,0 +1,98 @@ +/** + * @vitest-environment node + * + * End-to-end over a real socket: the guard resolves, classifies, pins, and + * connects. Nothing about the transport is mocked, which is what makes this the + * check that the policy is actually wired to the wire. + * + * Uses a private address on a real interface rather than a public DNS name that + * resolves to loopback, so the suite needs no outbound DNS. + */ +import { createServer, type Server } from 'node:http' +import type { AddressInfo } from 'node:net' +import { networkInterfaces } from 'node:os' +import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' + +/** A non-loopback RFC1918 address on this machine, or null when there is none. */ +function privateInterfaceAddress(): string | null { + for (const addresses of Object.values(networkInterfaces())) { + for (const address of addresses ?? []) { + if (address.family !== 'IPv4' || address.internal) continue + if (/^(10\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.)/.test(address.address)) { + return address.address + } + } + } + return null +} + +const host = privateInterfaceAddress() + +let server: Server +let port = 0 + +beforeAll(async () => { + if (!host) return + server = createServer((request, response) => { + request.resume() + response.writeHead(200, { 'content-type': 'text/plain' }) + response.end('reached') + }) + await new Promise((resolve) => server.listen(0, host, resolve)) + port = (server.address() as AddressInfo).port +}) + +afterAll(async () => { + if (server) await new Promise((resolve) => server.close(() => resolve())) + resetEnvFlagsMock() +}) + +afterEach(resetEnvFlagsMock) + +// Skipped on a host with no private interface (some CI sandboxes); the policy +// itself is covered without a socket in packages/security. +describe.skipIf(!host)('issue #7200 — reaching a service on a private network', () => { + it('refuses an unlisted destination and names the setting that would permit it', async () => { + await expect( + secureFetchWithValidation(`http://${host}:${port}/`, { profile: 'requestTarget' }) + ).rejects.toThrow(/EGRESS_ALLOWED_HOSTS/) + }) + + it('reaches it over plain HTTP once the operator names the range', async () => { + setEnvFlags({ egressAllowedIpRanges: `${host}/32` }) + + const response = await secureFetchWithValidation(`http://${host}:${port}/`, { + profile: 'requestTarget', + }) + + expect(response.status).toBe(200) + expect(await response.text()).toBe('reached') + }) + + it('does not extend that reach to a content-provenance URL', async () => { + setEnvFlags({ egressAllowedIpRanges: `${host}/32` }) + + await expect( + secureFetchWithValidation(`http://${host}:${port}/`, { profile: 'contentFetch' }) + ).rejects.toThrow() + }) + + it('reaches a loopback service without any allowlist, as a self-hosted deployment does', async () => { + const local = createServer((request, response) => { + request.resume() + response.end('local') + }) + await new Promise((resolve) => local.listen(0, '127.0.0.1', resolve)) + const localPort = (local.address() as AddressInfo).port + try { + const response = await secureFetchWithValidation(`http://localhost:${localPort}/`, { + profile: 'selfHostedService', + }) + expect(await response.text()).toBe('local') + } finally { + await new Promise((resolve) => local.close(() => resolve())) + } + }) +}) diff --git a/helm/sim/values.schema.json b/helm/sim/values.schema.json index 920918cd422..f11f92789ad 100644 --- a/helm/sim/values.schema.json +++ b/helm/sim/values.schema.json @@ -159,6 +159,14 @@ "type": "string", "description": "Comma-separated reverse-proxy IPs or CIDR ranges in front of the app (e.g. '10.0.0.0/16'). Better Auth walks x-forwarded-for right to left, skips these hops, and uses the first untrusted address as the client IP." }, + "EGRESS_ALLOWED_HOSTS": { + "type": "string", + "description": "Comma-separated hostnames on a private network that outbound requests may reach, leading wildcard allowed (e.g. '*.svc.cluster.local'). Naming a destination also permits plain HTTP to it and lifts the blocked-port list for it. Cloud metadata endpoints stay blocked regardless. Ignored on the hosted platform." + }, + "EGRESS_ALLOWED_IP_RANGES": { + "type": "string", + "description": "Comma-separated CIDRs or IPs on a private network that outbound requests may reach (e.g. '10.0.0.0/8'). Cloud metadata endpoints stay blocked regardless. Ignored on the hosted platform." + }, "SSO_TRUSTED_PROVIDER_IDS": { "type": "string", "description": "Comma-separated SSO provider IDs to trust for automatic account linking when an SSO sign-in matches an existing account's email. Only needed for IdPs that do not assert email_verified. Merged into Better Auth accountLinking.trustedProviders." diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index 785272fb8cb..1df20786d5d 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -84,6 +84,14 @@ app: # these hops, and uses the first untrusted address as the client IP. Required for correct # session IPs and rate-limit keying behind a multi-hop proxy chain (e.g. "10.0.0.0/16"). AUTH_TRUSTED_PROXIES: "" + # EGRESS_ALLOWED_HOSTS / EGRESS_ALLOWED_IP_RANGES: destinations on your private network that + # workflows may reach. Outbound requests to private, reserved, and loopback addresses are + # blocked by default; naming a destination here permits it, allows plain HTTP to it, and lifts + # the blocked-port list for it. In-cluster services need the hostname form, e.g. + # "*.svc.cluster.local" or "vllm.ai.svc.cluster.local". Cloud metadata endpoints stay blocked + # however broad the list is. Pair with a NetworkPolicy that constrains what the pod can reach. + EGRESS_ALLOWED_HOSTS: "" + EGRESS_ALLOWED_IP_RANGES: "" # SOCKET_SERVER_URL: Auto-detected when realtime.enabled=true (uses internal service) # NEXT_PUBLIC_SOCKET_URL: public WebSocket URL for browsers. Leave empty to default to the # page's own origin (assumes the ingress/reverse proxy routes /socket.io to the realtime service). From e9b7f0ffa4aa6e56bc6cc6f934d6052fd4fb478f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 18:06:44 -0700 Subject: [PATCH 09/20] chore(helm): bump chart to 1.7.0 for the egress allowlist values --- helm/sim/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/sim/Chart.yaml b/helm/sim/Chart.yaml index 2b09f2bde3a..08d4dd33c1a 100644 --- a/helm/sim/Chart.yaml +++ b/helm/sim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: sim description: A Helm chart for Sim - the open-source AI workspace where teams build, deploy, and manage AI agents type: application -version: 1.6.2 +version: 1.7.0 appVersion: "v0.7.44" kubeVersion: ">=1.25.0-0" home: https://sim.ai From b78574e58a67736dfc5ea31ff098189e697f17fa Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 18:42:03 -0700 Subject: [PATCH 10/20] refactor(egress): drop a redundant policy dial and prove resolvedIP at the type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `denyServicePorts` was wrong on inspection. The port list is already lifted for a vouched destination, and a private service reachable at all is one the operator allowlisted — so the dial's only effect was letting `selfHostedService` reach a *public* host on port 22/3306/5432/…, which none of vLLM, Jupyter, 1Password, ClickHouse or MCP needs. On the hosted platform, where nothing is ever vouched, it was a quiet loosening. Removed, with a test pinning that a service port on a public host stays refused whatever the profile. `AsyncValidationResult` was a bag of optionals, so twenty call sites wrote `validation.resolvedIP!` to get the pin address past the compiler — an assertion on the one value that must not be undefined, since it is what the socket dials. It is a discriminated union now and every assertion is gone, the compiler proving what they asserted. No call site changed shape. Adds a test fixing the hosted posture in place: private, loopback and metadata addresses refused for every profile, service ports refused on a public host, plain HTTP still available only to `selfHostedService`. --- .../sim/app/api/tools/imap/mailboxes/route.ts | 2 +- apps/sim/lib/core/security/egress/profiles.ts | 10 ++--- .../core/security/input-validation.server.ts | 13 +++--- apps/sim/lib/internal/clickhouse/client.ts | 2 +- apps/sim/lib/internal/google-drive/client.ts | 2 +- apps/sim/lib/internal/onepassword/client.ts | 2 +- apps/sim/lib/internal/slack/operations.ts | 2 +- apps/sim/lib/internal/stt/operations.ts | 2 +- .../lib/internal/textract/document-input.ts | 2 +- apps/sim/lib/internal/whatsapp/operations.ts | 2 +- apps/sim/lib/mcp/domain-check.ts | 2 +- .../lib/uploads/utils/file-utils.server.ts | 2 +- apps/sim/lib/webhooks/polling/imap.ts | 2 +- apps/sim/lib/webhooks/polling/rss.ts | 2 +- apps/sim/lib/webhooks/providers/emailbison.ts | 4 +- .../lib/webhooks/providers/microsoft-teams.ts | 2 +- apps/sim/lib/webhooks/providers/slack.ts | 2 +- apps/sim/tools/index.ts | 2 +- .../src/egress-hosted-posture.test.ts | 44 +++++++++++++++++++ packages/security/src/egress.test.ts | 9 ++++ packages/security/src/egress.ts | 16 +++---- 21 files changed, 84 insertions(+), 42 deletions(-) create mode 100644 packages/security/src/egress-hosted-posture.test.ts diff --git a/apps/sim/app/api/tools/imap/mailboxes/route.ts b/apps/sim/app/api/tools/imap/mailboxes/route.ts index b66c9eb34d4..e0507a590c5 100644 --- a/apps/sim/app/api/tools/imap/mailboxes/route.ts +++ b/apps/sim/app/api/tools/imap/mailboxes/route.ts @@ -49,7 +49,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } const client = new ImapFlow({ - host: hostValidation.resolvedIP!, + host: hostValidation.resolvedIP, servername: host, port, secure, diff --git a/apps/sim/lib/core/security/egress/profiles.ts b/apps/sim/lib/core/security/egress/profiles.ts index 8f62d904a32..d771c88851f 100644 --- a/apps/sim/lib/core/security/egress/profiles.ts +++ b/apps/sim/lib/core/security/egress/profiles.ts @@ -42,9 +42,9 @@ import { * be named rather than assumed. * - `selfHostedService` — a configured endpoint for software normally run * on-prem: vLLM, Jupyter, 1Password Connect, ClickHouse, an MCP server. Same - * reachability as `configuredEndpoint`, but plain HTTP and an arbitrary port - * are expected rather than conditional, because that is how these are - * ordinarily deployed inside a network. + * reachability as `configuredEndpoint`, but plain HTTP is expected rather than + * conditional, because that is how these are ordinarily served inside a + * network. An arbitrary internal port comes with being allowlisted. * - `proxy` — the egress proxy itself. Held to the strictest rule of all, * because it is the component that decides where everything else may go: plain * HTTP by protocol, but public destinations only, and no allowlist. @@ -68,8 +68,6 @@ interface ProfileSpec { readonly honorsAllowlist: boolean /** When plain HTTP is acceptable for this provenance. */ readonly insecureHttp: InsecureHttpPolicy - /** Whether the non-HTTP service ports are refused. Defaults to refusing them. */ - readonly denyServicePorts?: boolean /** * Whether loopback is reachable without being allowlisted, off the hosted * platform. True for the profiles whose URLs someone deliberately configured — @@ -99,7 +97,6 @@ const PROFILE_SPECS: Record = { honorsAllowlist: true, insecureHttp: 'always', allowLoopbackOffHosted: true, - denyServicePorts: false, }, requestTarget: { honorsAllowlist: true, @@ -149,7 +146,6 @@ function buildPolicies(config: DeploymentConfig): Record[2], href: string, address: string) { + return evaluateAddress(new URL(href), address, policy) +} + +describe('hosted platform', () => { + it.each([ + ['https://x.example/', '10.0.0.5', 'RFC1918'], + ['https://x.example/', '127.0.0.1', 'loopback'], + ['https://x.example/', '169.254.169.254', 'metadata'], + ['https://x.example/', '192.168.1.1', 'RFC1918'], + ])('refuses %s resolving to %s — %s', (href, address) => { + expect(decide(publicApi, href, address).allowed).toBe(false) + expect(decide(selfHostedService, href, address).allowed).toBe(false) + }) + + it('refuses a service port on a public host', () => { + expect(decide(publicApi, 'https://x.example:5432/', '93.184.216.34').allowed).toBe(false) + expect(decide(selfHostedService, 'http://x.example:5432/', '93.184.216.34').allowed).toBe(false) + }) + + it('permits ordinary public HTTPS', () => { + expect(decide(publicApi, 'https://x.example/', '93.184.216.34').allowed).toBe(true) + }) + + it('keeps plain HTTP available only to the self-hosted-service profile', () => { + expect(decide(publicApi, 'http://x.example/', '93.184.216.34').allowed).toBe(false) + expect(decide(selfHostedService, 'http://x.example/', '93.184.216.34').allowed).toBe(true) + }) +}) diff --git a/packages/security/src/egress.test.ts b/packages/security/src/egress.test.ts index a58ac6dc3b2..3c9f42060ec 100644 --- a/packages/security/src/egress.test.ts +++ b/packages/security/src/egress.test.ts @@ -240,9 +240,18 @@ describe('denied ports', () => { }) it('lifts the port denylist for a vouched destination', () => { + // Being vouched is what lifts it, which is why no profile needs its own + // opt-out: an internal Elasticsearch is reachable because it was named. expect(decide(selfHosted, 'http://host.docker.internal:9200/', '10.0.0.5').allowed).toBe(true) }) + it('keeps refusing a service port on a public host, whatever the profile', () => { + const selfHostedService = createEgressPolicy({ insecureHttp: 'always', allowLoopback: true }) + expect(reason(selfHostedService, 'http://example.com:5432/', '93.184.216.34')).toBe( + 'port-denied' + ) + }) + it('leaves ordinary ports alone', () => { expect(decide(hosted, 'https://example.com:8443/', '93.184.216.34').allowed).toBe(true) }) diff --git a/packages/security/src/egress.ts b/packages/security/src/egress.ts index 321624f0ef4..1618834305e 100644 --- a/packages/security/src/egress.ts +++ b/packages/security/src/egress.ts @@ -82,8 +82,10 @@ const METADATA_ADDRESSES: readonly string[] = [ /** * Ports that speak a non-HTTP protocol on a conventional deployment. Refusing * them blunts protocol-smuggling through a URL the caller does not control. - * Lifted for an allowlisted destination, so an operator can reach their own - * internal Elasticsearch on 9200 after naming it. + * + * Lifted for a vouched destination, which is what lets an operator reach their + * own internal Elasticsearch on 9200 after naming it — so no profile needs to + * opt out of the list separately. */ const DENIED_PORTS: ReadonlySet = new Set([ 22, // SSH @@ -132,11 +134,6 @@ export interface EgressPolicy { * address check outright; a named allowlist is the supported form. */ readonly allowPrivate: boolean - /** - * Whether the non-HTTP service ports are refused on an unvouched destination. - * False for operator-run services, which legitimately bind arbitrary ports. - */ - readonly denyServicePorts: boolean readonly allowedHosts: readonly HostPattern[] readonly allowedRanges: readonly CidrRange[] } @@ -156,8 +153,6 @@ export interface EgressPolicySpec { readonly allowLoopback?: boolean /** Whether every private address is vouched for. See {@link EgressPolicy.allowPrivate}. */ readonly allowPrivate?: boolean - /** Whether to refuse the non-HTTP service ports. Defaults to `true`. */ - readonly denyServicePorts?: boolean /** * Names of the settings these lists came from, used verbatim in the error a * malformed entry throws so the operator knows which value to fix. @@ -228,7 +223,6 @@ export function createEgressPolicy(spec: EgressPolicySpec = {}): EgressPolicy { insecureHttp: spec.insecureHttp ?? 'never', allowLoopback: spec.allowLoopback ?? false, allowPrivate: spec.allowPrivate ?? false, - denyServicePorts: spec.denyServicePorts ?? true, allowedHosts: splitEntries(spec.allowedHosts).map((entry) => parseHostPattern(entry, sourceNames.hosts) ), @@ -341,7 +335,7 @@ function checkSchemeAndPort(url: URL, vouched: boolean, policy: EgressPolicy): E return deny('insecure-scheme', `plain http to ${url.hostname}`) } - if (policy.denyServicePorts && !vouched && url.port) { + if (!vouched && url.port) { const port = Number.parseInt(url.port, 10) if (DENIED_PORTS.has(port)) { return deny('port-denied', `port ${port}`) From f07c64e2d22e35223072e3d2340b0fcf5f482338 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 19:06:41 -0700 Subject: [PATCH 11/20] fix(helm): make lint:helm run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unrelated to the egress work, but broken in a way worth a one-line fix while the chart is already in the diff. `lint:helm` pointed at `helm/sim/test/values-lint.yaml`, which has never existed in this repository — the script has been dead since the chart landed in #813. Nobody hit it because CI lints the chart directly rather than through the script. The reason it was never noticed is the other half: `.gitignore` ignored `helm/sim/test`, so whoever wrote the script had that file locally and it worked only on their machine. That entry is removed too. It was also a trap, sitting one letter away from `helm/sim/tests`, the real committed helm-unittest directory — anyone who created the singular path would have had it silently ignored. The script now runs exactly what the workflow runs, so a developer checking locally gets the same answer as the gate. `--strict` is dropped for the same reason: a local check stricter than CI reports failures the gate will not. --- .gitignore | 3 --- package.json | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 4371d6d73f7..6bbecf8f295 100644 --- a/.gitignore +++ b/.gitignore @@ -84,9 +84,6 @@ start-collector.sh # IntelliJ .idea -## Helm Chart Tests -helm/sim/test - ## Claude Code .claude/launch.json .claude/worktrees/ diff --git a/package.json b/package.json index 7937b83cb66..297e32f4656 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "format:check": "turbo run format:check", "lint": "turbo run lint", "lint:check": "turbo run lint:check", - "lint:helm": "helm lint ./helm/sim --strict --values ./helm/sim/test/values-lint.yaml", + "lint:helm": "helm lint helm/sim --values helm/sim/ci/default-values.yaml", "lint:all": "turbo run lint && bun run lint:helm", "check": "turbo run format:check", "check:egress-boundary": "bun run scripts/check-egress-boundary.ts", From 1584c3a060d3f2404ee6d609f2574a8790e7b042 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 19:51:48 -0700 Subject: [PATCH 12/20] fix(egress): address the second review round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified each finding before changing anything; one was rejected as unreachable. The P1 is real and mine. `validateMcpServerSsrf` used one profile for every caller, so OAuth legs — whose URLs come out of authorization-server metadata, as that file's own docstring says — inherited configured-endpoint privileges. A hostile MCP server could have steered discovery, token exchange or revocation at whatever the operator allowlisted for their own workflows. Those legs take `contentFetch` now. That made two branches in the OAuth fetch dead, one of them badly: a null resolution fell through to an unguarded `globalThis.fetch`. Under `contentFetch` the only way to get there is an unresolved env-var hostname, which is never a real authorization-server URL, so it refuses instead. No leg of that flow reaches the network unguarded now. Also restored: 26 tests for `validateSupabaseProjectId`, which is live in two production call sites. Its suite was nested inside the `validateMondayColumnId` describe on staging, so removing that dead block took it as collateral. It is top-level now, where it belongs. The rest: - The boundary script missed `await import()` and `require()`, and falsely flagged `import type`. All four forms are covered and verified with probes. - `validateVendorHostedUrl` accepted plain HTTP to ServiceNow, Workday and Databricks if an operator happened to allowlist the vendor's domain. They are HTTPS-only SaaS; enforced before the general policy runs. - STT resolved an internal file to a presigned storage URL and then judged it as content, so a self-hosted MinIO on a private address would have failed. Same provenance threading as textract. - The env-flags warning printed the allowlist entries; internal topology stays out of a log line that may leave the deployment. - The testing mock served egress config while hosted, which production never does — a test could assert a posture that cannot exist. - The security doc's example allowlisted `*.svc.cluster.local` and all of `10.0.0.0/8` directly under a warning to name specific hosts. - `--strict` restored on lint:helm; it passes, and the original intent stands. - My own end-to-end test rejected on scheme rather than address, so it would have passed even if `contentFetch` started honouring the allowlist. Not changed: the guarded lookup ignoring the request profile. Both callers route a private resolution to the pinned path (`client.ts`, `pinned-fetch.ts`), so an allowlisted private address never reaches that lookup, and it fails closed. --- .../docs/platform/self-hosting/security.mdx | 6 +- .../mcp/servers/test-connection/route.test.ts | 1 + apps/sim/lib/core/config/env-flags.ts | 5 +- .../security/egress-end-to-end.server.test.ts | 6 +- .../core/security/input-validation.test.ts | 120 ++++++++++++++++++ .../sim/lib/core/security/input-validation.ts | 7 + apps/sim/lib/internal/stt/operations.ts | 10 +- apps/sim/lib/mcp/domain-check.ts | 22 +++- apps/sim/lib/mcp/oauth/revoke.test.ts | 6 +- .../orchestration/server-lifecycle.test.ts | 1 + apps/sim/lib/mcp/pinned-fetch.test.ts | 68 ++++------ apps/sim/lib/mcp/pinned-fetch.ts | 49 ++++--- apps/sim/lib/mcp/service-pool.test.ts | 2 + apps/sim/lib/mcp/service.test.ts | 2 + package.json | 2 +- packages/testing/src/mocks/env-flags.mock.ts | 23 +++- scripts/check-egress-boundary.ts | 49 +++++-- 17 files changed, 277 insertions(+), 102 deletions(-) diff --git a/apps/docs/content/docs/platform/self-hosting/security.mdx b/apps/docs/content/docs/platform/self-hosting/security.mdx index 36f7e938e3c..cb9d30f97f3 100644 --- a/apps/docs/content/docs/platform/self-hosting/security.mdx +++ b/apps/docs/content/docs/platform/self-hosting/security.mdx @@ -149,10 +149,12 @@ Content fetches never reach a private destination, allowlist or not — that is Deployments frequently need to reach an internal service by name or address. Name the destinations: ```bash -EGRESS_ALLOWED_HOSTS=host.docker.internal,*.svc.cluster.local -EGRESS_ALLOWED_IP_RANGES=10.0.0.0/8,192.168.65.254/32 +EGRESS_ALLOWED_HOSTS=host.docker.internal,vllm.ai.svc.cluster.local +EGRESS_ALLOWED_IP_RANGES=10.4.2.17/32,10.4.9.0/24 ``` +A wildcard (`*.svc.cluster.local`) and a broad range (`10.0.0.0/8`) are accepted, but they hand every workflow author the whole namespace or network. Name the hosts you actually use. + Naming a destination permits plain HTTP to it and lifts the blocked-port list for it, since those are the same decision about the same host. Cloud metadata endpoints (`169.254.169.254` and equivalents) stay blocked no matter how broad the allowlist is, and both variables are ignored entirely on Sim Cloud. To reach a service on the Docker host, pair the allowlist with the host alias that Compose already sets up: diff --git a/apps/sim/app/api/mcp/servers/test-connection/route.test.ts b/apps/sim/app/api/mcp/servers/test-connection/route.test.ts index 2b685bd6bc5..e3acaed7246 100644 --- a/apps/sim/app/api/mcp/servers/test-connection/route.test.ts +++ b/apps/sim/app/api/mcp/servers/test-connection/route.test.ts @@ -51,6 +51,7 @@ vi.mock('@/lib/mcp/client', () => ({ vi.mock('@/lib/mcp/domain-check', () => ({ MCP_EGRESS_PROFILE: 'selfHostedService', + OAUTH_EGRESS_PROFILE: 'contentFetch', McpDnsResolutionError: class extends Error {}, McpDomainNotAllowedError: class extends Error {}, McpSsrfError: MockMcpSsrfError, diff --git a/apps/sim/lib/core/config/env-flags.ts b/apps/sim/lib/core/config/env-flags.ts index 0c4265f2eea..a501ceb7a5e 100644 --- a/apps/sim/lib/core/config/env-flags.ts +++ b/apps/sim/lib/core/config/env-flags.ts @@ -196,9 +196,10 @@ if (env.EGRESS_ALLOWED_HOSTS || env.EGRESS_ALLOWED_IP_RANGES) { 'EGRESS_ALLOWED_HOSTS/EGRESS_ALLOWED_IP_RANGES are set but ignored on hosted environment. Private, reserved, and loopback destinations remain blocked for security.' ) } else { + // The entries themselves are internal network topology and stay out of + // the log line, which may leave the deployment. logger.warn( - 'Private-network egress allowlist is configured. Outbound requests may reach the listed destinations. Only use this on a trusted private network.', - { hosts: env.EGRESS_ALLOWED_HOSTS, ipRanges: env.EGRESS_ALLOWED_IP_RANGES } + 'Private-network egress allowlist is configured. Outbound requests may reach the listed destinations. Only use this on a trusted private network.' ) } }) diff --git a/apps/sim/lib/core/security/egress-end-to-end.server.test.ts b/apps/sim/lib/core/security/egress-end-to-end.server.test.ts index dbc1ed30c3c..6c5c9e8cd2d 100644 --- a/apps/sim/lib/core/security/egress-end-to-end.server.test.ts +++ b/apps/sim/lib/core/security/egress-end-to-end.server.test.ts @@ -74,9 +74,11 @@ describe.skipIf(!host)('issue #7200 — reaching a service on a private network' it('does not extend that reach to a content-provenance URL', async () => { setEnvFlags({ egressAllowedIpRanges: `${host}/32` }) + // https, so the refusal has to come from the address rather than the scheme — + // otherwise this passes even if contentFetch started honouring the allowlist. await expect( - secureFetchWithValidation(`http://${host}:${port}/`, { profile: 'contentFetch' }) - ).rejects.toThrow() + secureFetchWithValidation(`https://${host}:${port}/`, { profile: 'contentFetch' }) + ).rejects.toThrow(/private or reserved address/) }) it('reaches a loopback service without any allowlist, as a self-hosted deployment does', async () => { diff --git a/apps/sim/lib/core/security/input-validation.test.ts b/apps/sim/lib/core/security/input-validation.test.ts index 8b2bcef3797..97dffe3a421 100644 --- a/apps/sim/lib/core/security/input-validation.test.ts +++ b/apps/sim/lib/core/security/input-validation.test.ts @@ -17,6 +17,7 @@ import { validatePathSegment, validateS3BucketName, validateServiceNowInstanceUrl, + validateSupabaseProjectId, validateWorkdayTenantUrl, } from '@/lib/core/security/input-validation' import { @@ -1931,3 +1932,122 @@ describe('validateWorkdayTenantUrl', () => { }) }) }) + +describe('validateSupabaseProjectId', () => { + describe('valid inputs', () => { + it.concurrent('should accept a typical 20-char lowercase alphanumeric project ID', () => { + const result = validateSupabaseProjectId('jdrkgepadsdopsntdlom') + expect(result.isValid).toBe(true) + expect(result.sanitized).toBe('jdrkgepadsdopsntdlom') + }) + + it.concurrent('should accept project IDs with digits', () => { + const result = validateSupabaseProjectId('abc123def456ghi789jk') + expect(result.isValid).toBe(true) + }) + + it.concurrent('should accept IDs at the minimum length boundary (10)', () => { + const result = validateSupabaseProjectId('abcdefghij') + expect(result.isValid).toBe(true) + }) + + it.concurrent('should accept IDs at the maximum length boundary (40)', () => { + const result = validateSupabaseProjectId('a'.repeat(40)) + expect(result.isValid).toBe(true) + }) + }) + + describe('SSRF attack vectors', () => { + it.concurrent('should reject fragment injection (#)', () => { + const result = validateSupabaseProjectId('evil#attacker.com') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject @ for authority injection', () => { + const result = validateSupabaseProjectId('evil@attacker.com') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject path traversal with slashes', () => { + const result = validateSupabaseProjectId('evil/../../etc/passwd') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject dots (subdomain manipulation)', () => { + const result = validateSupabaseProjectId('evil.attacker.com') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject backslashes', () => { + const result = validateSupabaseProjectId('evil\\path') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject colons (port injection)', () => { + const result = validateSupabaseProjectId('evil:8080') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject URL-encoded characters', () => { + const result = validateSupabaseProjectId('evil%23attacker') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject spaces', () => { + const result = validateSupabaseProjectId('evil host') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject newlines (header injection)', () => { + const result = validateSupabaseProjectId('evil\r\nHost: attacker.com') + expect(result.isValid).toBe(false) + }) + }) + + describe('invalid formats', () => { + it.concurrent('should reject null', () => { + const result = validateSupabaseProjectId(null) + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject undefined', () => { + const result = validateSupabaseProjectId(undefined) + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject empty string', () => { + const result = validateSupabaseProjectId('') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject uppercase letters', () => { + const result = validateSupabaseProjectId('JDRKGEPADSDOPSNTDLOM') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject mixed case', () => { + const result = validateSupabaseProjectId('jdrkGEPadsdOPSntdlom') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject hyphens', () => { + const result = validateSupabaseProjectId('jdrk-gepa-dsdo-psnt') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject underscores', () => { + const result = validateSupabaseProjectId('jdrk_gepa_dsdo_psnt') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject IDs shorter than 10 characters', () => { + const result = validateSupabaseProjectId('abcdefghi') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject IDs longer than 40 characters', () => { + const result = validateSupabaseProjectId('a'.repeat(41)) + expect(result.isValid).toBe(false) + }) + }) +}) diff --git a/apps/sim/lib/core/security/input-validation.ts b/apps/sim/lib/core/security/input-validation.ts index 3602d0d233a..f3da13ddf12 100644 --- a/apps/sim/lib/core/security/input-validation.ts +++ b/apps/sim/lib/core/security/input-validation.ts @@ -1123,6 +1123,13 @@ function validateVendorHostedUrl( const candidate = assumeHttps && !/^https?:\/\//i.test(raw) ? `https://${raw}` : raw + // These vendors are public SaaS reached over TLS. Enforced here rather than + // left to the egress policy, which would permit plain HTTP to a host an + // operator happened to put in their allowlist. + if (/^http:\/\//i.test(candidate)) { + return { isValid: false, error: `${paramName} must use https://` } + } + const urlResult = validateExternalUrl(candidate, paramName, 'configuredEndpoint') if (!urlResult.isValid) return urlResult diff --git a/apps/sim/lib/internal/stt/operations.ts b/apps/sim/lib/internal/stt/operations.ts index 2c5d6908700..9ec4b22e0d4 100644 --- a/apps/sim/lib/internal/stt/operations.ts +++ b/apps/sim/lib/internal/stt/operations.ts @@ -3,6 +3,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { extractAudioFromVideo, isVideoFile } from '@/lib/audio/extractor' import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' +import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -255,13 +256,18 @@ export async function executeSttOperation( } } - const urlValidation = await validateUrlWithDNS(audioUrl, 'audioUrl', 'contentFetch') + // A caller-supplied audio URL is content; a resolved internal one is a + // presigned URL against Sim's own storage, which on a self-hosted + // deployment legitimately sits on a private address. + const audioProfile: EgressProfile = internalAudioUrl ? 'configuredEndpoint' : 'contentFetch' + + const urlValidation = await validateUrlWithDNS(audioUrl, 'audioUrl', audioProfile) if (!urlValidation.isValid) { return Response.json({ error: urlValidation.error }, { status: 400 }) } const response = await secureFetchWithPinnedIP(audioUrl, urlValidation.resolvedIP, { - profile: 'contentFetch', + profile: audioProfile, method: 'GET', maxResponseBytes: MAX_FILE_SIZE, signal, diff --git a/apps/sim/lib/mcp/domain-check.ts b/apps/sim/lib/mcp/domain-check.ts index fb3d292ff6b..cb10a27610f 100644 --- a/apps/sim/lib/mcp/domain-check.ts +++ b/apps/sim/lib/mcp/domain-check.ts @@ -13,6 +13,17 @@ const logger = createLogger('McpDomainCheck') */ export const MCP_EGRESS_PROFILE: EgressProfile = 'selfHostedService' +/** + * Profile for an MCP OAuth leg — discovery, registration, token exchange, + * revocation. + * + * Every hop after the first takes its URL from authorization-server metadata, + * which the remote server controls. Treating those as configured endpoints would + * let a hostile server steer a leg at whatever the operator allowlisted for their + * own workflows, so they get the provenance they actually have. + */ +export const OAUTH_EGRESS_PROFILE: EgressProfile = 'contentFetch' + export class McpDomainNotAllowedError extends Error { constructor(domain: string) { super(`MCP server domain "${domain}" is not allowed by the server's ALLOWED_MCP_DOMAINS policy`) @@ -113,6 +124,10 @@ export function validateMcpDomain(url: string | undefined): void { * list disabled this entirely — which left an allowlisted domain free to redirect * anywhere, cloud metadata included. * + * `profile` defaults to the configured-server one. An OAuth leg passes + * `contentFetch` instead, because those URLs come out of authorization-server + * metadata rather than from whoever configured the server. + * * Returns null only when the hostname still contains an unresolved env-var * reference. That URL is checked again after resolution, at which point it takes * the normal path. @@ -120,11 +135,14 @@ export function validateMcpDomain(url: string | undefined): void { * @throws McpSsrfError when the policy refuses the destination * @throws McpDnsResolutionError when the hostname cannot be resolved */ -export async function validateMcpServerSsrf(url: string | undefined): Promise { +export async function validateMcpServerSsrf( + url: string | undefined, + profile: EgressProfile = MCP_EGRESS_PROFILE +): Promise { if (!url) return null if (hasEnvVarInHostname(url)) return null - const validation = await validateUrlWithDNS(url, 'MCP server URL', MCP_EGRESS_PROFILE) + const validation = await validateUrlWithDNS(url, 'MCP server URL', profile) if (validation.isValid) return validation.resolvedIP const error = validation.error ?? 'MCP server URL is not reachable' diff --git a/apps/sim/lib/mcp/oauth/revoke.test.ts b/apps/sim/lib/mcp/oauth/revoke.test.ts index 1f54008343a..abae32cc3b7 100644 --- a/apps/sim/lib/mcp/oauth/revoke.test.ts +++ b/apps/sim/lib/mcp/oauth/revoke.test.ts @@ -45,6 +45,8 @@ vi.mock('@sim/security/ssrf', () => ({ })) vi.mock('@/lib/mcp/domain-check', () => ({ MCP_EGRESS_PROFILE: 'selfHostedService', + OAUTH_EGRESS_PROFILE: 'contentFetch', + McpSsrfError: class McpSsrfError extends Error {}, validateMcpServerSsrf: mockValidateMcpServerSsrf, })) vi.mock('@modelcontextprotocol/sdk/client/auth.js', () => ({ @@ -115,7 +117,7 @@ describe('revokeMcpOauthTokens — SSRF guard', () => { it('validates the attacker-controlled revocation_endpoint before issuing the request', async () => { await revokeMcpOauthTokens('server-1', 'workspace-1') - expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith(BLOCKED_ENDPOINT) + expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith(BLOCKED_ENDPOINT, 'contentFetch') }) it('never issues an outbound request to the blocked revocation endpoint', async () => { @@ -146,7 +148,7 @@ describe('revokeMcpOauthTokens — SSRF guard', () => { await revokeMcpOauthTokens('server-1', 'workspace-1') - expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith(publicEndpoint) + expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith(publicEndpoint, 'contentFetch') const revokeCalls = mockUndiciFetch.mock.calls.filter((call) => { const target = typeof call[0] === 'string' ? call[0] : String(call[0]) return target === publicEndpoint diff --git a/apps/sim/lib/mcp/orchestration/server-lifecycle.test.ts b/apps/sim/lib/mcp/orchestration/server-lifecycle.test.ts index 9d4f347010f..489d9cf4d05 100644 --- a/apps/sim/lib/mcp/orchestration/server-lifecycle.test.ts +++ b/apps/sim/lib/mcp/orchestration/server-lifecycle.test.ts @@ -39,6 +39,7 @@ vi.mock('@sim/utils/id', () => ({ generateId: vi.fn() })) vi.mock('@/lib/core/security/encryption', () => encryptionMock) vi.mock('@/lib/mcp/domain-check', () => ({ MCP_EGRESS_PROFILE: 'selfHostedService', + OAUTH_EGRESS_PROFILE: 'contentFetch', McpDnsResolutionError: class extends Error {}, McpDomainNotAllowedError: class extends Error {}, McpSsrfError: class extends Error {}, diff --git a/apps/sim/lib/mcp/pinned-fetch.test.ts b/apps/sim/lib/mcp/pinned-fetch.test.ts index f059750f648..2bd532389b9 100644 --- a/apps/sim/lib/mcp/pinned-fetch.test.ts +++ b/apps/sim/lib/mcp/pinned-fetch.test.ts @@ -32,9 +32,12 @@ vi.mock('@sim/security/ssrf', () => ({ })) vi.mock('@/lib/mcp/domain-check', () => ({ MCP_EGRESS_PROFILE: 'selfHostedService', + OAUTH_EGRESS_PROFILE: 'contentFetch', + McpSsrfError: class McpSsrfError extends Error {}, validateMcpServerSsrf: mockValidateMcpServerSsrf, })) +import { McpSsrfError } from '@/lib/mcp/domain-check' import { createGuardedMcpFetch, createSsrfGuardedMcpFetch } from '@/lib/mcp/pinned-fetch' /** The per-request guarded Agent is always built with a DoS-backstop response cap. */ @@ -114,7 +117,10 @@ describe('createSsrfGuardedMcpFetch', () => { const fetchLike = createSsrfGuardedMcpFetch() await fetchLike('https://attacker.example/revoke', { method: 'POST' }) - expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith('https://attacker.example/revoke') + expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith( + 'https://attacker.example/revoke', + 'contentFetch' + ) // The guarded Agent is always built with the DoS-backstop response-size cap. expect(mockCreateGuardedFetchWithDispatcher).toHaveBeenCalledWith(withResponseCap) expect(sentinelFetch).toHaveBeenCalledWith( @@ -183,27 +189,6 @@ describe('createSsrfGuardedMcpFetch', () => { expect(mockDestroy).toHaveBeenCalledTimes(1) }) - it('returns a streaming response live (un-buffered) over the unpinned fallback', async () => { - // resolvedIP null → global fetch; a text/event-stream reply (the auth-type probe) - // must be handed back as-is so the caller reads headers without draining the stream. - // Identity (same object) proves it was NOT re-wrapped into a buffered copy. - mockValidateMcpServerSsrf.mockResolvedValue(null) - const streamingRes = new Response(new ReadableStream({ start() {} }), { - headers: { 'content-type': 'text/event-stream' }, - }) - const globalFetch = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => streamingRes) - try { - const fetchLike = createSsrfGuardedMcpFetch() - const res = await fetchLike('https://allowed.internal/mcp', { method: 'POST' }) - - expect(res).toBe(streamingRes) - // No per-request Agent on the unpinned path, so nothing to tear down. - expect(mockDestroy).not.toHaveBeenCalled() - } finally { - globalFetch.mockRestore() - } - }) - it('streams (does not buffer) a pinned text/event-stream reply and tears down after it drains', async () => { // The guard resolves the IP itself, so the probe's initialize over the guarded path // DOES get a pinned Agent. A streaming reply must still be handed back live (not @@ -370,23 +355,27 @@ describe('createSsrfGuardedMcpFetch', () => { const fetchLike = createSsrfGuardedMcpFetch() await fetchLike(new URL('https://attacker.example/discover')) - expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith('https://attacker.example/discover') + expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith( + 'https://attacker.example/discover', + 'contentFetch' + ) expect(mockCreateGuardedFetchWithDispatcher).toHaveBeenCalledWith(withResponseCap) }) - it('falls back to global fetch when validation returns no IP', async () => { + it('refuses rather than falling back to an unguarded fetch when validation yields no IP', async () => { mockValidateMcpServerSsrf.mockResolvedValue(null) const globalFetch = vi .spyOn(globalThis, 'fetch') .mockImplementation(async () => new Response('ok')) try { const fetchLike = createSsrfGuardedMcpFetch() - await fetchLike('https://allowed.internal/mcp') + await expect(fetchLike('https://allowed.internal/mcp')).rejects.toThrow( + 'could not be validated' + ) + // No leg of the OAuth flow may reach the network unguarded. + expect(globalFetch).not.toHaveBeenCalled() expect(mockCreateGuardedFetchWithDispatcher).not.toHaveBeenCalled() - expect(globalFetch).toHaveBeenCalledTimes(1) - // No pinned Agent was created, so there is nothing to tear down. - expect(mockDestroy).not.toHaveBeenCalled() } finally { globalFetch.mockRestore() } @@ -394,22 +383,15 @@ describe('createSsrfGuardedMcpFetch', () => { }) describe('self-hosted private-resolution carve-out', () => { - it('keeps the legacy pin for a loopback-resolving host (guarded lookup would filter it)', async () => { - // Self-hosted DNS alias -> 127.0.0.1: policy allows it. The guarded lookup would - // strand the connect and an unguarded fallback would reopen rebinding — so this case - // pins to the validated address, preserving the old behavior and its security property. - mockValidateMcpServerSsrf.mockResolvedValue('127.0.0.1') - mockCreatePinnedFetchWithDispatcher.mockReturnValue({ - fetch: sentinelFetch, - dispatcher: { destroy: mockDestroy }, - }) - sentinelFetch.mockImplementation(async () => new Response('ok')) + it('refuses a loopback resolution instead of pinning to it', async () => { + // OAuth legs run under `contentFetch`, which vouches for nothing — so a + // hostile authorization server cannot steer a leg at the deployment's own + // loopback, which the previous pinned carve-out would have permitted. + mockValidateMcpServerSsrf.mockRejectedValue(new McpSsrfError('blocked')) const fetchLike = createSsrfGuardedMcpFetch() - await fetchLike('https://my-local-alias/mcp') - expect(mockCreatePinnedFetchWithDispatcher).toHaveBeenCalledWith( - '127.0.0.1', - expect.objectContaining({ maxResponseSize: expect.any(Number) }) - ) + + await expect(fetchLike('https://my-local-alias/mcp')).rejects.toThrow(McpSsrfError) + expect(mockCreatePinnedFetchWithDispatcher).not.toHaveBeenCalled() expect(mockCreateGuardedFetchWithDispatcher).not.toHaveBeenCalled() }) }) diff --git a/apps/sim/lib/mcp/pinned-fetch.ts b/apps/sim/lib/mcp/pinned-fetch.ts index 3d7a638f3be..b055cf6a812 100644 --- a/apps/sim/lib/mcp/pinned-fetch.ts +++ b/apps/sim/lib/mcp/pinned-fetch.ts @@ -1,12 +1,16 @@ import type { FetchLike } from '@modelcontextprotocol/sdk/shared/transport.js' import { createLogger } from '@sim/logger' -import { isPrivateIp } from '@sim/security/ssrf' import type { Agent } from 'undici' import { createPinnedFetchWithDispatcher, createSsrfGuardedFetchWithDispatcher, } from '@/lib/core/security/input-validation.server' -import { MCP_EGRESS_PROFILE, validateMcpServerSsrf } from '@/lib/mcp/domain-check' +import { + MCP_EGRESS_PROFILE, + McpSsrfError, + OAUTH_EGRESS_PROFILE, + validateMcpServerSsrf, +} from '@/lib/mcp/domain-check' import { McpError } from '@/lib/mcp/types' const logger = createLogger('McpOauthFetch') @@ -293,30 +297,25 @@ export function createSsrfGuardedMcpFetch(timeoutMs: number = OAUTH_FETCH_TIMEOU let dispatcher: Agent | undefined try { logger.info('OAuth guarded fetch: validating', { host }) - const resolvedIP = await withDeadline(validateMcpServerSsrf(target), signal) - logger.info('OAuth guarded fetch: requesting', { host, guarded: Boolean(resolvedIP) }) - let response: Response - if (resolvedIP && isPrivateIp(resolvedIP)) { - // Self-hosted private/loopback resolution (policy-permitted): the guarded lookup - // would filter the address, and an unguarded fallback would reopen rebinding — - // keep the legacy pin to the validated address for exactly this case. - const pinned = createPinnedFetchWithDispatcher(resolvedIP, { - profile: MCP_EGRESS_PROFILE, - maxResponseSize: MAX_OAUTH_RESPONSE_BYTES, - }) - dispatcher = pinned.dispatcher - response = await withDeadline(pinned.fetch(url, { ...init, signal }), signal) - } else if (resolvedIP) { - const guarded = createSsrfGuardedFetchWithDispatcher({ - profile: MCP_EGRESS_PROFILE, - maxResponseSize: MAX_OAUTH_RESPONSE_BYTES, - }) - dispatcher = guarded.dispatcher - response = await withDeadline(guarded.fetch(url, { ...init, signal }), signal) - } else { - // No guard (self-hosted allowlist / localhost carve-out) — global fetch as before. - response = await withDeadline(globalThis.fetch(url, { ...init, signal }), signal) + const resolvedIP = await withDeadline( + validateMcpServerSsrf(target, OAUTH_EGRESS_PROFILE), + signal + ) + if (!resolvedIP) { + // No leg of this flow may run unguarded. Under `contentFetch` the only + // way here is an unresolved env-var reference in the hostname, which is + // never a real authorization-server URL by the time OAuth runs. + throw new McpSsrfError('MCP OAuth request URL could not be validated') } + logger.info('OAuth guarded fetch: requesting', { host }) + // Always the guarded connector: `contentFetch` cannot yield a private + // address, so there is no pinned-private case to carve out here. + const guarded = createSsrfGuardedFetchWithDispatcher({ + profile: OAUTH_EGRESS_PROFILE, + maxResponseSize: MAX_OAUTH_RESPONSE_BYTES, + }) + dispatcher = guarded.dispatcher + const response = await withDeadline(guarded.fetch(url, { ...init, signal }), signal) // The probe's `initialize` can stream (text/event-stream); hand it back live so the // buffer doesn't drain/stall it. Every OAuth leg is single-shot JSON and is buffered. const contentType = response.headers.get('content-type') ?? '' diff --git a/apps/sim/lib/mcp/service-pool.test.ts b/apps/sim/lib/mcp/service-pool.test.ts index 2ea39812211..3f1566b7ab9 100644 --- a/apps/sim/lib/mcp/service-pool.test.ts +++ b/apps/sim/lib/mcp/service-pool.test.ts @@ -102,6 +102,8 @@ const SERVER_ROW = { vi.mock('@/lib/mcp/domain-check', () => ({ MCP_EGRESS_PROFILE: 'selfHostedService', + OAUTH_EGRESS_PROFILE: 'contentFetch', + McpSsrfError: class McpSsrfError extends Error {}, isMcpDomainAllowed: () => true, validateMcpDomain: () => {}, validateMcpServerSsrf: async () => '203.0.113.10', diff --git a/apps/sim/lib/mcp/service.test.ts b/apps/sim/lib/mcp/service.test.ts index 3abab84350a..8b297d5272d 100644 --- a/apps/sim/lib/mcp/service.test.ts +++ b/apps/sim/lib/mcp/service.test.ts @@ -100,6 +100,8 @@ vi.mock('@/lib/mcp/connection-manager', () => ({ vi.mock('@/lib/mcp/domain-check', () => ({ MCP_EGRESS_PROFILE: 'selfHostedService', + OAUTH_EGRESS_PROFILE: 'contentFetch', + McpSsrfError: class McpSsrfError extends Error {}, isMcpDomainAllowed: (...args: unknown[]) => mockIsDomainAllowed(...args), validateMcpDomain: (...args: unknown[]) => mockValidateDomain(...args), validateMcpServerSsrf: (...args: unknown[]) => mockValidateSsrf(...args), diff --git a/package.json b/package.json index 297e32f4656..ca7e199be1b 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "format:check": "turbo run format:check", "lint": "turbo run lint", "lint:check": "turbo run lint:check", - "lint:helm": "helm lint helm/sim --values helm/sim/ci/default-values.yaml", + "lint:helm": "helm lint helm/sim --strict --values helm/sim/ci/default-values.yaml", "lint:all": "turbo run lint && bun run lint:helm", "check": "turbo run format:check", "check:egress-boundary": "bun run scripts/check-egress-boundary.ts", diff --git a/packages/testing/src/mocks/env-flags.mock.ts b/packages/testing/src/mocks/env-flags.mock.ts index 24868e1af2b..471798c780c 100644 --- a/packages/testing/src/mocks/env-flags.mock.ts +++ b/packages/testing/src/mocks/env-flags.mock.ts @@ -125,13 +125,18 @@ export const envFlagsMockFns = { * Egress config is exposed as functions by the real module, but held as * mutable state here so a test can still write * `envFlagsMock.egressAllowedHosts = '...'` and have the read observe it. + * + * The hosted gate is mirrored from production: a deployment on sim.ai ignores + * these entirely, so a test that sets both must see the same thing. */ - getEgressAllowedHosts: vi.fn<() => string | undefined>(() => envFlagsState.egressAllowedHosts), - getEgressAllowedIpRanges: vi.fn<() => string | undefined>( - () => envFlagsState.egressAllowedIpRanges + getEgressAllowedHosts: vi.fn<() => string | undefined>(() => + envFlagsState.isHosted ? undefined : envFlagsState.egressAllowedHosts + ), + getEgressAllowedIpRanges: vi.fn<() => string | undefined>(() => + envFlagsState.isHosted ? undefined : envFlagsState.egressAllowedIpRanges ), isLegacyPrivateDatabaseAccessAllowed: vi.fn<() => boolean>( - () => envFlagsState.legacyPrivateDatabaseAccess + () => !envFlagsState.isHosted && envFlagsState.legacyPrivateDatabaseAccess ), getAllowedIntegrationsFromEnv: vi.fn<() => string[] | null>(() => null), getPreviewBlocksFromEnv: vi.fn<() => string[]>(() => []), @@ -169,13 +174,17 @@ export function resetEnvFlagsMock(): void { envFlagsMockFns.getCostMultiplier.mockReset().mockImplementation(() => 1) envFlagsMockFns.getEgressAllowedHosts .mockReset() - .mockImplementation(() => envFlagsState.egressAllowedHosts) + .mockImplementation(() => + envFlagsState.isHosted ? undefined : envFlagsState.egressAllowedHosts + ) envFlagsMockFns.getEgressAllowedIpRanges .mockReset() - .mockImplementation(() => envFlagsState.egressAllowedIpRanges) + .mockImplementation(() => + envFlagsState.isHosted ? undefined : envFlagsState.egressAllowedIpRanges + ) envFlagsMockFns.isLegacyPrivateDatabaseAccessAllowed .mockReset() - .mockImplementation(() => envFlagsState.legacyPrivateDatabaseAccess) + .mockImplementation(() => !envFlagsState.isHosted && envFlagsState.legacyPrivateDatabaseAccess) } /** diff --git a/scripts/check-egress-boundary.ts b/scripts/check-egress-boundary.ts index 93de4c99a47..a7caf7f70f8 100644 --- a/scripts/check-egress-boundary.ts +++ b/scripts/check-egress-boundary.ts @@ -14,7 +14,7 @@ * * Not checked: bare `fetch()`. It is used constantly for same-origin and * server-action calls where the guard does not apply, so flagging it would be - * noise. The transports it can reach are covered by the import rule above. + * noise. The transports it can reach are covered by the rules above. * * Usage: bun run scripts/check-egress-boundary.ts */ @@ -35,14 +35,32 @@ const SCAN_DIRS = [ const SKIP_DIRS = new Set(['node_modules', 'dist', '.next', '.turbo', 'coverage']) +/** Modules that can open a socket directly. */ +const TRANSPORTS = ['http', 'https', 'undici', 'http-proxy-agent', 'https-proxy-agent'] + +const MODULE_ALTERNATION = TRANSPORTS.map((name) => name.replace(/[-]/g, '\\-')).join('|') +const SPECIFIER = `['"](?:node:)?(?:${MODULE_ALTERNATION})['"]` + /** - * Raw HTTP transports. Reaching one directly bypasses DNS pinning. + * Every way a module reaches one of these at runtime. * * Matched against the whole source rather than line by line, because an import - * list broken across lines would otherwise slip past. + * list broken across lines would otherwise slip past. `import type` is excluded: + * a type has no runtime presence and cannot open anything. */ -const TRANSPORT_IMPORT = - /^[ \t]*import\b[\s\S]*?from\s*['"](?:node:)?(?:http|https|undici|http-proxy-agent|https-proxy-agent)['"]/gm +const RUNTIME_LOADS: ReadonlyArray<{ pattern: RegExp; kind: string }> = [ + { + pattern: new RegExp(`^[ \t]*import\\s+(?!type\\s)[\\s\\S]*?from\\s*${SPECIFIER}`, 'gm'), + kind: 'import', + }, + { pattern: new RegExp(`^[ \t]*import\\s*${SPECIFIER}`, 'gm'), kind: 'side-effect import' }, + { + pattern: new RegExp(`^[ \t]*export\\s+(?!type\\s)[\\s\\S]*?from\\s*${SPECIFIER}`, 'gm'), + kind: 're-export', + }, + { pattern: new RegExp(`\\bimport\\s*\\(\\s*${SPECIFIER}\\s*\\)`, 'g'), kind: 'dynamic import' }, + { pattern: new RegExp(`\\brequire\\s*\\(\\s*${SPECIFIER}\\s*\\)`, 'g'), kind: 'require' }, +] /** * Modules allowed to hold a transport import, each because it *is* part of the @@ -70,6 +88,7 @@ function walk(dir: string, out: string[] = []): string[] { interface Violation { file: string line: number + kind: string snippet: string } @@ -84,14 +103,16 @@ function main() { if (ALLOWED.has(rel)) continue scanned++ const source = readFileSync(file, 'utf8') - TRANSPORT_IMPORT.lastIndex = 0 - for (const match of source.matchAll(TRANSPORT_IMPORT)) { - const line = source.slice(0, match.index).split('\n').length - violations.push({ - file: rel, - line, - snippet: match[0].replace(/\s+/g, ' ').trim(), - }) + for (const { pattern, kind } of RUNTIME_LOADS) { + pattern.lastIndex = 0 + for (const match of source.matchAll(pattern)) { + violations.push({ + file: rel, + line: source.slice(0, match.index).split('\n').length, + kind, + snippet: match[0].replace(/\s+/g, ' ').trim(), + }) + } } } } @@ -103,7 +124,7 @@ function main() { console.error('✗ check-egress-boundary: raw HTTP transport outside the egress guard\n') for (const violation of violations) { - console.error(` ${violation.file}:${violation.line}`) + console.error(` ${violation.file}:${violation.line} (${violation.kind})`) console.error(` ${violation.snippet}`) } console.error( From fab48b2a080d8027fe5c3862048d21f8224928c8 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 20:49:01 -0700 Subject: [PATCH 13/20] fix(egress): address the third review round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All six verified before changing anything; two were bugs I introduced in the previous round, and one of my own fixes for them was wrong on the first attempt. `evaluateAddress` failed **open** on an unparseable address: a hostname in the allowlist vouched for it before anything checked it could be parsed. Probed and confirmed `allowed: true` for garbage. The address is validated before any vouching rule now, which is what the module claims to do. `assertGuardedRedirectTarget` returned early for any hostname target, so a redirect could downgrade to plain HTTP or land on a denied port as long as it was named rather than numbered. Hostname hops get the pre-DNS half of the policy; their address is still judged at connect time. Making OAuth legs `contentFetch` last round broke discovery against a self-hosted MCP server, which starts at the configured URL. The OAuth fetch now takes that URL: a leg on the same origin keeps `MCP_EGRESS_PROFILE`, anything the metadata names gets `contentFetch`. The revoke suite covers both — a same-origin revocation endpoint keeps the configured profile, the metadata IP does not. The synchronous `validateExternalUrl` refused destinations only an IP range could permit, so a drain at `http://10.0.0.5` could not be configured even where the allowlist covered it. It defers those to the resolving check — but my first attempt deferred on `policyCanVouch`, which is true whenever loopback is permitted and turned the whole check into a no-op (caught by 30 tests). Added `policyDefersToAddress`, which covers only what an address can actually change. The boundary script matched inside comments and strings, so a docstring warning against `require('undici')` failed CI. My first fix blanked string literals and silently disabled every rule — the module specifier is itself a string — which a probe caught. It blanks comments and rejects matches starting inside a string instead; all eight cases re-verified by probe. The pinned-fetch allowlist test redirected at the metadata endpoint, which is refused unconditionally, so it would have passed even if the allowlist regressed to permitting every private address. It uses an address genuinely outside the range, with the metadata case kept as its own test. --- apps/sim/lib/core/security/egress/validate.ts | 9 +++ .../core/security/input-validation.server.ts | 20 ++++-- .../sim/lib/core/security/input-validation.ts | 22 +++++-- .../core/security/pinned-fetch.server.test.ts | 19 +++++- apps/sim/lib/mcp/oauth/auth.ts | 2 +- apps/sim/lib/mcp/oauth/probe.ts | 2 +- apps/sim/lib/mcp/oauth/revoke.test.ts | 5 +- apps/sim/lib/mcp/oauth/revoke.ts | 2 +- apps/sim/lib/mcp/pinned-fetch.test.ts | 12 ++-- apps/sim/lib/mcp/pinned-fetch.ts | 47 ++++++++++---- packages/security/src/egress.ts | 19 ++++++ scripts/check-egress-boundary.ts | 65 ++++++++++++++++++- 12 files changed, 185 insertions(+), 39 deletions(-) diff --git a/apps/sim/lib/core/security/egress/validate.ts b/apps/sim/lib/core/security/egress/validate.ts index 059b420ad78..f48201c8653 100644 --- a/apps/sim/lib/core/security/egress/validate.ts +++ b/apps/sim/lib/core/security/egress/validate.ts @@ -152,3 +152,12 @@ export function checkResolvedEgress( ): EgressDecision { return evaluateAddress(url, address, resolveEgressPolicy(profile)) } + +/** + * The pre-DNS half of the same check, for a destination whose address is not + * known yet — a redirect target named by hostname, where the scheme and port + * still have to answer to the request's own policy. + */ +export function checkEgressUrl(url: URL, profile: EgressProfile): EgressDecision { + return evaluateUrl(url, resolveEgressPolicy(profile)) +} diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index 933a4a39f01..743513ef29e 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -17,7 +17,11 @@ import { request as undiciRequest, } from 'undici' import { describeEgressDenial, type EgressProfile } from '@/lib/core/security/egress/profiles' -import { checkResolvedEgress, validateEgressUrl } from '@/lib/core/security/egress/validate' +import { + checkEgressUrl, + checkResolvedEgress, + validateEgressUrl, +} from '@/lib/core/security/egress/validate' import type { HttpRedirectPolicy } from '@/lib/core/security/http-redirect-policy' import type { ValidationResult } from '@/lib/core/security/input-validation' import { nodeReadableToWebStream } from '@/lib/core/utils/node-stream' @@ -483,16 +487,20 @@ const MAX_GUARDED_REDIRECTS = 5 * targets are covered by {@link createSsrfGuardedLookup} at connect time. */ function assertGuardedRedirectTarget(url: URL, profile: EgressProfile): void { - if (url.protocol !== 'http:' && url.protocol !== 'https:') { - throw new Error(`Blocked by SSRF policy: redirect to unsupported protocol ${url.protocol}`) - } const host = unwrapIpv6Brackets(url.hostname) - if (!isIpLiteral(host)) return // The request's own policy decides, which is how a self-hosted server on a // permitted private address stays reachable across a hop. It replaced a // carve-out that permitted one pinned IP and could not express anything else. - const decision = checkResolvedEgress(url, host, profile) + // + // A literal is judged completely here. A hostname gets the pre-DNS half — + // scheme and port — which used to be skipped entirely, so a hop could downgrade + // to plain HTTP or land on a denied port as long as it was named rather than + // numbered. Its address is judged by the connect-time lookup. + const decision = isIpLiteral(host) + ? checkResolvedEgress(url, host, profile) + : checkEgressUrl(url, profile) + if (!decision.allowed) { throw new Error( `Blocked by SSRF policy: ${describeEgressDenial(decision, 'redirect', profile)}` diff --git a/apps/sim/lib/core/security/input-validation.ts b/apps/sim/lib/core/security/input-validation.ts index f3da13ddf12..0d185fee5d7 100644 --- a/apps/sim/lib/core/security/input-validation.ts +++ b/apps/sim/lib/core/security/input-validation.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { evaluateUrl } from '@sim/security/egress' +import { evaluateUrl, isLiftableByVouching, policyDefersToAddress } from '@sim/security/egress' import { describeEgressDenial, type EgressProfile, @@ -481,6 +481,9 @@ export function validateJiraIssueKey( * only a resolved address can be classified. Use this for form/contract * validation; use the DNS-resolving variant before connecting. * + * It also declines to refuse anything the resolved address could permit, so a + * destination allowlisted by IP range is still configurable. + * * @param url - The URL to validate * @param paramName - Name of the parameter for error messages * @param profile - Where this URL came from; see {@link EgressProfile} @@ -510,10 +513,19 @@ export function validateExternalUrl( return { isValid: false, error: `${paramName} must be a valid URL` } } - const decision = evaluateUrl(parsed, resolveEgressPolicy(profile)) - return decision.allowed - ? { isValid: true } - : { isValid: false, error: describeEgressDenial(decision, paramName, profile) } + const policy = resolveEgressPolicy(profile) + const decision = evaluateUrl(parsed, policy) + if (decision.allowed) return { isValid: true } + + // A refusal the resolved address could lift is not this check's to make: a + // host permitted only by EGRESS_ALLOWED_IP_RANGES cannot be recognised until + // DNS runs, and refusing here would stop it being configured at all. + // validateUrlWithDNS makes the authoritative call before anything is dialled. + if (policyDefersToAddress(policy) && isLiftableByVouching(decision.reason)) { + return { isValid: true } + } + + return { isValid: false, error: describeEgressDenial(decision, paramName, profile) } } /** diff --git a/apps/sim/lib/core/security/pinned-fetch.server.test.ts b/apps/sim/lib/core/security/pinned-fetch.server.test.ts index 6953f3f6bee..ded5a698a08 100644 --- a/apps/sim/lib/core/security/pinned-fetch.server.test.ts +++ b/apps/sim/lib/core/security/pinned-fetch.server.test.ts @@ -198,6 +198,23 @@ describe('createPinnedFetch', () => { it('still blocks a redirect to a private IP outside the allowlist', async () => { setEnvFlags({ egressAllowedIpRanges: '10.0.0.0/8' }) + // A genuine private address outside the allowlisted range, not the metadata + // endpoint — that one is refused unconditionally, so it would pass here even + // if the allowlist had regressed to permitting all private addresses. + mockUndiciRequest.mockResolvedValueOnce( + undiciReply(302, { location: 'https://192.168.1.5/internal' }, byteStream('')) + ) + const pinned = createPinnedFetch('10.0.0.5', { profile: 'configuredEndpoint' }) + + await expect(pinned('http://10.0.0.5:3000/mcp', { method: 'GET' })).rejects.toThrow( + /private or reserved address/ + ) + // The initial request happened; the redirect out of the range was refused. + expect(mockUndiciRequest).toHaveBeenCalledTimes(1) + }) + + it('still blocks a redirect to the metadata endpoint from inside the allowlist', async () => { + setEnvFlags({ egressAllowedIpRanges: '10.0.0.0/8,169.254.0.0/16' }) mockUndiciRequest.mockResolvedValueOnce( undiciReply(302, { location: 'http://169.254.169.254/latest/meta-data/' }, byteStream('')) ) @@ -206,8 +223,6 @@ describe('createPinnedFetch', () => { await expect(pinned('http://10.0.0.5:3000/mcp', { method: 'GET' })).rejects.toThrow( /cloud metadata endpoint/ ) - // The initial request happened; the redirect to the metadata IP was refused. - expect(mockUndiciRequest).toHaveBeenCalledTimes(1) }) it('reuses one dispatcher across all calls of a single instance', async () => { diff --git a/apps/sim/lib/mcp/oauth/auth.ts b/apps/sim/lib/mcp/oauth/auth.ts index 2787486e546..092f90c5f3d 100644 --- a/apps/sim/lib/mcp/oauth/auth.ts +++ b/apps/sim/lib/mcp/oauth/auth.ts @@ -17,6 +17,6 @@ export function mcpAuthGuarded( ): ReturnType { return auth(provider, { ...options, - fetchFn: options.fetchFn ?? createSsrfGuardedMcpFetch(), + fetchFn: options.fetchFn ?? createSsrfGuardedMcpFetch({ serverUrl: String(options.serverUrl) }), }) } diff --git a/apps/sim/lib/mcp/oauth/probe.ts b/apps/sim/lib/mcp/oauth/probe.ts index da17123cc4f..c39f521fe01 100644 --- a/apps/sim/lib/mcp/oauth/probe.ts +++ b/apps/sim/lib/mcp/oauth/probe.ts @@ -39,7 +39,7 @@ export async function detectMcpAuthType( const pinned = resolvedIP ? createPinnedFetchWithDispatcher(resolvedIP, { profile: MCP_EGRESS_PROFILE }) : undefined - const probeFetch: FetchLike = pinned?.fetch ?? createSsrfGuardedMcpFetch() + const probeFetch: FetchLike = pinned?.fetch ?? createSsrfGuardedMcpFetch({ serverUrl: url }) const controller = new AbortController() const timer = setTimeout(() => controller.abort(), PROBE_TIMEOUT_MS) diff --git a/apps/sim/lib/mcp/oauth/revoke.test.ts b/apps/sim/lib/mcp/oauth/revoke.test.ts index abae32cc3b7..abfcea1b7b8 100644 --- a/apps/sim/lib/mcp/oauth/revoke.test.ts +++ b/apps/sim/lib/mcp/oauth/revoke.test.ts @@ -148,7 +148,10 @@ describe('revokeMcpOauthTokens — SSRF guard', () => { await revokeMcpOauthTokens('server-1', 'workspace-1') - expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith(publicEndpoint, 'contentFetch') + // Same origin as the configured server, so it keeps that server's profile — + // the metadata pointed back at the host the operator already chose. The + // blocked-endpoint test above covers the cross-origin case, which does not. + expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith(publicEndpoint, 'selfHostedService') const revokeCalls = mockUndiciFetch.mock.calls.filter((call) => { const target = typeof call[0] === 'string' ? call[0] : String(call[0]) return target === publicEndpoint diff --git a/apps/sim/lib/mcp/oauth/revoke.ts b/apps/sim/lib/mcp/oauth/revoke.ts index 89c9760d2c3..fc154978730 100644 --- a/apps/sim/lib/mcp/oauth/revoke.ts +++ b/apps/sim/lib/mcp/oauth/revoke.ts @@ -37,7 +37,7 @@ export async function revokeMcpOauthTokens( const row = await loadOauthRow({ mcpServerId }) if (!row?.tokens) return - const ssrfGuardedFetch = createSsrfGuardedMcpFetch() + const ssrfGuardedFetch = createSsrfGuardedMcpFetch({ serverUrl: server.url }) const info = await discoverOAuthServerInfo(server.url, { fetchFn: ssrfGuardedFetch }).catch( () => undefined ) diff --git a/apps/sim/lib/mcp/pinned-fetch.test.ts b/apps/sim/lib/mcp/pinned-fetch.test.ts index 2bd532389b9..4d80484c0df 100644 --- a/apps/sim/lib/mcp/pinned-fetch.test.ts +++ b/apps/sim/lib/mcp/pinned-fetch.test.ts @@ -254,7 +254,7 @@ describe('createSsrfGuardedMcpFetch', () => { signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) }) ) - const fetchLike = createSsrfGuardedMcpFetch(5) + const fetchLike = createSsrfGuardedMcpFetch({ timeoutMs: 5 }) await expect(fetchLike('https://slow.example/token', { method: 'POST' })).rejects.toThrow( /timed out after 5ms/ @@ -269,7 +269,7 @@ describe('createSsrfGuardedMcpFetch', () => { sentinelFetch.mockImplementation( async () => new Response(new ReadableStream({ start() {} })) ) - const fetchLike = createSsrfGuardedMcpFetch(5) + const fetchLike = createSsrfGuardedMcpFetch({ timeoutMs: 5 }) await expect(fetchLike('https://slow-body.example/token', { method: 'POST' })).rejects.toThrow( /timed out after 5ms/ @@ -280,7 +280,7 @@ describe('createSsrfGuardedMcpFetch', () => { it('bounds a stalled SSRF/DNS validation by the deadline', async () => { // Validation never resolves (mimics a hanging dns.lookup, which takes no signal). mockValidateMcpServerSsrf.mockReturnValue(new Promise(() => {})) - const fetchLike = createSsrfGuardedMcpFetch(5) + const fetchLike = createSsrfGuardedMcpFetch({ timeoutMs: 5 }) await expect(fetchLike('https://slow-dns.example/token')).rejects.toThrow(/timed out after 5ms/) // Never got past validation, so no request was issued and no Agent was created. @@ -295,7 +295,7 @@ describe('createSsrfGuardedMcpFetch', () => { mockValidateMcpServerSsrf.mockRejectedValue(new Error('blocked late')) const controller = new AbortController() controller.abort(new Error('pre-aborted')) - const fetchLike = createSsrfGuardedMcpFetch(60_000) + const fetchLike = createSsrfGuardedMcpFetch({ timeoutMs: 60_000 }) await expect( fetchLike('https://slow.example/token', { signal: controller.signal }) @@ -309,7 +309,7 @@ describe('createSsrfGuardedMcpFetch', () => { // Validation hangs; the caller's abort — well before the 60s deadline — must settle it. mockValidateMcpServerSsrf.mockReturnValue(new Promise(() => {})) const controller = new AbortController() - const fetchLike = createSsrfGuardedMcpFetch(60_000) + const fetchLike = createSsrfGuardedMcpFetch({ timeoutMs: 60_000 }) const pending = fetchLike('https://slow-dns.example/token', { signal: controller.signal }) controller.abort(new Error('caller cancelled')) @@ -332,7 +332,7 @@ describe('createSsrfGuardedMcpFetch', () => { ) const controller = new AbortController() // Long deadline so the caller's abort — not the timeout — is what settles the request. - const fetchLike = createSsrfGuardedMcpFetch(60_000) + const fetchLike = createSsrfGuardedMcpFetch({ timeoutMs: 60_000 }) const pending = fetchLike('https://slow.example/token', { signal: controller.signal }) controller.abort(new Error('caller cancelled')) diff --git a/apps/sim/lib/mcp/pinned-fetch.ts b/apps/sim/lib/mcp/pinned-fetch.ts index b055cf6a812..6f40027a1c2 100644 --- a/apps/sim/lib/mcp/pinned-fetch.ts +++ b/apps/sim/lib/mcp/pinned-fetch.ts @@ -1,5 +1,6 @@ import type { FetchLike } from '@modelcontextprotocol/sdk/shared/transport.js' import { createLogger } from '@sim/logger' +import { isPrivateIp } from '@sim/security/ssrf' import type { Agent } from 'undici' import { createPinnedFetchWithDispatcher, @@ -285,10 +286,26 @@ function releaseStreamOnSettle( * @throws McpSsrfError if a request URL resolves to a blocked IP address * @throws McpError if a request exceeds `timeoutMs` */ -export function createSsrfGuardedMcpFetch(timeoutMs: number = OAUTH_FETCH_TIMEOUT_MS): FetchLike { +export function createSsrfGuardedMcpFetch( + options: { serverUrl?: string; timeoutMs?: number } = {} +): FetchLike { + const { serverUrl, timeoutMs = OAUTH_FETCH_TIMEOUT_MS } = options + // The origin the operator configured. A leg that stays on it is the server + // they chose; everything else was named by that server's metadata. + let configuredOrigin: string | undefined + if (serverUrl && URL.canParse(serverUrl)) configuredOrigin = new URL(serverUrl).origin + return (async (url, init) => { const target = typeof url === 'string' ? url : url.href const host = URL.canParse(target) ? new URL(target).host : target + const sameAsConfigured = + configuredOrigin !== undefined && + URL.canParse(target) && + new URL(target).origin === configuredOrigin + // The first hop is the configured server and keeps its privileges — a + // self-hosted MCP on an allowlisted private address must still be able to + // start discovery. Every hop the metadata names is judged as content. + const profile = sameAsConfigured ? MCP_EGRESS_PROFILE : OAUTH_EGRESS_PROFILE const startedAt = Date.now() const timeoutSignal = AbortSignal.timeout(timeoutMs) // Bound every phase — validation, request, body read — by the deadline + caller signal. @@ -297,25 +314,27 @@ export function createSsrfGuardedMcpFetch(timeoutMs: number = OAUTH_FETCH_TIMEOU let dispatcher: Agent | undefined try { logger.info('OAuth guarded fetch: validating', { host }) - const resolvedIP = await withDeadline( - validateMcpServerSsrf(target, OAUTH_EGRESS_PROFILE), - signal - ) + const resolvedIP = await withDeadline(validateMcpServerSsrf(target, profile), signal) if (!resolvedIP) { // No leg of this flow may run unguarded. Under `contentFetch` the only // way here is an unresolved env-var reference in the hostname, which is // never a real authorization-server URL by the time OAuth runs. throw new McpSsrfError('MCP OAuth request URL could not be validated') } - logger.info('OAuth guarded fetch: requesting', { host }) - // Always the guarded connector: `contentFetch` cannot yield a private - // address, so there is no pinned-private case to carve out here. - const guarded = createSsrfGuardedFetchWithDispatcher({ - profile: OAUTH_EGRESS_PROFILE, - maxResponseSize: MAX_OAUTH_RESPONSE_BYTES, - }) - dispatcher = guarded.dispatcher - const response = await withDeadline(guarded.fetch(url, { ...init, signal }), signal) + logger.info('OAuth guarded fetch: requesting', { host, configured: sameAsConfigured }) + // A private address only survives validation on the configured first hop, + // and the guarded lookup would filter it, so that case pins instead. + const transport = isPrivateIp(resolvedIP) + ? createPinnedFetchWithDispatcher(resolvedIP, { + profile, + maxResponseSize: MAX_OAUTH_RESPONSE_BYTES, + }) + : createSsrfGuardedFetchWithDispatcher({ + profile, + maxResponseSize: MAX_OAUTH_RESPONSE_BYTES, + }) + dispatcher = transport.dispatcher + const response = await withDeadline(transport.fetch(url, { ...init, signal }), signal) // The probe's `initialize` can stream (text/event-stream); hand it back live so the // buffer doesn't drain/stall it. Every OAuth leg is single-shot JSON and is buffered. const contentType = response.headers.get('content-type') ?? '' diff --git a/packages/security/src/egress.ts b/packages/security/src/egress.ts index 1618834305e..18c5942d8b7 100644 --- a/packages/security/src/egress.ts +++ b/packages/security/src/egress.ts @@ -405,6 +405,19 @@ export function policyCanVouch(policy: EgressPolicy): boolean { ) } +/** + * Whether a refusal could be reversed specifically by learning the resolved + * address — an IP-range entry, or the legacy blanket private grant. + * + * Narrower than {@link policyCanVouch}: a hostname allowlist entry and the + * loopback carve-out are both decided from the hostname, so {@link evaluateUrl} + * has already applied them. A synchronous caller must use this rather than the + * broader predicate, or it defers everything and stops refusing anything. + */ +export function policyDefersToAddress(policy: EgressPolicy): boolean { + return policy.allowedRanges.length > 0 || policy.allowPrivate +} + /** * Whether a refusal could be lifted by learning the destination's address. * `scheme-not-permitted` and `address-metadata` never can be. @@ -432,6 +445,12 @@ export function evaluateAddress(url: URL, address: string, policy: EgressPolicy) return deny('address-metadata', address) } + // Before any vouching rule: an address that cannot be parsed cannot be + // classified, and an allowlisted hostname must not carry it past the check. + if (!isIpLiteral(unwrapIpv6Brackets(address))) { + return deny('address-blocked', `${address} is not a valid address`) + } + const vouched = isVouched(url, address, policy) const shape = checkSchemeAndPort(url, vouched, policy) diff --git a/scripts/check-egress-boundary.ts b/scripts/check-egress-boundary.ts index a7caf7f70f8..123f51efe74 100644 --- a/scripts/check-egress-boundary.ts +++ b/scripts/check-egress-boundary.ts @@ -75,6 +75,58 @@ const ALLOWED = new Set([ 'apps/sim/lib/core/utils/fetch-deadline.ts', ]) +/** + * Blanks comment bodies, preserving byte offsets so reported line numbers stay + * exact. Without this the rules match their own documentation: a comment warning + * against `require('undici')` reads identically to the call. + * + * Strings are deliberately left intact — the module specifier is itself a string, + * so blanking them would stop every rule matching anything. A match that starts + * inside a string is rejected separately by {@link stringRanges}. + */ +function blankComments(source: string): string { + const out = source.split('') + let i = 0 + while (i < source.length) { + const two = source.slice(i, i + 2) + if (two === '//' || two === '/*') { + const end = + two === '//' + ? (source.indexOf('\n', i) + 1 || source.length + 1) - 1 + : source.indexOf('*/', i + 2) + 2 || source.length + for (let j = i; j < end; j++) if (out[j] !== '\n') out[j] = ' ' + i = end + continue + } + if (two[0] === '"' || two[0] === "'" || two[0] === '`') { + let j = i + 1 + while (j < source.length && source[j] !== two[0]) j += source[j] === '\\' ? 2 : 1 + i = j + 1 + continue + } + i++ + } + return out.join('') +} + +/** Half-open [start, end) ranges covering every string literal body. */ +function stringRanges(source: string): Array<[number, number]> { + const ranges: Array<[number, number]> = [] + let i = 0 + while (i < source.length) { + const ch = source[i] + if (ch === '"' || ch === "'" || ch === '`') { + let j = i + 1 + while (j < source.length && source[j] !== ch) j += source[j] === '\\' ? 2 : 1 + ranges.push([i + 1, j]) + i = j + 1 + continue + } + i++ + } + return ranges +} + function walk(dir: string, out: string[] = []): string[] { for (const entry of readdirSync(dir, { withFileTypes: true })) { if (SKIP_DIRS.has(entry.name)) continue @@ -102,15 +154,24 @@ function main() { const rel = path.relative(ROOT, file).split(path.sep).join('/') if (ALLOWED.has(rel)) continue scanned++ - const source = readFileSync(file, 'utf8') + const raw = readFileSync(file, 'utf8') + const source = blankComments(raw) + const strings = stringRanges(source) + const insideString = (index: number) => + strings.some(([from, to]) => index >= from && index < to) + for (const { pattern, kind } of RUNTIME_LOADS) { pattern.lastIndex = 0 for (const match of source.matchAll(pattern)) { + if (match.index === undefined || insideString(match.index)) continue violations.push({ file: rel, line: source.slice(0, match.index).split('\n').length, kind, - snippet: match[0].replace(/\s+/g, ' ').trim(), + snippet: raw + .slice(match.index, match.index + match[0].length) + .replace(/\s+/g, ' ') + .trim(), }) } } From 0a50b90bc684320660661ff09827068a1143cec4 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 21:44:28 -0700 Subject: [PATCH 14/20] test(knowledge): state the egress profile on the rebased retry test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Staging's #7255 added this case while this branch was making `profile` part of SecureFetchOptions. The rebase kept both, leaving a call that exercised the fail-closed fallback rather than asserting a real profile — tests are excluded from type-check, so nothing caught it. --- apps/sim/lib/knowledge/documents/utils.test.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/knowledge/documents/utils.test.ts b/apps/sim/lib/knowledge/documents/utils.test.ts index d6a116e0674..da505416f91 100644 --- a/apps/sim/lib/knowledge/documents/utils.test.ts +++ b/apps/sim/lib/knowledge/documents/utils.test.ts @@ -786,13 +786,18 @@ describe('secureFetchWithRetry', () => { const response = await secureFetchWithRetry('https://example.com/api', { method: 'GET', headers: { Accept: 'application/json' }, + profile: 'configuredEndpoint', }) expect(response.status).toBe(200) expect(mockSecureFetchWithValidation).toHaveBeenCalledTimes(1) const [url, options, paramName] = mockSecureFetchWithValidation.mock.calls[0] expect(url).toBe('https://example.com/api') - expect(options).toMatchObject({ method: 'GET', headers: { Accept: 'application/json' } }) + expect(options).toMatchObject({ + method: 'GET', + headers: { Accept: 'application/json' }, + profile: 'configuredEndpoint', + }) expect(paramName).toBe('url') }) From bb0037652ce4ea3bc677d52e76a8e012b95d2c2b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 21:52:49 -0700 Subject: [PATCH 15/20] fix(egress): judge IP literals synchronously and match every IPv4 spelling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more from review, both verified with probes first. The deferral added last round applied to every liftable refusal whenever an IP range was configured — literals included. A literal has already been judged against its own address, so a lookup can add nothing, and deferring accepted literals outside every configured range: with only `10.0.0.0/8` allowlisted, `https://192.168.1.1` came back valid. It now defers hostnames only. `matchesRangeAllowlist` compared the raw parsed address, so a resolver answering with `::a00:1` — which is 10.0.0.1 — was refused by a `10.0.0.0/8` entry. It over-blocks rather than under-blocks, but an allowlisted destination became unreachable depending on what DNS returned. `canonicalAddress` already existed for the metadata comparison and simply was not used here. Folding needed a carve-out the metadata path did not: `::` and `::1` are the unspecified and loopback addresses, not an IPv4 carried inside IPv6. Folding `::1` to `0.0.0.1` would have let a `0.0.0.0/8` entry match loopback and stopped `::1/128` matching it. Both directions are pinned by tests, alongside the four spellings an IPv4 range must accept. --- .../core/security/input-validation.test.ts | 16 +++++++++++ .../sim/lib/core/security/input-validation.ts | 11 +++++++- packages/security/src/egress.test.ts | 27 +++++++++++++++++++ packages/security/src/egress.ts | 12 ++++++--- 4 files changed, 62 insertions(+), 4 deletions(-) diff --git a/apps/sim/lib/core/security/input-validation.test.ts b/apps/sim/lib/core/security/input-validation.test.ts index 97dffe3a421..fcbc5510004 100644 --- a/apps/sim/lib/core/security/input-validation.test.ts +++ b/apps/sim/lib/core/security/input-validation.test.ts @@ -473,6 +473,22 @@ describe('validateUrlWithDNS', () => { expect(result.error).toContain('private or reserved address') }) + it('refuses an IP literal outside the configured range without deferring', () => { + // A literal was judged against its own address, so a lookup could add + // nothing — deferring it would accept literals outside every range. + envFlagsMock.egressAllowedIpRanges = '10.0.0.0/8' + try { + expect( + validateExternalUrl('https://192.168.1.1/x', 'url', 'configuredEndpoint').isValid + ).toBe(false) + expect(validateExternalUrl('https://10.0.0.5/x', 'url', 'configuredEndpoint').isValid).toBe( + true + ) + } finally { + envFlagsMock.egressAllowedIpRanges = undefined + } + }) + it('permits a private IP once the operator allowlists its range', async () => { envFlagsMock.egressAllowedIpRanges = '192.168.0.0/16' try { diff --git a/apps/sim/lib/core/security/input-validation.ts b/apps/sim/lib/core/security/input-validation.ts index 0d185fee5d7..4355a5f7520 100644 --- a/apps/sim/lib/core/security/input-validation.ts +++ b/apps/sim/lib/core/security/input-validation.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { evaluateUrl, isLiftableByVouching, policyDefersToAddress } from '@sim/security/egress' +import { isIpLiteral, unwrapIpv6Brackets } from '@sim/security/ssrf' import { describeEgressDenial, type EgressProfile, @@ -521,7 +522,15 @@ export function validateExternalUrl( // host permitted only by EGRESS_ALLOWED_IP_RANGES cannot be recognised until // DNS runs, and refusing here would stop it being configured at all. // validateUrlWithDNS makes the authoritative call before anything is dialled. - if (policyDefersToAddress(policy) && isLiftableByVouching(decision.reason)) { + // + // Only for a hostname. A literal was judged against its own address, so there + // is nothing a lookup could add and deferring would accept a literal outside + // every configured range. + if ( + !isIpLiteral(unwrapIpv6Brackets(parsed.hostname)) && + policyDefersToAddress(policy) && + isLiftableByVouching(decision.reason) + ) { return { isValid: true } } diff --git a/packages/security/src/egress.test.ts b/packages/security/src/egress.test.ts index 3c9f42060ec..ffdaef200a7 100644 --- a/packages/security/src/egress.test.ts +++ b/packages/security/src/egress.test.ts @@ -155,6 +155,33 @@ describe('operator allowlist — the self-hosted posture', () => { }) }) +describe('an IPv4 range matches every spelling of the same address', () => { + const ranged = createEgressPolicy({ allowedRanges: '10.0.0.0/8' }) + + it.each([ + ['10.0.0.1', 'plain IPv4'], + ['::a00:1', 'the IPv4-compatible form a resolver can return'], + ['::ffff:10.0.0.1', 'the IPv4-mapped form'], + ['::10.0.0.1', 'IPv4-compatible written long-hand'], + ])('permits %s — %s', (address) => { + expect(decide(ranged, 'https://svc.internal/', address).allowed).toBe(true) + }) + + it('does not fold the addresses that are IPv6 in their own right', () => { + // `::1` is loopback, not 0.0.0.1 carried inside IPv6 — folding it would let + // a 0.0.0.0/8 entry match it, and stop `::1/128` matching it. + const loopback = createEgressPolicy({ allowedRanges: '::1/128' }) + expect(decide(loopback, 'https://svc.internal/', '::1').allowed).toBe(true) + + const zeroPage = createEgressPolicy({ allowedRanges: '0.0.0.0/8' }) + expect(decide(zeroPage, 'https://svc.internal/', '::1').allowed).toBe(false) + }) + + it('still refuses an address outside the range in any spelling', () => { + expect(reason(ranged, 'https://svc.internal/', '::c0a8:101')).toBe('address-blocked') + }) +}) + describe('the same operator config is inert on the hosted posture', () => { it.each([ ['http://host.docker.internal/', '192.168.65.254'], diff --git a/packages/security/src/egress.ts b/packages/security/src/egress.ts index 18c5942d8b7..f6f85a62832 100644 --- a/packages/security/src/egress.ts +++ b/packages/security/src/egress.ts @@ -245,8 +245,10 @@ function matchesHostAllowlist(host: string, policy: EgressPolicy): boolean { function matchesRangeAllowlist(address: string, policy: EgressPolicy): boolean { if (policy.allowedRanges.length === 0) return false - const clean = unwrapIpv6Brackets(address) - if (!ipaddr.isValid(clean)) return false + // Canonical form, so an operator's IPv4 CIDR still matches a resolver that + // answered with the IPv4-compatible IPv6 spelling of the same address. + const clean = canonicalAddress(address) + if (clean === null) return false const parsed = ipaddr.process(clean) return policy.allowedRanges.some( (range) => @@ -268,7 +270,11 @@ function canonicalAddress(address: string): string | null { const parsed = ipaddr.process(clean) if (parsed.kind() === 'ipv6') { const parts = (parsed as ipaddr.IPv6).parts - if (parts.slice(0, 6).every((part) => part === 0)) { + const embedded = ((parts[6] << 16) >>> 0) + parts[7] + // `::` and `::1` are the unspecified and loopback addresses, not an IPv4 + // carried inside IPv6 — folding them would turn `::1` into `0.0.0.1` and + // stop an operator's `::1/128` entry matching it. + if (parts.slice(0, 6).every((part) => part === 0) && embedded > 1) { return ipaddr .fromByteArray([ (parts[6] >> 8) & 0xff, From 085e3ae52ba2797f0ef4804bee153671ace00680 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 22:09:02 -0700 Subject: [PATCH 16/20] fix(egress): fold NAT64 addresses, and parse the boundary check properly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A DNS64 resolver returns an IPv4 destination wrapped in the RFC 6052 well-known prefix, so `64:ff9b::a9fe:a9fe` is the metadata endpoint. Canonicalization did not recognise that, and an allowlisted hostname resolving there was permitted — probed and confirmed. Narrower than it first looked: `isPrivateIp` already rejects every NAT64 form, so an ordinary destination was never exposed; the hole was only for a vouched one, where the class check is skipped and the metadata exception is all that stands in the way. Folding it also makes an operator's IPv4 range match the NAT64 spelling of an address inside it. The boundary check is parsed with the TypeScript AST instead of matched with a regex. Two rounds found holes in both directions — a comment or string naming a transport reported a violation that did not exist, and a regex literal containing a quote hid one that did — which is what a scanner that does not understand the grammar will keep doing. A template interpolation was the third. Parsing surfaced a false positive the regex never had: `import { type X } from 'undici'` is elided under `verbatimModuleSyntax: false`, so it cannot load anything. Elision is now modelled properly — a default or namespace binding keeps an import alive, an all-type named import does not. Eleven forms verified by probe. That needs `@typescript/typescript6` declared at the root rather than relied on by hoisting from apps/sim. It adds no new `tsc` bin, and check:native-typecheck still reports 7.0.2. Docs: the provenance table was missing `proxy`, and the line about naming a destination read as though the allowlist reaches everything. It does not reach a content fetch or a proxy — a proxy must be public, since it decides where every other request may go. --- .../docs/platform/self-hosting/security.mdx | 5 +- bun.lock | 1 + package.json | 1 + packages/security/src/egress.test.ts | 9 + packages/security/src/egress.ts | 31 ++- scripts/check-egress-boundary.ts | 188 ++++++++---------- 6 files changed, 123 insertions(+), 112 deletions(-) diff --git a/apps/docs/content/docs/platform/self-hosting/security.mdx b/apps/docs/content/docs/platform/self-hosting/security.mdx index cb9d30f97f3..cdf23faff7a 100644 --- a/apps/docs/content/docs/platform/self-hosting/security.mdx +++ b/apps/docs/content/docs/platform/self-hosting/security.mdx @@ -143,8 +143,9 @@ Sim blocks outbound requests to private, reserved, and loopback addresses. This | Request target | The HTTP block's URL, an A2A agent, an RSS feed, a Function block's `fetch` | Yes | | Database host | A database, cache, or mail connector's host | Yes | | Content fetch | An image URL, a file imported by URL, a link from a third-party API response | **No** | +| Proxy | The outbound HTTP proxy itself | **No** | -Content fetches never reach a private destination, allowlist or not — that is the class where SSRF is actually exploited. +Content fetches never reach a private destination, allowlist or not — that is the class where SSRF is actually exploited. Nor does the proxy: it is the component deciding where everything else may go, so it is held to public destinations regardless of what the allowlist says. Deployments frequently need to reach an internal service by name or address. Name the destinations: @@ -157,6 +158,8 @@ A wildcard (`*.svc.cluster.local`) and a broad range (`10.0.0.0/8`) are accepted Naming a destination permits plain HTTP to it and lifts the blocked-port list for it, since those are the same decision about the same host. Cloud metadata endpoints (`169.254.169.254` and equivalents) stay blocked no matter how broad the allowlist is, and both variables are ignored entirely on Sim Cloud. +The allowlist reaches the four provenances marked **Yes** above. It does not reach a content fetch, and it does not reach a proxy: an HTTP block's `proxyUrl` must be a public address, because the proxy is what decides where every other request may go. Adding an internal proxy to the allowlist will not make it work. + To reach a service on the Docker host, pair the allowlist with the host alias that Compose already sets up: ```bash diff --git a/bun.lock b/bun.lock index a5f36c89f8f..d93417d79b1 100644 --- a/bun.lock +++ b/bun.lock @@ -11,6 +11,7 @@ "@octokit/rest": "^21.0.0", "@types/opentype.js": "1.3.10", "@typescript/native": "npm:typescript@^7.0.2", + "@typescript/typescript6": "^6.0.2", "@vercel/og": "0.6.8", "chalk": "5.6.2", "glob": "13.0.0", diff --git a/package.json b/package.json index ca7e199be1b..caf10b1d7b4 100644 --- a/package.json +++ b/package.json @@ -136,6 +136,7 @@ "@clack/prompts": "1.7.0", "@octokit/rest": "^21.0.0", "@types/opentype.js": "1.3.10", + "@typescript/typescript6": "^6.0.2", "@typescript/native": "npm:typescript@^7.0.2", "@vercel/og": "0.6.8", "chalk": "5.6.2", diff --git a/packages/security/src/egress.test.ts b/packages/security/src/egress.test.ts index ffdaef200a7..39750ceb0fa 100644 --- a/packages/security/src/egress.test.ts +++ b/packages/security/src/egress.test.ts @@ -100,6 +100,14 @@ describe('cloud metadata is never reachable', () => { ) }) + it.each([ + ['64:ff9b::a9fe:a9fe', 'the NAT64 form a DNS64 resolver returns'], + ['64:ff9b::169.254.169.254', 'NAT64 written long-hand'], + ])('blocks %s through an allowlisted hostname — %s', (address) => { + const permissive = createEgressPolicy({ allowedHosts: 'internal.corp' }) + expect(reason(permissive, 'https://internal.corp/', address)).toBe('address-metadata') + }) + it('blocks the AWS IPv6 metadata address', () => { expect(reason(hosted, 'https://[fd00:ec2::254]/')).toBe('address-metadata') }) @@ -163,6 +171,7 @@ describe('an IPv4 range matches every spelling of the same address', () => { ['::a00:1', 'the IPv4-compatible form a resolver can return'], ['::ffff:10.0.0.1', 'the IPv4-mapped form'], ['::10.0.0.1', 'IPv4-compatible written long-hand'], + ['64:ff9b::a00:1', 'the NAT64 form'], ])('permits %s — %s', (address) => { expect(decide(ranged, 'https://svc.internal/', address).allowed).toBe(true) }) diff --git a/packages/security/src/egress.ts b/packages/security/src/egress.ts index f6f85a62832..e678bcd963c 100644 --- a/packages/security/src/egress.ts +++ b/packages/security/src/egress.ts @@ -263,6 +263,20 @@ function matchesRangeAllowlist(address: string, policy: EgressPolicy): boolean { * URL parser normalizes to `::a9fe:a9fe`), so comparing without this misses the * metadata endpoint written that way. */ +function embeddedIpv4(parts: readonly number[]): string { + return ipaddr + .fromByteArray([ + (parts[6] >> 8) & 0xff, + parts[6] & 0xff, + (parts[7] >> 8) & 0xff, + parts[7] & 0xff, + ]) + .toString() +} + +/** RFC 6052 well-known NAT64 prefix, `64:ff9b::/96`. */ +const NAT64_WELL_KNOWN_PREFIX = [0x0064, 0xff9b, 0, 0, 0, 0] as const + function canonicalAddress(address: string): string | null { const clean = unwrapIpv6Brackets(address) if (!ipaddr.isValid(clean)) return null @@ -270,19 +284,20 @@ function canonicalAddress(address: string): string | null { const parsed = ipaddr.process(clean) if (parsed.kind() === 'ipv6') { const parts = (parsed as ipaddr.IPv6).parts + + // A DNS64 resolver hands back the IPv4 destination wrapped in the well-known + // NAT64 prefix. Left unfolded, `64:ff9b::a9fe:a9fe` does not read as the + // metadata endpoint it is, and a vouched destination would reach it. + if (NAT64_WELL_KNOWN_PREFIX.every((part, index) => parts[index] === part)) { + return embeddedIpv4(parts) + } + const embedded = ((parts[6] << 16) >>> 0) + parts[7] // `::` and `::1` are the unspecified and loopback addresses, not an IPv4 // carried inside IPv6 — folding them would turn `::1` into `0.0.0.1` and // stop an operator's `::1/128` entry matching it. if (parts.slice(0, 6).every((part) => part === 0) && embedded > 1) { - return ipaddr - .fromByteArray([ - (parts[6] >> 8) & 0xff, - parts[6] & 0xff, - (parts[7] >> 8) & 0xff, - parts[7] & 0xff, - ]) - .toString() + return embeddedIpv4(parts) } } return parsed.toString() diff --git a/scripts/check-egress-boundary.ts b/scripts/check-egress-boundary.ts index 123f51efe74..831bafabc4a 100644 --- a/scripts/check-egress-boundary.ts +++ b/scripts/check-egress-boundary.ts @@ -9,17 +9,24 @@ * `undici` directly gets none of that, and the omission is invisible — the code * works, it just has no guard. * - * This checks the import edge rather than the call, because that is the part + * This checks the module edge rather than the call, because that is the part * that cannot be hidden behind a helper. * + * Parsed with the TypeScript AST rather than matched with a regex. Two rounds of + * review found regex holes in both directions — a comment or string naming a + * transport reported a violation that was not there, and a regex literal + * containing a quote hid a real one — which is what a scanner that does not + * understand the grammar will keep doing. + * * Not checked: bare `fetch()`. It is used constantly for same-origin and * server-action calls where the guard does not apply, so flagging it would be - * noise. The transports it can reach are covered by the rules above. + * noise. The transports it can reach are covered by the rules below. * * Usage: bun run scripts/check-egress-boundary.ts */ import { readdirSync, readFileSync } from 'node:fs' import path from 'node:path' +import ts from '@typescript/typescript6' const ROOT = path.resolve(import.meta.dir, '..') @@ -36,31 +43,15 @@ const SCAN_DIRS = [ const SKIP_DIRS = new Set(['node_modules', 'dist', '.next', '.turbo', 'coverage']) /** Modules that can open a socket directly. */ -const TRANSPORTS = ['http', 'https', 'undici', 'http-proxy-agent', 'https-proxy-agent'] - -const MODULE_ALTERNATION = TRANSPORTS.map((name) => name.replace(/[-]/g, '\\-')).join('|') -const SPECIFIER = `['"](?:node:)?(?:${MODULE_ALTERNATION})['"]` - -/** - * Every way a module reaches one of these at runtime. - * - * Matched against the whole source rather than line by line, because an import - * list broken across lines would otherwise slip past. `import type` is excluded: - * a type has no runtime presence and cannot open anything. - */ -const RUNTIME_LOADS: ReadonlyArray<{ pattern: RegExp; kind: string }> = [ - { - pattern: new RegExp(`^[ \t]*import\\s+(?!type\\s)[\\s\\S]*?from\\s*${SPECIFIER}`, 'gm'), - kind: 'import', - }, - { pattern: new RegExp(`^[ \t]*import\\s*${SPECIFIER}`, 'gm'), kind: 'side-effect import' }, - { - pattern: new RegExp(`^[ \t]*export\\s+(?!type\\s)[\\s\\S]*?from\\s*${SPECIFIER}`, 'gm'), - kind: 're-export', - }, - { pattern: new RegExp(`\\bimport\\s*\\(\\s*${SPECIFIER}\\s*\\)`, 'g'), kind: 'dynamic import' }, - { pattern: new RegExp(`\\brequire\\s*\\(\\s*${SPECIFIER}\\s*\\)`, 'g'), kind: 'require' }, -] +const TRANSPORTS = new Set([ + 'http', + 'https', + 'node:http', + 'node:https', + 'undici', + 'http-proxy-agent', + 'https-proxy-agent', +]) /** * Modules allowed to hold a transport import, each because it *is* part of the @@ -75,58 +66,6 @@ const ALLOWED = new Set([ 'apps/sim/lib/core/utils/fetch-deadline.ts', ]) -/** - * Blanks comment bodies, preserving byte offsets so reported line numbers stay - * exact. Without this the rules match their own documentation: a comment warning - * against `require('undici')` reads identically to the call. - * - * Strings are deliberately left intact — the module specifier is itself a string, - * so blanking them would stop every rule matching anything. A match that starts - * inside a string is rejected separately by {@link stringRanges}. - */ -function blankComments(source: string): string { - const out = source.split('') - let i = 0 - while (i < source.length) { - const two = source.slice(i, i + 2) - if (two === '//' || two === '/*') { - const end = - two === '//' - ? (source.indexOf('\n', i) + 1 || source.length + 1) - 1 - : source.indexOf('*/', i + 2) + 2 || source.length - for (let j = i; j < end; j++) if (out[j] !== '\n') out[j] = ' ' - i = end - continue - } - if (two[0] === '"' || two[0] === "'" || two[0] === '`') { - let j = i + 1 - while (j < source.length && source[j] !== two[0]) j += source[j] === '\\' ? 2 : 1 - i = j + 1 - continue - } - i++ - } - return out.join('') -} - -/** Half-open [start, end) ranges covering every string literal body. */ -function stringRanges(source: string): Array<[number, number]> { - const ranges: Array<[number, number]> = [] - let i = 0 - while (i < source.length) { - const ch = source[i] - if (ch === '"' || ch === "'" || ch === '`') { - let j = i + 1 - while (j < source.length && source[j] !== ch) j += source[j] === '\\' ? 2 : 1 - ranges.push([i + 1, j]) - i = j + 1 - continue - } - i++ - } - return ranges -} - function walk(dir: string, out: string[] = []): string[] { for (const entry of readdirSync(dir, { withFileTypes: true })) { if (SKIP_DIRS.has(entry.name)) continue @@ -141,7 +80,69 @@ interface Violation { file: string line: number kind: string - snippet: string + specifier: string +} + +/** + * Whether TypeScript drops this import at emit, leaving nothing that could load + * the module. + * + * True for `import type … from 'm'` and for a named import whose every binding + * is marked `type` — the repo compiles with `verbatimModuleSyntax: false`, so + * that form is elided rather than kept as a side-effect import. A default or + * namespace binding is a value and keeps the import alive, and a bare + * `import 'm'` has no clause at all and always runs. + */ +function isElidedImport(node: ts.ImportDeclaration): boolean { + const clause = node.importClause + if (!clause) return false + if (clause.isTypeOnly) return true + if (clause.name) return false + + const bindings = clause.namedBindings + if (!bindings || !ts.isNamedImports(bindings)) return false + return bindings.elements.every((element) => element.isTypeOnly) +} + +/** + * Every runtime reference to a transport module. An import TypeScript elides is + * skipped: it has no runtime presence and cannot open anything. + */ +function findTransportLoads(file: string, source: string): Array> { + const sourceFile = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true) + const found: Array> = [] + + const record = (node: ts.Node, specifier: string, kind: string) => { + if (!TRANSPORTS.has(specifier)) return + const { line } = sourceFile.getLineAndCharacterOfPosition(node.getStart(sourceFile)) + found.push({ line: line + 1, kind, specifier }) + } + + const visit = (node: ts.Node): void => { + if (ts.isImportDeclaration(node) && ts.isStringLiteralLike(node.moduleSpecifier)) { + if (!isElidedImport(node)) { + record(node, node.moduleSpecifier.text, node.importClause ? 'import' : 'side-effect import') + } + } else if ( + ts.isExportDeclaration(node) && + node.moduleSpecifier && + ts.isStringLiteralLike(node.moduleSpecifier) && + !node.isTypeOnly + ) { + record(node, node.moduleSpecifier.text, 're-export') + } else if (ts.isCallExpression(node)) { + const isDynamicImport = node.expression.kind === ts.SyntaxKind.ImportKeyword + const isRequire = ts.isIdentifier(node.expression) && node.expression.text === 'require' + const argument = node.arguments[0] + if ((isDynamicImport || isRequire) && argument && ts.isStringLiteralLike(argument)) { + record(node, argument.text, isDynamicImport ? 'dynamic import' : 'require') + } + } + ts.forEachChild(node, visit) + } + + visit(sourceFile) + return found } function main() { @@ -149,31 +150,12 @@ function main() { let scanned = 0 for (const scanDir of SCAN_DIRS) { - const abs = path.join(ROOT, scanDir) - for (const file of walk(abs)) { + for (const file of walk(path.join(ROOT, scanDir))) { const rel = path.relative(ROOT, file).split(path.sep).join('/') if (ALLOWED.has(rel)) continue scanned++ - const raw = readFileSync(file, 'utf8') - const source = blankComments(raw) - const strings = stringRanges(source) - const insideString = (index: number) => - strings.some(([from, to]) => index >= from && index < to) - - for (const { pattern, kind } of RUNTIME_LOADS) { - pattern.lastIndex = 0 - for (const match of source.matchAll(pattern)) { - if (match.index === undefined || insideString(match.index)) continue - violations.push({ - file: rel, - line: source.slice(0, match.index).split('\n').length, - kind, - snippet: raw - .slice(match.index, match.index + match[0].length) - .replace(/\s+/g, ' ') - .trim(), - }) - } + for (const load of findTransportLoads(rel, readFileSync(file, 'utf8'))) { + violations.push({ file: rel, ...load }) } } } @@ -186,7 +168,7 @@ function main() { console.error('✗ check-egress-boundary: raw HTTP transport outside the egress guard\n') for (const violation of violations) { console.error(` ${violation.file}:${violation.line} (${violation.kind})`) - console.error(` ${violation.snippet}`) + console.error(` ${violation.specifier}`) } console.error( '\n These modules can open a socket without resolving and classifying the\n' + From 2488ca57134df283978fef592ad1639f0aceb070 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 22:28:58 -0700 Subject: [PATCH 17/20] fix(egress): close an http2 gap and a regression from the last round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `node:http2` is an HTTP client that opens outbound sockets, and the boundary check did not list it — probed and confirmed it passed. `axios`, `node-fetch` and `got` are also present as transitive dependencies; nothing scanned imports any of them today, and listing them keeps that true. The pinned fetch refused an allowlisted plain-HTTP destination before connecting. That is a regression from last round: making the initial target re-check the hostname pre-DNS meant a host allowlisted by IP range came back `insecure-scheme`, since a range match is only visible after resolution. The caller already holds the resolved address, so it passes it for the initial target. Redirect hops are still judged afresh. The loopback carve-out swallowed the allowlist. With `allowLoopback` on, a `localhost` that resolved to a non-loopback address returned unvouched immediately rather than falling through, so `10.0.0.5` was refused even with `10.0.0.0/8` allowlisted. It falls through now, with both directions covered. Docs: "Sim blocks outbound requests to private, reserved, and loopback addresses" is not true on a self-hosted deployment — allowlisted provenances reach private destinations, and loopback is reachable with no allowlist at all. It says "By default" and names both exceptions. --- .../docs/platform/self-hosting/security.mdx | 2 +- .../core/security/input-validation.server.ts | 30 +++++++++++++------ packages/security/src/egress.test.ts | 12 ++++++++ packages/security/src/egress.ts | 9 ++++-- scripts/check-egress-boundary.ts | 8 +++++ 5 files changed, 48 insertions(+), 13 deletions(-) diff --git a/apps/docs/content/docs/platform/self-hosting/security.mdx b/apps/docs/content/docs/platform/self-hosting/security.mdx index cdf23faff7a..30c9b6e7bce 100644 --- a/apps/docs/content/docs/platform/self-hosting/security.mdx +++ b/apps/docs/content/docs/platform/self-hosting/security.mdx @@ -134,7 +134,7 @@ Resource ceilings for the in-process path: ## The SSRF boundary -Sim blocks outbound requests to private, reserved, and loopback addresses. This stops a workflow from being used to scan your internal network. Every outbound request is classified by where its URL came from: +By default Sim blocks outbound requests to private, reserved, and loopback addresses. This stops a workflow from being used to scan your internal network. Two things soften it on a self-hosted deployment: the provenances marked **Yes** below reach whatever you allowlist, and a destination written as `localhost` or a loopback literal is reachable without any allowlist at all — a local Ollama or Jupyter is the ordinary case. Neither applies on Sim Cloud. Every outbound request is classified by where its URL came from: | Provenance | Examples | Reaches allowlisted private destinations | |---|---|---| diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index 743513ef29e..cbd61709230 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -486,7 +486,11 @@ const MAX_GUARDED_REDIRECTS = 5 * a 3xx to `http://169.254.169.254/` would otherwise connect directly. Hostname * targets are covered by {@link createSsrfGuardedLookup} at connect time. */ -function assertGuardedRedirectTarget(url: URL, profile: EgressProfile): void { +function assertGuardedRedirectTarget( + url: URL, + profile: EgressProfile, + knownAddress?: string +): void { const host = unwrapIpv6Brackets(url.hostname) // The request's own policy decides, which is how a self-hosted server on a @@ -497,9 +501,14 @@ function assertGuardedRedirectTarget(url: URL, profile: EgressProfile): void { // scheme and port — which used to be skipped entirely, so a hop could downgrade // to plain HTTP or land on a denied port as long as it was named rather than // numbered. Its address is judged by the connect-time lookup. - const decision = isIpLiteral(host) - ? checkResolvedEgress(url, host, profile) - : checkEgressUrl(url, profile) + // `knownAddress` is the address a caller already resolved for this exact URL. + // Re-judging the hostname without it would refuse a destination the operator + // allowlisted by IP range, since a range match is only visible post-DNS. + const decision = knownAddress + ? checkResolvedEgress(url, knownAddress, profile) + : isIpLiteral(host) + ? checkResolvedEgress(url, host, profile) + : checkEgressUrl(url, profile) if (!decision.allowed) { throw new Error( @@ -570,12 +579,15 @@ export async function followRedirectsGuarded( rawFetch: (url: string, init: UndiciRequestInit) => Promise, input: string, init: UndiciRequestInit, - profile: EgressProfile + profile: EgressProfile, + initialAddress?: string ): Promise { let currentUrl = new URL(input) - // The initial URL gets the same IP-literal check as redirect hops, so the exported guard is - // self-contained even when a caller skips its own up-front validation. - assertGuardedRedirectTarget(currentUrl, profile) + // The initial URL is checked too, so the guard is self-contained even when a + // caller skips its own up-front validation. A caller that already resolved it + // passes that address, so a destination allowlisted by range is not refused + // here for want of a lookup. Redirect hops are always judged afresh. + assertGuardedRedirectTarget(currentUrl, profile, initialAddress) let method = (init.method ?? 'GET').toUpperCase() let body = init.body let headers = init.headers @@ -971,7 +983,7 @@ export function createPinnedFetchWithDispatcher( } return response } - return followRedirectsGuarded(rawFetch, target, undiciInit, options.profile) + return followRedirectsGuarded(rawFetch, target, undiciInit, options.profile, resolvedIP) } return { fetch: pinned, dispatcher } diff --git a/packages/security/src/egress.test.ts b/packages/security/src/egress.test.ts index 39750ceb0fa..aeab8bc1ac7 100644 --- a/packages/security/src/egress.test.ts +++ b/packages/security/src/egress.test.ts @@ -232,6 +232,18 @@ describe('loopback is vouched by name, never by resolved address', () => { expect(reason(selfHostedLoopback, 'https://svc.internal/', '10.0.0.5')).toBe('address-blocked') }) + it('falls through to the allowlist when localhost resolves off loopback', () => { + // The carve-out not applying is not a refusal: an operator who allowlisted + // the range the resolver actually answered with still gets their host. + const withRange = createEgressPolicy({ + allowedRanges: '10.0.0.0/8', + allowLoopback: true, + insecureHttp: 'whenVouched', + }) + expect(decide(withRange, 'https://localhost/', '10.0.0.5').allowed).toBe(true) + expect(reason(withRange, 'https://localhost/', '172.16.0.5')).toBe('address-blocked') + }) + it('is absent when the policy does not permit loopback', () => { expect(reason(hosted, 'http://localhost:11434/api', '127.0.0.1')).toBe('insecure-scheme') }) diff --git a/packages/security/src/egress.ts b/packages/security/src/egress.ts index e678bcd963c..6857a64d330 100644 --- a/packages/security/src/egress.ts +++ b/packages/security/src/egress.ts @@ -336,10 +336,13 @@ function isVouched(url: URL, address: string | undefined, policy: EgressPolicy): if (matchesHostAllowlist(url.hostname, policy)) return true if (policy.allowLoopback && isLoopbackDestination(url.hostname)) { + // Before DNS there is no address to judge; evaluateAddress rules later. + if (address === undefined) return true // The address must land on loopback too, so a resolver answering - // `localhost` with a routable address cannot borrow the carve-out. Before - // DNS there is no address to judge, and evaluateAddress rules later. - return address === undefined || isLoopbackIp(unwrapIpv6Brackets(address)) + // `localhost` with a routable address cannot borrow the carve-out — but it + // may still be vouched by an allowlist entry, so this falls through rather + // than refusing outright. + if (isLoopbackIp(unwrapIpv6Brackets(address))) return true } if (address === undefined) return false diff --git a/scripts/check-egress-boundary.ts b/scripts/check-egress-boundary.ts index 831bafabc4a..5d760fd0197 100644 --- a/scripts/check-egress-boundary.ts +++ b/scripts/check-egress-boundary.ts @@ -46,11 +46,19 @@ const SKIP_DIRS = new Set(['node_modules', 'dist', '.next', '.turbo', 'coverage' const TRANSPORTS = new Set([ 'http', 'https', + 'http2', 'node:http', 'node:https', + 'node:http2', 'undici', 'http-proxy-agent', 'https-proxy-agent', + // Present in node_modules as transitive dependencies. Nothing scanned imports + // them today; listing them keeps that true. + 'axios', + 'node-fetch', + 'got', + 'superagent', ]) /** From e3f91eb7b59d89e14e4efd407b7cb433c8d7bf67 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 22:43:28 -0700 Subject: [PATCH 18/20] fix(egress): elide type-only re-exports, and qualify the loopback carve-out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `export { type X } from 'undici'` sets `isTypeOnly` on the specifiers rather than the declaration, so the check reported a transport TypeScript elides. The import path already modelled this; the export path did not. Both do now, with `export *` and value re-exports still caught. The docs said a loopback destination is reachable without any allowlist. True for a configured endpoint, self-hosted service, or request target — not for a database host, which deliberately has no such carve-out because loopback is where Sim's own database and Redis listen. As written it would have told someone configuring Postgres on localhost that an unreachable setup works. --- .../docs/platform/self-hosting/security.mdx | 2 +- scripts/check-egress-boundary.ts | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/apps/docs/content/docs/platform/self-hosting/security.mdx b/apps/docs/content/docs/platform/self-hosting/security.mdx index 30c9b6e7bce..6a9aa6cf4b7 100644 --- a/apps/docs/content/docs/platform/self-hosting/security.mdx +++ b/apps/docs/content/docs/platform/self-hosting/security.mdx @@ -134,7 +134,7 @@ Resource ceilings for the in-process path: ## The SSRF boundary -By default Sim blocks outbound requests to private, reserved, and loopback addresses. This stops a workflow from being used to scan your internal network. Two things soften it on a self-hosted deployment: the provenances marked **Yes** below reach whatever you allowlist, and a destination written as `localhost` or a loopback literal is reachable without any allowlist at all — a local Ollama or Jupyter is the ordinary case. Neither applies on Sim Cloud. Every outbound request is classified by where its URL came from: +By default Sim blocks outbound requests to private, reserved, and loopback addresses. This stops a workflow from being used to scan your internal network. Two things soften it on a self-hosted deployment: the provenances marked **Yes** below reach whatever you allowlist, and a configured endpoint, self-hosted service, or request target written as `localhost` or a loopback literal is reachable without any allowlist at all — a local Ollama or Jupyter is the ordinary case. That second carve-out stops there: a database, cache, or mail connector on `localhost` still has to be allowlisted, because loopback is where Sim's own database and Redis listen. Neither softening applies on Sim Cloud. Every outbound request is classified by where its URL came from: | Provenance | Examples | Reaches allowlisted private destinations | |---|---|---| diff --git a/scripts/check-egress-boundary.ts b/scripts/check-egress-boundary.ts index 5d760fd0197..26f0fdb2ab3 100644 --- a/scripts/check-egress-boundary.ts +++ b/scripts/check-egress-boundary.ts @@ -112,6 +112,21 @@ function isElidedImport(node: ts.ImportDeclaration): boolean { return bindings.elements.every((element) => element.isTypeOnly) } +/** + * Whether TypeScript drops this re-export at emit. + * + * True for `export type { … } from 'm'` and for a named re-export whose every + * specifier is marked `type`. `export * from 'm'` re-exports values and always + * runs the module. + */ +function isElidedExport(node: ts.ExportDeclaration): boolean { + if (node.isTypeOnly) return true + + const clause = node.exportClause + if (!clause || !ts.isNamedExports(clause)) return false + return clause.elements.every((element) => element.isTypeOnly) +} + /** * Every runtime reference to a transport module. An import TypeScript elides is * skipped: it has no runtime presence and cannot open anything. @@ -135,7 +150,7 @@ function findTransportLoads(file: string, source: string): Array Date: Fri, 28 Aug 2026 23:15:30 -0700 Subject: [PATCH 19/20] fix(egress): close two IPv6 fail-opens, and separate a carve-out from an allowlist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An audit sweep over the whole diff turned up four things worth fixing in the policy core and two in the profile layer. Fail-opens, both IPv6 translation prefixes the address folding missed: - `::ffff:0:a9fe:a9fe` (RFC 6145 IPv4-translated) was not folded, so a vouched destination whose resolver answered with it reached cloud metadata — the guarantee that metadata is never liftable did not hold for that spelling. - `64:ff9b:1::/48` (RFC 8215 local-use NAT64) carries its IPv4 destination at an offset the network operator chooses, so it cannot be read off the address at all. `ipaddr.js` calls it plain unicast, which made it a route to metadata or loopback under even the strict hosted policy. An address whose real destination cannot be determined is now refused rather than judged on its wrapper. Policy corrections: - The loopback carve-out no longer lifts the blocked-port list. It is granted without anyone asking for it, and loopback is exactly where Sim's own Postgres and Redis listen, so `http://localhost:5432` was reachable from an HTTP block on any self-hosted deployment. Only an operator naming a destination lifts ports now — a vouch carries the kind that earned it. - A hostname that says it is loopback is refused before DNS when the policy does not permit loopback, so the synchronous validator stops accepting `https://localhost/x` on the hosted platform. - `insecureHttp: 'always'` is capped at `whenVouched` on the hosted platform, where nothing is vouched. Software served without TLS is a self-hosted arrangement; a hosted deployment should not send a credential in the clear to a user-supplied MCP or vLLM host. - Wildcard allowlist entries are validated like every other entry. `*.foo.com/x` and `*..com` were accepted silently and then matched nothing. Transport: - An IP-literal redirect target is judged as the literal even when the caller supplied a resolved address, because `net.connect` dials a numeric host directly and the literal is what the socket reaches. - The connect-time lookup classifies against the request's own policy instead of a hand-rolled private-address filter, so an allowlisted private destination the redirect check permitted is no longer stranded at connect. - 300, 305 and 306 are no longer followed. 305 redirects a request into a server-named proxy, which is the one hop a guard must never take. Provenance corrections: a Vision image resolved from an internal file URL and Buffer's media probe are presigned URLs against Sim's own storage, so they take `configuredEndpoint` the way STT and Textract already do — on a self-hosted deployment with private object storage they were unreachable. Microsoft Word's upload URL comes out of a Graph response rather than configuration, so it takes `contentFetch`, matching SharePoint and Windchill. Also: `policyCanVouch` is gone, since the synchronous callers both need the narrow predicate; `validateDatabaseHost` classifies each address once instead of twice; the dead `|| !validation.resolvedIP` conjuncts the discriminated union made unreachable are removed across ~32 call sites; the boundary check scans `background/` and `blocks/`; and the docs correct the MCP row, the in-cluster naming claim, the refusal-message description, and add the upgrade notes. --- apps/docs/content/docs/agents/mcp.mdx | 2 + .../self-hosting/environment-variables.mdx | 7 +- .../docs/platform/self-hosting/security.mdx | 24 ++- .../platform/self-hosting/troubleshooting.mdx | 4 +- apps/sim/.env.example | 4 +- apps/sim/app/api/auth/sso/register/route.ts | 2 +- apps/sim/lib/a2a/client.ts | 2 +- .../security/egress-end-to-end.server.test.ts | 18 +- .../lib/core/security/egress/profiles.test.ts | 140 +++++++++++++ apps/sim/lib/core/security/egress/profiles.ts | 45 ++-- apps/sim/lib/core/security/egress/validate.ts | 11 +- .../core/security/input-validation.server.ts | 92 +++++---- apps/sim/lib/internal/agiloft/client.ts | 2 +- apps/sim/lib/internal/brex/client.ts | 2 +- apps/sim/lib/internal/buffer/operations.ts | 8 +- apps/sim/lib/internal/cursor/operations.ts | 2 +- apps/sim/lib/internal/extend/client.ts | 2 +- apps/sim/lib/internal/github/operations.ts | 4 +- .../lib/internal/google-slides/operations.ts | 2 +- .../lib/internal/google-vault/operations.ts | 2 +- apps/sim/lib/internal/grafana/client.ts | 2 +- apps/sim/lib/internal/image/fetch.ts | 2 +- apps/sim/lib/internal/image/operations.ts | 2 +- apps/sim/lib/internal/jupyter/client.ts | 2 +- apps/sim/lib/internal/linq/client.ts | 2 +- .../sim/lib/internal/microsoft-word/client.ts | 37 ++-- apps/sim/lib/internal/mistral/client.ts | 2 +- apps/sim/lib/internal/onedrive/operations.ts | 2 +- apps/sim/lib/internal/onepassword/client.ts | 2 +- apps/sim/lib/internal/pipedrive/client.ts | 4 +- apps/sim/lib/internal/pulse/client.ts | 2 +- apps/sim/lib/internal/reducto/client.ts | 2 +- apps/sim/lib/internal/sharepoint/client.ts | 2 +- .../lib/internal/twilio-voice/operations.ts | 2 +- apps/sim/lib/internal/typeform/operations.ts | 2 +- apps/sim/lib/internal/vision/client.ts | 4 +- .../lib/internal/vision/operations.test.ts | 5 +- apps/sim/lib/internal/vision/operations.ts | 15 +- apps/sim/lib/internal/zoom/operations.ts | 2 +- apps/sim/lib/mcp/domain-check.test.ts | 40 +++- apps/sim/lib/mcp/domain-check.ts | 8 +- apps/sim/lib/mcp/pinned-fetch.ts | 6 +- apps/sim/lib/media/falai.ts | 2 +- .../contexts/workspace/fetch-external-url.ts | 2 +- .../providers/azure-anthropic/index.test.ts | 13 -- apps/sim/providers/azure-anthropic/index.ts | 3 - apps/sim/providers/azure-openai/index.test.ts | 13 -- apps/sim/providers/azure-openai/index.ts | 3 - apps/sim/providers/vllm/index.test.ts | 16 -- apps/sim/providers/vllm/index.ts | 5 +- apps/sim/tools/bitbucket/utils.server.ts | 2 +- apps/sim/tools/github/utils.server.ts | 2 +- helm/sim/values.yaml | 9 +- .../src/egress-hosted-posture.test.ts | 44 ---- packages/security/src/egress.test.ts | 133 ++++++++++++ packages/security/src/egress.ts | 192 +++++++++++------- scripts/check-egress-boundary.ts | 15 +- 57 files changed, 663 insertions(+), 313 deletions(-) create mode 100644 apps/sim/lib/core/security/egress/profiles.test.ts delete mode 100644 packages/security/src/egress-hosted-posture.test.ts diff --git a/apps/docs/content/docs/agents/mcp.mdx b/apps/docs/content/docs/agents/mcp.mdx index f970a7abca6..7fb57f23627 100644 --- a/apps/docs/content/docs/agents/mcp.mdx +++ b/apps/docs/content/docs/agents/mcp.mdx @@ -87,6 +87,8 @@ Self-hosted deployments can restrict which MCP server domains are allowed by set This governs which domains may be used. It is separate from where those domains are allowed to resolve: an MCP server on a private address is reached by naming it in `EGRESS_ALLOWED_HOSTS` or `EGRESS_ALLOWED_IP_RANGES`, described in [Security](/platform/self-hosting/security#the-ssrf-boundary). Both checks apply. +The allowlist covers the server URL itself. If the server requires OAuth, the endpoints its authorization-server metadata names are treated as content rather than as configuration, so they have to be publicly routable. + ## Using MCP Tools in Agents Once MCP servers are configured, their tools become available within your agent blocks: diff --git a/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx b/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx index 2fd60ac5faf..2111c649095 100644 --- a/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx +++ b/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx @@ -147,8 +147,11 @@ Without a remote provider, user code runs in an in-process V8 isolate inside the | `WEBHOOK_EXECUTION_CONCURRENCY_LIMIT` | `75` | Webhook-triggered executions in parallel | | `SCHEDULE_EXECUTION_CONCURRENCY_LIMIT` | `30` | Scheduled executions in parallel | | `RESUME_EXECUTION_CONCURRENCY_LIMIT` | `50` | Resumed executions in parallel | -| `EGRESS_ALLOWED_HOSTS` | unset | Comma-separated hostnames outbound requests may reach on a private network. Leading wildcard allowed, e.g. `host.docker.internal,*.svc.cluster.local` | -| `EGRESS_ALLOWED_IP_RANGES` | unset | Comma-separated CIDRs or IPs outbound requests may reach on a private network, e.g. `10.0.0.0/8` | +| `EGRESS_ALLOWED_HOSTS` | unset | Comma-separated hostnames outbound requests may reach on a private network. Leading wildcard allowed, e.g. `host.docker.internal,*.svc.cluster.local`. Not honored for URLs harvested from content or a third-party response, nor for an HTTP block's `proxyUrl` | +| `EGRESS_ALLOWED_IP_RANGES` | unset | Comma-separated CIDRs or IPs outbound requests may reach on a private network, e.g. `10.0.0.0/8`. Same exclusions | +| `ALLOW_PRIVATE_DATABASE_HOSTS` | unset | **Deprecated.** Vouches for the entire private address space, for database, cache, and mail connector hosts only. Replace it with the two settings above naming specific destinations | + +A malformed entry in either allowlist stops the app at startup with a message naming the setting. See [the SSRF boundary](/platform/self-hosting/security#the-ssrf-boundary) for the accepted syntax and for what the allowlist does and does not reach. Your reverse proxy's body-size limit must be at least as large as the app limits above. See [Networking](/platform/self-hosting/networking). diff --git a/apps/docs/content/docs/platform/self-hosting/security.mdx b/apps/docs/content/docs/platform/self-hosting/security.mdx index 6a9aa6cf4b7..fee4765d84d 100644 --- a/apps/docs/content/docs/platform/self-hosting/security.mdx +++ b/apps/docs/content/docs/platform/self-hosting/security.mdx @@ -134,12 +134,12 @@ Resource ceilings for the in-process path: ## The SSRF boundary -By default Sim blocks outbound requests to private, reserved, and loopback addresses. This stops a workflow from being used to scan your internal network. Two things soften it on a self-hosted deployment: the provenances marked **Yes** below reach whatever you allowlist, and a configured endpoint, self-hosted service, or request target written as `localhost` or a loopback literal is reachable without any allowlist at all — a local Ollama or Jupyter is the ordinary case. That second carve-out stops there: a database, cache, or mail connector on `localhost` still has to be allowlisted, because loopback is where Sim's own database and Redis listen. Neither softening applies on Sim Cloud. Every outbound request is classified by where its URL came from: +By default Sim blocks outbound requests to private, reserved, and loopback addresses. This stops a workflow from being used to scan your internal network. Two things soften it on a self-hosted deployment: the provenances marked **Yes** below reach whatever you allowlist, and a configured endpoint, self-hosted service, or request target written as `localhost` or a loopback literal is reachable without any allowlist at all — a local Ollama or Jupyter is the ordinary case. That second carve-out stops short in two places: it does not lift the blocked-port list, and it does not extend to a database, cache, or mail connector on `localhost` — loopback is where Sim's own database and Redis listen, so reaching them has to be asked for. Neither softening applies on Sim Cloud. Every outbound request is classified by where its URL came from: | Provenance | Examples | Reaches allowlisted private destinations | |---|---|---| -| Configured endpoint | GitHub Enterprise, Grafana, an MCP server, a data-drain destination, a connector's host | Yes | -| Self-hosted service | vLLM, Jupyter, 1Password Connect, ClickHouse — software usually run on-prem without TLS, so plain HTTP is expected | Yes | +| Configured endpoint | GitHub Enterprise, Grafana, a data-drain destination, a connector's host | Yes | +| Self-hosted service | vLLM, Jupyter, 1Password Connect, ClickHouse, an MCP server — software usually run on-prem without TLS, so plain HTTP is expected | Yes | | Request target | The HTTP block's URL, an A2A agent, an RSS feed, a Function block's `fetch` | Yes | | Database host | A database, cache, or mail connector's host | Yes | | Content fetch | An image URL, a file imported by URL, a link from a third-party API response | **No** | @@ -156,7 +156,9 @@ EGRESS_ALLOWED_IP_RANGES=10.4.2.17/32,10.4.9.0/24 A wildcard (`*.svc.cluster.local`) and a broad range (`10.0.0.0/8`) are accepted, but they hand every workflow author the whole namespace or network. Name the hosts you actually use. -Naming a destination permits plain HTTP to it and lifts the blocked-port list for it, since those are the same decision about the same host. Cloud metadata endpoints (`169.254.169.254` and equivalents) stay blocked no matter how broad the allowlist is, and both variables are ignored entirely on Sim Cloud. +Both lists are validated when Sim starts, and a malformed entry stops it with a message naming the setting. `EGRESS_ALLOWED_HOSTS` takes hostnames only — a URL or a CIDR is rejected — and a wildcard has to be a leading `*.` covering at least two labels, so `*.local` is refused and `*.svc.cluster.local` matches `vllm.ai.svc.cluster.local` but not the bare `svc.cluster.local`. `EGRESS_ALLOWED_IP_RANGES` takes CIDRs and bare addresses; `0.0.0.0/0` is refused as a catch-all. + +Naming a destination permits plain HTTP to it and lifts the blocked-port list for it, since those are the same decision about the same host. A database, cache, or mail connector's host carries no scheme or port of its own, so naming one of those only lifts the private-address block. Cloud metadata endpoints (`169.254.169.254` and equivalents) stay blocked no matter how broad the allowlist is, and both variables are ignored entirely on Sim Cloud. The allowlist reaches the four provenances marked **Yes** above. It does not reach a content fetch, and it does not reach a proxy: an HTTP block's `proxyUrl` must be a public address, because the proxy is what decides where every other request may go. Adding an internal proxy to the allowlist will not make it work. @@ -167,9 +169,19 @@ EGRESS_ALLOWED_HOSTS=host.docker.internal ``` - An allowlist widens what every workflow author on the instance can reach. Name specific hosts and narrow ranges rather than whole private networks, and pair it with a NetworkPolicy that constrains what the app can actually reach. + An allowlist widens what every workflow author on the instance can reach. Name specific hosts and narrow ranges rather than whole private networks, and pair it with a NetworkPolicy that constrains what the app can actually reach. The chart's own NetworkPolicy permits broad egress on port 443 only, so an allowlisted in-cluster target on another port also has to be added to `networkPolicy.egress`. +### Upgrading from an earlier release + +The allowlist replaces four separate escape hatches, so a few deployments that worked before now need a destination named: + +- **`ALLOW_PRIVATE_DATABASE_HOSTS`** still works, but it is deprecated and logs a warning at startup. It vouches for the whole private address space for database, cache, and mail connector hosts. Replace it with `EGRESS_ALLOWED_HOSTS` or `EGRESS_ALLOWED_IP_RANGES` naming the hosts you actually use. +- **1Password Connect** on a private, non-loopback address, and an **MCP server** on a private address or reached through a DNS name that points at loopback, are no longer reachable implicitly. Name them. +- **`ALLOWED_MCP_DOMAINS`** governs which domains may be used; it no longer disables the address check, so an MCP server on a private address needs the allowlist too. +- **Content fetches** — an image URL, a file imported by URL, an OIDC endpoint discovered from a provider's metadata, an MCP OAuth endpoint the server's metadata names — never use the allowlist. Those destinations have to be publicly routable. +- **Redirects** are re-judged at every hop, so a redirect that downgrades to plain HTTP or lands on a blocked port is now refused. Credentials are dropped when a redirect crosses origins, and a cross-origin redirect that would carry a request body to the new origin is refused outright rather than replayed — a POST that lands on a cross-origin redirect now fails with a message saying so. + ## Client IP and forwarded headers Behind a load balancer, `X-Forwarded-For` is client-controllable. Set `AUTH_TRUSTED_PROXIES` to your proxies' actual addresses so Better Auth resolves the real client IP, and `TRUSTED_ORIGINS` if users reach Sim from more than one origin. Both are covered in [Authentication](/platform/self-hosting/authentication#behind-a-load-balancer). @@ -216,5 +228,5 @@ The service bundles ~2.2 GB of spaCy models, so first start takes around three m { question: "Can I rotate ENCRYPTION_KEY?", answer: "Not without re-encrypting everything it protects. Changing it makes workspace environment variables, stored provider API keys, MCP OAuth credentials, and deployment secrets permanently unreadable. Treat it as a permanent, backed-up value rather than a rotating secret."}, { question: "Where does user-authored code run?", answer: "By default in an in-process V8 isolate inside the app container, which isolates at the JS-engine level but shares the container's network and filesystem context. For untrusted authors, or to run Python at all, use E2B or Daytona so each execution runs in a remote sandbox."}, { question: "Why does the chart's NetworkPolicy allow traffic from any pod?", answer: "networkPolicy.ingressFrom defaults to an empty peer selector as a simple default that works on any cluster. On a shared cluster you should scope it to your ingress controller's namespace."}, - { question: "How do I reach an internal service from a workflow?", answer: "Name it in EGRESS_ALLOWED_HOSTS (hostnames, leading wildcard allowed) or EGRESS_ALLOWED_IP_RANGES (CIDRs). That permits plain HTTP to it and lifts the blocked-port list for it. Cloud metadata endpoints stay blocked regardless, content fetches never use the allowlist, and both variables are ignored on Sim Cloud."}, + { question: "How do I reach an internal service from a workflow?", answer: "Name it in EGRESS_ALLOWED_HOSTS (hostnames, leading wildcard allowed) or EGRESS_ALLOWED_IP_RANGES (CIDRs). That permits plain HTTP to it and lifts the blocked-port list for HTTP destinations; a database, cache, or mail host carries no scheme or port of its own, so naming it only lifts the private-address block. Cloud metadata endpoints stay blocked regardless, content fetches never use the allowlist, and both variables are ignored on Sim Cloud."}, ]} /> diff --git a/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx b/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx index e120504dc01..9d5845aa1cc 100644 --- a/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx +++ b/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx @@ -27,7 +27,7 @@ OLLAMA_URL=http://192.168.1.x:11434 # Linux (use actual IP) ## A Workflow Cannot Reach a Service on Your Network -Outbound requests to private, reserved, and loopback addresses are blocked by default, so a workflow pointed at your Docker host, a LAN service, or a Kubernetes service name fails with a message naming the address it resolved to. +Outbound requests to private, reserved, and loopback addresses are blocked by default, so a workflow pointed at your Docker host, a LAN service, or a Kubernetes service name fails with a message naming the blocker — the private or loopback address it resolved to, a blocked port, or `must use https:// to a public destination` when the URL is plain HTTP — and pointing at the allowlist variables. Name the destination: @@ -41,7 +41,7 @@ Naming a destination also permits plain HTTP to it and lifts the blocked-port li Two things this does not cover: - Inside a container `localhost` is the container itself, so it will never reach a service on your host. Use `host.docker.internal` (the Compose files map it) and name it above. -- URLs harvested from content or from a third-party API response — an image URL, a file imported by URL — never reach a private network, allowlist or not. +- URLs harvested from content or from a third-party API response — an image URL, a file imported by URL, an OIDC or MCP OAuth endpoint discovered from a provider's metadata — never reach a private network, allowlist or not. Nor does an HTTP block's `proxyUrl`. ## LM Studio Requests Route to Ollama diff --git a/apps/sim/.env.example b/apps/sim/.env.example index 1ec0347adac..9a937e18436 100644 --- a/apps/sim/.env.example +++ b/apps/sim/.env.example @@ -12,9 +12,9 @@ BETTER_AUTH_URL=http://localhost:3000 # Authentication Bypass (Optional - for self-hosted deployments behind private networks) # DISABLE_AUTH=true # Uncomment to bypass authentication entirely. Creates an anonymous session for all requests. -# Private Database Hosts (Optional - for self-hosted deployments only) +# Private-network egress allowlist (Optional - self-hosted only; ignored on Sim Cloud) # EGRESS_ALLOWED_HOSTS=host.docker.internal,*.svc.cluster.local # Uncomment to let outbound requests reach these hosts on a private network. Widens the SSRF boundary; only use on a trusted private network. -# EGRESS_ALLOWED_IP_RANGES=10.0.0.0/8 # Same, by CIDR. Cloud metadata endpoints stay blocked regardless. +# EGRESS_ALLOWED_IP_RANGES=10.0.0.0/8 # Same, by CIDR. Cloud metadata endpoints stay blocked regardless, and neither setting is honored for URLs harvested from content or for a proxy. # NextJS (Required) NEXT_PUBLIC_APP_URL=http://localhost:3000 diff --git a/apps/sim/app/api/auth/sso/register/route.ts b/apps/sim/app/api/auth/sso/register/route.ts index b7b120b215a..fcbe0a0c082 100644 --- a/apps/sim/app/api/auth/sso/register/route.ts +++ b/apps/sim/app/api/auth/sso/register/route.ts @@ -61,7 +61,7 @@ async function fetchOIDCDiscoveryDocument(discoveryUrl: string): Promise { const validation = await validateUrlWithDNS(agentUrl, 'agentUrl', 'requestTarget') - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new Error(validation.error || 'Agent URL validation failed') } const { resolvedIP } = validation diff --git a/apps/sim/lib/core/security/egress-end-to-end.server.test.ts b/apps/sim/lib/core/security/egress-end-to-end.server.test.ts index 6c5c9e8cd2d..87583cd946b 100644 --- a/apps/sim/lib/core/security/egress-end-to-end.server.test.ts +++ b/apps/sim/lib/core/security/egress-end-to-end.server.test.ts @@ -53,11 +53,12 @@ afterEach(resetEnvFlagsMock) // Skipped on a host with no private interface (some CI sandboxes); the policy // itself is covered without a socket in packages/security. -describe.skipIf(!host)('issue #7200 — reaching a service on a private network', () => { +describe.skipIf(!host)('reaching a service on a private network', () => { it('refuses an unlisted destination and names the setting that would permit it', async () => { + // https, so the refusal comes from the address rather than the scheme. await expect( - secureFetchWithValidation(`http://${host}:${port}/`, { profile: 'requestTarget' }) - ).rejects.toThrow(/EGRESS_ALLOWED_HOSTS/) + secureFetchWithValidation(`https://${host}:${port}/`, { profile: 'requestTarget' }) + ).rejects.toThrow(/private or reserved address.*EGRESS_ALLOWED_HOSTS/s) }) it('reaches it over plain HTTP once the operator names the range', async () => { @@ -80,8 +81,11 @@ describe.skipIf(!host)('issue #7200 — reaching a service on a private network' secureFetchWithValidation(`https://${host}:${port}/`, { profile: 'contentFetch' }) ).rejects.toThrow(/private or reserved address/) }) +}) - it('reaches a loopback service without any allowlist, as a self-hosted deployment does', async () => { +// Needs no private interface, so it runs everywhere the suite above may not. +describe('reaching a loopback service', () => { + it('works without any allowlist, as a self-hosted deployment expects', async () => { const local = createServer((request, response) => { request.resume() response.end('local') @@ -97,4 +101,10 @@ describe.skipIf(!host)('issue #7200 — reaching a service on a private network' await new Promise((resolve) => local.close(() => resolve())) } }) + + it('does not extend that to a content-provenance URL', async () => { + await expect( + secureFetchWithValidation('https://localhost:1/', { profile: 'contentFetch' }) + ).rejects.toThrow(/loopback/) + }) }) diff --git a/apps/sim/lib/core/security/egress/profiles.test.ts b/apps/sim/lib/core/security/egress/profiles.test.ts new file mode 100644 index 00000000000..d9bad1b130b --- /dev/null +++ b/apps/sim/lib/core/security/egress/profiles.test.ts @@ -0,0 +1,140 @@ +/** + * @vitest-environment node + * + * Drives `resolveEgressPolicy` through the real profile table on both + * deployment postures, so a change to `PROFILE_SPECS` or to the hosted gate is + * visible here rather than only in whatever call site happens to notice. + */ + +import { evaluateAddress, evaluateUrl } from '@sim/security/egress' +import { envFlagsMock, resetEnvFlagsMock } from '@sim/testing' +import { afterEach, describe, expect, it } from 'vitest' +import { + describeEgressDenial, + type EgressProfile, + resolveEgressPolicy, +} from '@/lib/core/security/egress/profiles' + +afterEach(resetEnvFlagsMock) + +const ALLOWLIST_PROFILES: EgressProfile[] = [ + 'configuredEndpoint', + 'selfHostedService', + 'requestTarget', + 'databaseHost', +] +const LOCKED_PROFILES: EgressProfile[] = ['contentFetch', 'proxy'] + +function decide(profile: EgressProfile, href: string, address?: string) { + const url = new URL(href) + const policy = resolveEgressPolicy(profile) + return address === undefined ? evaluateUrl(url, policy) : evaluateAddress(url, address, policy) +} + +describe('the operator allowlist reaches exactly the provenances that honor it', () => { + it.each(ALLOWLIST_PROFILES)('%s honors an allowlisted range', (profile) => { + envFlagsMock.egressAllowedIpRanges = '10.0.0.0/8' + expect(decide(profile, 'https://internal.corp/', '10.4.2.9').allowed).toBe(true) + }) + + it.each(LOCKED_PROFILES)('%s ignores it', (profile) => { + envFlagsMock.egressAllowedIpRanges = '10.0.0.0/8' + expect(decide(profile, 'https://internal.corp/', '10.4.2.9').allowed).toBe(false) + }) +}) + +describe('plain HTTP', () => { + it('is unconditional for on-prem software off the hosted platform', () => { + expect(decide('selfHostedService', 'http://vllm.corp/', '93.184.216.34').allowed).toBe(true) + }) + + it('needs the destination vouched for a configured endpoint', () => { + expect(decide('configuredEndpoint', 'http://grafana.corp/', '93.184.216.34').allowed).toBe( + false + ) + envFlagsMock.egressAllowedHosts = 'grafana.corp' + expect(decide('configuredEndpoint', 'http://grafana.corp/', '93.184.216.34').allowed).toBe(true) + }) + + it('is never available to content-provenance URLs', () => { + envFlagsMock.egressAllowedHosts = 'cdn.corp' + expect(decide('contentFetch', 'http://cdn.corp/x.png', '93.184.216.34').allowed).toBe(false) + }) +}) + +describe('the loopback carve-out', () => { + it.each(['configuredEndpoint', 'selfHostedService', 'requestTarget'] as EgressProfile[])( + '%s reaches loopback unasked off the hosted platform', + (profile) => { + expect(decide(profile, 'http://localhost:11434/', '127.0.0.1').allowed).toBe(true) + } + ) + + it.each(['contentFetch', 'databaseHost', 'proxy'] as EgressProfile[])( + '%s does not', + (profile) => { + expect(decide(profile, 'https://localhost/x', '127.0.0.1').allowed).toBe(false) + } + ) +}) + +describe('the hosted platform ignores every softening', () => { + it('drops the operator allowlist', () => { + envFlagsMock.isHosted = true + envFlagsMock.egressAllowedHosts = 'internal.corp' + envFlagsMock.egressAllowedIpRanges = '10.0.0.0/8' + for (const profile of ALLOWLIST_PROFILES) { + expect(decide(profile, 'https://internal.corp/', '10.4.2.9').allowed).toBe(false) + } + }) + + it('drops the loopback carve-out', () => { + envFlagsMock.isHosted = true + expect(decide('configuredEndpoint', 'https://localhost/x', '127.0.0.1').allowed).toBe(false) + }) + + it('caps plain HTTP, which is a self-hosted arrangement', () => { + envFlagsMock.isHosted = true + expect(decide('selfHostedService', 'http://vllm.example/', '93.184.216.34').allowed).toBe(false) + expect(decide('proxy', 'http://proxy.example/', '93.184.216.34').allowed).toBe(false) + }) + + it('offers no remedy in the refusal, where the variables would do nothing', () => { + envFlagsMock.isHosted = true + const decision = decide('requestTarget', 'https://internal.corp/', '10.4.2.9') + expect(decision.allowed).toBe(false) + if (decision.allowed) return + expect(describeEgressDenial(decision, 'url', 'requestTarget')).not.toContain('EGRESS_ALLOWED') + }) +}) + +describe('the deprecated ALLOW_PRIVATE_DATABASE_HOSTS', () => { + it('reaches database hosts only', () => { + envFlagsMock.legacyPrivateDatabaseAccess = true + expect(decide('databaseHost', 'https://pg.corp/', '10.4.2.9').allowed).toBe(true) + expect(decide('configuredEndpoint', 'https://pg.corp/', '10.4.2.9').allowed).toBe(false) + }) + + it('still cannot reach a metadata endpoint', () => { + envFlagsMock.legacyPrivateDatabaseAccess = true + expect(decide('databaseHost', 'https://pg.corp/', '169.254.169.254').allowed).toBe(false) + }) +}) + +describe('an unrecognized profile falls back to the strictest one', () => { + it('refuses what contentFetch refuses', () => { + envFlagsMock.egressAllowedIpRanges = '10.0.0.0/8' + const policy = resolveEgressPolicy('not-a-profile' as EgressProfile) + expect(evaluateAddress(new URL('https://internal.corp/'), '10.4.2.9', policy).allowed).toBe( + false + ) + }) +}) + +describe('the policy cache follows the configuration', () => { + it('rebuilds when an allowlist changes rather than serving the previous value', () => { + expect(decide('requestTarget', 'https://internal.corp/', '10.4.2.9').allowed).toBe(false) + envFlagsMock.egressAllowedIpRanges = '10.0.0.0/8' + expect(decide('requestTarget', 'https://internal.corp/', '10.4.2.9').allowed).toBe(true) + }) +}) diff --git a/apps/sim/lib/core/security/egress/profiles.ts b/apps/sim/lib/core/security/egress/profiles.ts index d771c88851f..a3851b4263e 100644 --- a/apps/sim/lib/core/security/egress/profiles.ts +++ b/apps/sim/lib/core/security/egress/profiles.ts @@ -28,9 +28,10 @@ import { /** * Where the URL for an outbound request came from. * - * - `configuredEndpoint` — a base or server URL entered during setup: a - * self-hosted vLLM or Jupyter instance, GitHub Enterprise, Grafana, - * ClickHouse, an MCP server, a data-drain destination, a connector's host. + * - `configuredEndpoint` — a base or server URL entered during setup, or a + * vendor host built in process: GitHub Enterprise, Grafana, a data-drain + * destination, a connector's host. See `selfHostedService` for the on-prem + * software that expects plain HTTP. * - `requestTarget` — supplied per run by the workflow author: the HTTP block's * `url`, an A2A agent URL, an RSS feed, a Function block's `fetch`. * - `contentFetch` — harvested from content, a third-party response, or model @@ -66,7 +67,13 @@ interface ProfileSpec { * deployment whose operator has allowlisted their entire internal range. */ readonly honorsAllowlist: boolean - /** When plain HTTP is acceptable for this provenance. */ + /** + * When plain HTTP is acceptable for this provenance, off the hosted platform. + * `always` is capped at `whenVouched` when hosted, where nothing is vouched — + * software served without TLS is a self-hosted arrangement, and a hosted + * deployment sending a credential over cleartext to a user-supplied host is + * not one this taxonomy should permit. + */ readonly insecureHttp: InsecureHttpPolicy /** * Whether loopback is reachable without being allowlisted, off the hosted @@ -143,7 +150,8 @@ function buildPolicies(config: DeploymentConfig): Record { - const config = readDeploymentConfig() - return { config, policies: buildPolicies(config) } -})() +let cache: { config: DeploymentConfig; policies: Record } | null = null /** * The policy governing requests of the given provenance on this deployment. @@ -185,19 +188,17 @@ let cache = (() => { */ export function resolveEgressPolicy(profile: EgressProfile): EgressPolicy { const config = readDeploymentConfig() - if (!sameConfig(cache.config, config)) { + if (cache === null || !sameConfig(cache.config, config)) { cache = { config, policies: buildPolicies(config) } } return cache.policies[profile] ?? cache.policies.contentFetch } /** - * Turns a refusal into a message the person who hit it can act on. - * - * The message this replaced said `url must use https:// protocol`, which was - * worse than unhelpful: it implied switching scheme would fix a destination that - * the address check was going to refuse anyway. Each reason here names the - * actual blocker and, where one exists, the remedy. + * Turns a refusal into a message the person who hit it can act on: each reason + * names the actual blocker and, where one exists, the remedy. A message that + * only names the scheme is worse than unhelpful, because it implies switching + * scheme would reach a destination the address check refuses anyway. */ export function describeEgressDenial( decision: Extract, diff --git a/apps/sim/lib/core/security/egress/validate.ts b/apps/sim/lib/core/security/egress/validate.ts index f48201c8653..92f710aeede 100644 --- a/apps/sim/lib/core/security/egress/validate.ts +++ b/apps/sim/lib/core/security/egress/validate.ts @@ -16,7 +16,7 @@ import { evaluateAddress, evaluateUrl, isLiftableByVouching, - policyCanVouch, + policyDefersToAddress, } from '@sim/security/egress' import { isIpLiteral, unwrapIpv6Brackets } from '@sim/security/ssrf' import { toError } from '@sim/utils/errors' @@ -90,9 +90,12 @@ export async function validateEgressUrl( const preflight = evaluateUrl(parsed, policy) if (!preflight.allowed) { // For a literal the pre-flight already had the address, so its verdict is - // final. For a hostname it judged the destination as unvouched; if this - // policy could still vouch for it, resolve and let the address rule. - const final = isLiteral || !policyCanVouch(policy) || !isLiftableByVouching(preflight.reason) + // final. For a hostname the host allowlist and the loopback carve-out were + // already applied, so only a policy that can vouch from the address itself + // has anything left to say — resolving otherwise leaks a lookup for a + // destination that is already refused. + const final = + isLiteral || !policyDefersToAddress(policy) || !isLiftableByVouching(preflight.reason) if (final) return fail(preflight, url, paramName, profile) } diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index cbd61709230..94b38d0b1b1 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -6,7 +6,8 @@ import https from 'https' import type { LookupFunction } from 'net' import { createLogger } from '@sim/logger' import { preferIpv4, resolveHostAddresses } from '@sim/security/dns' -import { isIpLiteral, isPrivateIp, unwrapIpv6Brackets } from '@sim/security/ssrf' +import type { EgressDecision } from '@sim/security/egress' +import { isIpLiteral, unwrapIpv6Brackets } from '@sim/security/ssrf' import { toError } from '@sim/utils/errors' import { HttpProxyAgent } from 'http-proxy-agent' import { HttpsProxyAgent } from 'https-proxy-agent' @@ -175,23 +176,21 @@ export async function validateDatabaseHost( try { const { addresses } = await resolveHostAddresses(cleanHost) - const blocked = addresses.find( - (candidate) => !checkResolvedEgress(asUrl, candidate, 'databaseHost').allowed - ) + let refusal: Extract | undefined + const blocked = addresses.find((candidate) => { + const decision = checkResolvedEgress(asUrl, candidate, 'databaseHost') + if (decision.allowed) return false + refusal = decision + return true + }) - if (blocked !== undefined) { - const decision = checkResolvedEgress(asUrl, blocked, 'databaseHost') + if (refusal !== undefined) { logger.warn('Database host resolves to blocked IP address', { paramName, hostname: host, resolvedIP: blocked, }) - return { - isValid: false, - error: decision.allowed - ? `${paramName} resolves to a blocked IP address` - : describeEgressDenial(decision, paramName, 'databaseHost'), - } + return { isValid: false, error: describeEgressDenial(refusal, paramName, 'databaseHost') } } return { @@ -412,8 +411,16 @@ export const DEFAULT_MAX_RESPONSE_BYTES = 100 * 1024 * 1024 /** Response cap for JSON/control-plane proxies to user-supplied hosts. */ export const MAX_JSON_API_RESPONSE_BYTES = 10 * 1024 * 1024 +/** + * The statuses that name a new destination to request. 300, 305 and 306 are + * deliberately absent: 305 (Use Proxy) redirects a request into a server-named + * proxy, which is the one hop a guard must never take, and the other two carry + * no single target. + */ +const FOLLOWED_REDIRECT_STATUSES: ReadonlySet = new Set([301, 302, 303, 307, 308]) + function isRedirectStatus(status: number): boolean { - return status >= 300 && status < 400 && status !== 304 + return FOLLOWED_REDIRECT_STATUSES.has(status) } function isRetryableHttpStatus(status: number): boolean { @@ -456,12 +463,30 @@ export function createPinnedLookup(resolvedIP: string): LookupFunction { * full public address set, so the OS/undici can fall back across addresses. * IPv4 is ordered first (`verbatim: false`) — our egress is IPv4-only. */ -export function createSsrfGuardedLookup(): LookupFunction { +function safeParseUrl(value: string): URL | null { + try { + return new URL(value) + } catch { + return null + } +} + +export function createSsrfGuardedLookup(profile: EgressProfile): LookupFunction { return (hostname, options, callback) => { + // Scheme and port were judged when the request URL was checked, so this + // stage only classifies addresses — but it classifies them against the + // request's own policy, so a destination the operator allowlisted is not + // stranded here after the redirect check permitted it. + const asUrl = safeParseUrl(`https://${hostname}`) dns .lookup(hostname, { all: true, verbatim: false }) .then((addresses) => { - const usable = addresses.filter((entry) => !isPrivateIp(entry.address)) + const usable = + asUrl === null + ? [] + : addresses.filter( + (entry) => checkResolvedEgress(asUrl, entry.address, profile).allowed + ) if (usable.length === 0) { callback( new Error(`Blocked by SSRF policy: ${hostname} has no publicly routable address`), @@ -494,20 +519,18 @@ function assertGuardedRedirectTarget( const host = unwrapIpv6Brackets(url.hostname) // The request's own policy decides, which is how a self-hosted server on a - // permitted private address stays reachable across a hop. It replaced a - // carve-out that permitted one pinned IP and could not express anything else. + // permitted private address stays reachable across a hop. // - // A literal is judged completely here. A hostname gets the pre-DNS half — - // scheme and port — which used to be skipped entirely, so a hop could downgrade - // to plain HTTP or land on a denied port as long as it was named rather than - // numbered. Its address is judged by the connect-time lookup. - // `knownAddress` is the address a caller already resolved for this exact URL. - // Re-judging the hostname without it would refuse a destination the operator - // allowlisted by IP range, since a range match is only visible post-DNS. - const decision = knownAddress - ? checkResolvedEgress(url, knownAddress, profile) - : isIpLiteral(host) - ? checkResolvedEgress(url, host, profile) + // A literal is judged completely here, and takes precedence over any address a + // caller resolved earlier: `net.connect` dials a numeric host directly, so the + // literal is what the socket will reach. A hostname is judged on + // `knownAddress` when the caller resolved this exact URL — a range-allowlist + // match is only visible post-DNS — and otherwise gets the pre-DNS half, scheme + // and port, with its address left to the connect-time lookup. + const decision = isIpLiteral(host) + ? checkResolvedEgress(url, host, profile) + : knownAddress + ? checkResolvedEgress(url, knownAddress, profile) : checkEgressUrl(url, profile) if (!decision.allowed) { @@ -878,7 +901,7 @@ export function createSsrfGuardedFetchWithDispatcher(options: { } { const dispatcher = new Agent({ allowH2: false, - connect: { lookup: createSsrfGuardedLookup() }, + connect: { lookup: createSsrfGuardedLookup(options.profile) }, ...(options.maxResponseSize !== undefined ? { maxResponseSize: options.maxResponseSize } : {}), }) @@ -1063,19 +1086,20 @@ export async function secureFetchWithPinnedIP( return } const redirectPolicy = options.redirectPolicy + const isCrossOrigin = new URL(redirectUrl).origin !== parsed.origin + // Legacy mode replays the method and body verbatim on every status, + // which is what persisted workflows were built against. const hop = redirectPolicy?.mode === 'standard' ? resolveRedirectHop({ status: statusCode, method: options.method ?? 'GET' }) : { method: options.method ?? 'GET', dropBody: false } - const isCrossOrigin = new URL(redirectUrl).origin !== parsed.origin let redirectHeaders = options.headers if (redirectHeaders && hop.dropBody) { redirectHeaders = stripHeaders(redirectHeaders, ENTITY_HEADERS) } // Credentials are dropped on a cross-origin hop unless a policy - // explicitly asks to keep them. Gating this on a policy being - // supplied at all, as it once was, meant the many callers that pass - // none handed their Authorization header to whatever host the + // explicitly asks to keep them, so a caller that passes no policy + // does not hand its Authorization header to whatever host the // redirect named. `host` always goes: it describes the old origin. if (redirectHeaders && isCrossOrigin) { const keepCredentials = redirectPolicy?.sendCredentialsOnCrossOriginRedirect === true @@ -1096,7 +1120,7 @@ export async function secureFetchWithPinnedIP( const redirectBody = hop.dropBody ? undefined : options.body // Refusing rather than quietly dropping the body: a bodyless replay // of a POST is a different request, and the caller cannot tell it - // happened. Matches followRedirectsGuarded, which has always refused. + // happened. Matches followRedirectsGuarded. if ( isCrossOrigin && redirectBody !== undefined && diff --git a/apps/sim/lib/internal/agiloft/client.ts b/apps/sim/lib/internal/agiloft/client.ts index 43cbbdada3e..d016c7e136f 100644 --- a/apps/sim/lib/internal/agiloft/client.ts +++ b/apps/sim/lib/internal/agiloft/client.ts @@ -33,7 +33,7 @@ export async function resolveAgiloftInstance( signal?.throwIfAborted() const validation = await validateUrlWithDNS(instanceUrl, 'instanceUrl', 'configuredEndpoint') signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new Error(validation.error || 'Invalid Agiloft instance URL') } return validation.resolvedIP diff --git a/apps/sim/lib/internal/brex/client.ts b/apps/sim/lib/internal/brex/client.ts index 0877bf5ded6..82f1b0fdca9 100644 --- a/apps/sim/lib/internal/brex/client.ts +++ b/apps/sim/lib/internal/brex/client.ts @@ -73,7 +73,7 @@ export class BrexReceiptClient { this.signal?.throwIfAborted() const validation = await validateUrlWithDNS(uri, 'uri', 'contentFetch') this.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new BrexReceiptError('Brex returned an invalid upload URL', 502) } const response = await secureFetchWithPinnedIP(uri, validation.resolvedIP, { diff --git a/apps/sim/lib/internal/buffer/operations.ts b/apps/sim/lib/internal/buffer/operations.ts index a6ca65c13e2..1bce11a0b6b 100644 --- a/apps/sim/lib/internal/buffer/operations.ts +++ b/apps/sim/lib/internal/buffer/operations.ts @@ -71,12 +71,14 @@ async function resolveMediaKind(args: { const extensionKind = mediaKindFromExtension(pathOrName) if (extensionKind) return extensionKind + // `fileUrl` is minted by `resolveFileInputToUrl` against Sim's own storage, + // which on a self-hosted deployment legitimately sits on a private address. try { - const validation = await validateUrlWithDNS(fileUrl, 'media', 'contentFetch') + const validation = await validateUrlWithDNS(fileUrl, 'media', 'configuredEndpoint') context.signal?.throwIfAborted() - if (validation.isValid && validation.resolvedIP) { + if (validation.isValid) { const probe = await secureFetchWithPinnedIP(fileUrl, validation.resolvedIP, { - profile: 'contentFetch', + profile: 'configuredEndpoint', method: 'HEAD', timeout: MEDIA_PROBE_TIMEOUT_MS, signal: context.signal, diff --git a/apps/sim/lib/internal/cursor/operations.ts b/apps/sim/lib/internal/cursor/operations.ts index 86d0b8cff1c..3365cd7969d 100644 --- a/apps/sim/lib/internal/cursor/operations.ts +++ b/apps/sim/lib/internal/cursor/operations.ts @@ -67,7 +67,7 @@ export async function downloadCursorArtifact( const validation = await validateUrlWithDNS(downloadUrl, 'downloadUrl', 'contentFetch') context.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new CursorOperationError(validation.error || 'Invalid download URL', 400) } const downloadResponse = await secureFetchWithPinnedIP(downloadUrl, validation.resolvedIP, { diff --git a/apps/sim/lib/internal/extend/client.ts b/apps/sim/lib/internal/extend/client.ts index 0b26f0eb73e..c1ee29b1499 100644 --- a/apps/sim/lib/internal/extend/client.ts +++ b/apps/sim/lib/internal/extend/client.ts @@ -27,7 +27,7 @@ export async function submitExtendParse( 'configuredEndpoint' ) signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new ExtendOperationError(502, { success: false, error: 'Failed to reach Extend API' }) } diff --git a/apps/sim/lib/internal/github/operations.ts b/apps/sim/lib/internal/github/operations.ts index 7759ea395e4..b30e3941b9a 100644 --- a/apps/sim/lib/internal/github/operations.ts +++ b/apps/sim/lib/internal/github/operations.ts @@ -321,7 +321,7 @@ async function fetchChangedFileContent( try { const validation = await validateUrlWithDNS(file.raw_url, 'rawUrl', 'contentFetch') context.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) return undefined + if (!validation.isValid) return undefined const response = await secureFetchWithPinnedIP(file.raw_url, validation.resolvedIP, { profile: 'contentFetch', headers: { @@ -359,7 +359,7 @@ export async function getGitHubLatestCommit( const commitUrl = `https://api.github.com/repos/${owner}/${repo}/commits/${revision}` const validation = await validateUrlWithDNS(commitUrl, 'commitUrl', 'configuredEndpoint') context.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new GitHubOperationError(validation.error || 'Invalid GitHub commit URL', 400) } diff --git a/apps/sim/lib/internal/google-slides/operations.ts b/apps/sim/lib/internal/google-slides/operations.ts index 044241a9cde..4f8041bd9ad 100644 --- a/apps/sim/lib/internal/google-slides/operations.ts +++ b/apps/sim/lib/internal/google-slides/operations.ts @@ -48,7 +48,7 @@ export async function exportGoogleSlidesPresentation( 'configuredEndpoint' ) context.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new GoogleSlidesOperationError( validation.error || 'Invalid Google Slides export URL', 400 diff --git a/apps/sim/lib/internal/google-vault/operations.ts b/apps/sim/lib/internal/google-vault/operations.ts index e74a9cc1048..87683ae4b2b 100644 --- a/apps/sim/lib/internal/google-vault/operations.ts +++ b/apps/sim/lib/internal/google-vault/operations.ts @@ -44,7 +44,7 @@ export async function downloadGoogleVaultExportFile( const downloadUrl = `https://storage.googleapis.com/storage/v1/b/${bucket}/o/${object}?alt=media` const validation = await validateUrlWithDNS(downloadUrl, 'downloadUrl', 'configuredEndpoint') context.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new GoogleVaultOperationError( enhanceGoogleVaultError(validation.error || 'Invalid URL'), 400 diff --git a/apps/sim/lib/internal/grafana/client.ts b/apps/sim/lib/internal/grafana/client.ts index c8b40350c7d..33a52364dab 100644 --- a/apps/sim/lib/internal/grafana/client.ts +++ b/apps/sim/lib/internal/grafana/client.ts @@ -30,7 +30,7 @@ export class GrafanaClient { const url = `${this.baseUrl}${path}` const validation = await validateUrlWithDNS(url, 'baseUrl', 'configuredEndpoint') this.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { return { success: false, error: `Invalid Grafana baseUrl: ${validation.error}` } } diff --git a/apps/sim/lib/internal/image/fetch.ts b/apps/sim/lib/internal/image/fetch.ts index 5691a756a83..40d1786a315 100644 --- a/apps/sim/lib/internal/image/fetch.ts +++ b/apps/sim/lib/internal/image/fetch.ts @@ -33,7 +33,7 @@ export async function fetchRemoteImage( ): Promise { signal?.throwIfAborted() const validation = await validateUrlWithDNS(imageUrl, 'imageUrl', 'contentFetch') - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new RemoteImageFetchError(validation.error || 'Invalid image URL', 403) } diff --git a/apps/sim/lib/internal/image/operations.ts b/apps/sim/lib/internal/image/operations.ts index c09d1ffba57..8b2ee45cbac 100644 --- a/apps/sim/lib/internal/image/operations.ts +++ b/apps/sim/lib/internal/image/operations.ts @@ -377,7 +377,7 @@ async function bufferFromImageUrl( } const urlValidation = await validateUrlWithDNS(url, 'imageUrl', 'contentFetch') - if (!urlValidation.isValid || !urlValidation.resolvedIP) { + if (!urlValidation.isValid) { throw new Error(urlValidation.error || 'Generated image URL failed validation') } diff --git a/apps/sim/lib/internal/jupyter/client.ts b/apps/sim/lib/internal/jupyter/client.ts index e1a10b0cb8d..3e1b3acc3cb 100644 --- a/apps/sim/lib/internal/jupyter/client.ts +++ b/apps/sim/lib/internal/jupyter/client.ts @@ -45,7 +45,7 @@ export async function requestJupyterApi( const urlValidation = await validateUrlWithDNS(url, 'serverUrl', 'selfHostedService') signal?.throwIfAborted() - if (!urlValidation.isValid || !urlValidation.resolvedIP) { + if (!urlValidation.isValid) { throw new InvalidJupyterTargetError(`Invalid Jupyter serverUrl: ${urlValidation.error}`) } diff --git a/apps/sim/lib/internal/linq/client.ts b/apps/sim/lib/internal/linq/client.ts index a772f4bbe2f..7ab0fd5e45f 100644 --- a/apps/sim/lib/internal/linq/client.ts +++ b/apps/sim/lib/internal/linq/client.ts @@ -88,7 +88,7 @@ export async function uploadLinqAttachmentBytes( signal?.throwIfAborted() const validation = await validateUrlWithDNS(registration.uploadUrl, 'uploadUrl', 'contentFetch') signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new LinqOperationError(validation.error || 'Invalid Linq upload URL', 400) } const response = await secureFetchWithPinnedIP(registration.uploadUrl, validation.resolvedIP, { diff --git a/apps/sim/lib/internal/microsoft-word/client.ts b/apps/sim/lib/internal/microsoft-word/client.ts index c6a9e4df282..28cf2ab189a 100644 --- a/apps/sim/lib/internal/microsoft-word/client.ts +++ b/apps/sim/lib/internal/microsoft-word/client.ts @@ -1,3 +1,4 @@ +import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -55,18 +56,16 @@ export class GraphRequestError extends Error { async function graphFetch( url: string, paramName: string, - options: Omit[2]>, 'profile'> + options: Omit[2]>, 'profile'>, + profile: EgressProfile = 'configuredEndpoint' ) { options.signal?.throwIfAborted() - const validation = await validateUrlWithDNS(url, paramName, 'configuredEndpoint') + const validation = await validateUrlWithDNS(url, paramName, profile) options.signal?.throwIfAborted() if (!validation.isValid) { throw new GraphRequestError(validation.error || `Invalid ${paramName}`, 400) } - return secureFetchWithPinnedIP(url, validation.resolvedIP as string, { - ...options, - profile: 'configuredEndpoint', - }) + return secureFetchWithPinnedIP(url, validation.resolvedIP, { ...options, profile }) } /** Reads a Graph error body and raises it as a {@link GraphRequestError}. */ @@ -305,7 +304,8 @@ const UPLOAD_FRAGMENT_BYTES = 10 * 1024 * 1024 * * The URL is preauthenticated and on another host; Graph documents that sending * `Authorization` here can itself fail the request with a 401, so no bearer - * token is attached. + * token is attached. It comes out of a Graph response rather than from + * configuration, so it is judged under the `contentFetch` provenance. * * @see https://learn.microsoft.com/en-us/graph/api/driveitem-createuploadsession */ @@ -320,15 +320,22 @@ async function uploadSessionBytes( const end = Math.min(start + UPLOAD_FRAGMENT_BYTES, total) - 1 const fragment = content.subarray(start, end + 1) - const response = await graphFetch(uploadUrl, 'documentUploadUrl', { - method: 'PUT', - headers: { - 'Content-Length': String(fragment.length), - 'Content-Range': `bytes ${start}-${end}/${total}`, + const response = await graphFetch( + uploadUrl, + 'documentUploadUrl', + { + method: 'PUT', + headers: { + 'Content-Length': String(fragment.length), + 'Content-Range': `bytes ${start}-${end}/${total}`, + }, + body: fragment, + signal, }, - body: fragment, - signal, - }) + // The upload URL is named by a Graph response rather than configured, so + // it is judged as content: preauthenticated, public, and https-only. + 'contentFetch' + ) if (response.status === 412 || response.status === 409) { throw documentChangedError() diff --git a/apps/sim/lib/internal/mistral/client.ts b/apps/sim/lib/internal/mistral/client.ts index 3394880cd7f..8ba528144ad 100644 --- a/apps/sim/lib/internal/mistral/client.ts +++ b/apps/sim/lib/internal/mistral/client.ts @@ -23,7 +23,7 @@ export async function submitMistralOcr( 'configuredEndpoint' ) signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new MistralOperationError(502, { success: false, error: 'Failed to reach Mistral API', diff --git a/apps/sim/lib/internal/onedrive/operations.ts b/apps/sim/lib/internal/onedrive/operations.ts index f8792ef4df6..c28f6002ab6 100644 --- a/apps/sim/lib/internal/onedrive/operations.ts +++ b/apps/sim/lib/internal/onedrive/operations.ts @@ -429,7 +429,7 @@ async function fetchGraph( ) { const validation = await validateUrlWithDNS(url, label, 'contentFetch') signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new OneDriveOperationError(validation.error || `Invalid ${label}`, 400) } return secureFetchWithPinnedIP(url, validation.resolvedIP, { diff --git a/apps/sim/lib/internal/onepassword/client.ts b/apps/sim/lib/internal/onepassword/client.ts index a13019ad7d8..1903bc34eec 100644 --- a/apps/sim/lib/internal/onepassword/client.ts +++ b/apps/sim/lib/internal/onepassword/client.ts @@ -280,7 +280,7 @@ export async function validateConnectServerUrl( ) signal?.throwIfAborted() if (!validation.isValid) { - throw new Error(validation.error ?? '1Password server URL is not reachable') + throw new Error(validation.error) } return validation.resolvedIP } diff --git a/apps/sim/lib/internal/pipedrive/client.ts b/apps/sim/lib/internal/pipedrive/client.ts index 8caad8443d7..0fb90aaf4f4 100644 --- a/apps/sim/lib/internal/pipedrive/client.ts +++ b/apps/sim/lib/internal/pipedrive/client.ts @@ -50,7 +50,7 @@ export async function listPipedriveFiles( if (input.start) url.searchParams.set('start', input.start) const validation = await validateUrlWithDNS(url.toString(), 'apiUrl', 'configuredEndpoint') signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new PipedriveOperationError(validation.error || 'Invalid Pipedrive API URL', 400) } const response = await secureFetchWithPinnedIP(url.toString(), validation.resolvedIP, { @@ -95,7 +95,7 @@ export async function downloadPipedriveFile( signal?.throwIfAborted() const validation = await validateUrlWithDNS(fileUrl, 'fileUrl', 'contentFetch') signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) return null + if (!validation.isValid) return null const authHeaders: Record = input.authStyle === 'x-api-token' ? { 'x-api-token': input.accessToken } diff --git a/apps/sim/lib/internal/pulse/client.ts b/apps/sim/lib/internal/pulse/client.ts index 85f8d299ef7..5518b945a2e 100644 --- a/apps/sim/lib/internal/pulse/client.ts +++ b/apps/sim/lib/internal/pulse/client.ts @@ -21,7 +21,7 @@ export async function submitPulseParse( signal?.throwIfAborted() const validation = await validateUrlWithDNS(PULSE_ENDPOINT, 'Pulse API URL', 'configuredEndpoint') signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new PulseOperationError(502, { success: false, error: 'Failed to reach Pulse API' }) } diff --git a/apps/sim/lib/internal/reducto/client.ts b/apps/sim/lib/internal/reducto/client.ts index bf324b903bf..8fd209b862f 100644 --- a/apps/sim/lib/internal/reducto/client.ts +++ b/apps/sim/lib/internal/reducto/client.ts @@ -25,7 +25,7 @@ export async function submitReductoParse( 'configuredEndpoint' ) signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new ReductoOperationError(502, { success: false, error: 'Failed to reach Reducto API', diff --git a/apps/sim/lib/internal/sharepoint/client.ts b/apps/sim/lib/internal/sharepoint/client.ts index fc2016c6902..fee7d105641 100644 --- a/apps/sim/lib/internal/sharepoint/client.ts +++ b/apps/sim/lib/internal/sharepoint/client.ts @@ -103,7 +103,7 @@ export class SharePointClient { this.signal?.throwIfAborted() const validation = await validateUrlWithDNS(url, paramName, 'configuredEndpoint') this.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new SharePointGraphError(validation.error || `Invalid ${paramName}`, 400) } return secureFetchWithPinnedIP(url, validation.resolvedIP, { diff --git a/apps/sim/lib/internal/twilio-voice/operations.ts b/apps/sim/lib/internal/twilio-voice/operations.ts index 827808bd7aa..7acc77c44e8 100644 --- a/apps/sim/lib/internal/twilio-voice/operations.ts +++ b/apps/sim/lib/internal/twilio-voice/operations.ts @@ -52,7 +52,7 @@ async function fetchPinned( ) { const validation = await validateUrlWithDNS(url, label, 'configuredEndpoint') context.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new TwilioVoiceOperationError(validation.error || `Invalid ${label}`, 400) } return secureFetchWithPinnedIP(url, validation.resolvedIP, { diff --git a/apps/sim/lib/internal/typeform/operations.ts b/apps/sim/lib/internal/typeform/operations.ts index 9f5a7b24a19..0d5ada8984d 100644 --- a/apps/sim/lib/internal/typeform/operations.ts +++ b/apps/sim/lib/internal/typeform/operations.ts @@ -46,7 +46,7 @@ export async function downloadTypeformFile( const fileUrl = buildTypeformFileUrl(input) const validation = await validateUrlWithDNS(fileUrl, 'typeformFileUrl', 'configuredEndpoint') context.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new TypeformOperationError(validation.error || 'Invalid Typeform file URL', 400) } const response = await secureFetchWithPinnedIP(fileUrl, validation.resolvedIP, { diff --git a/apps/sim/lib/internal/vision/client.ts b/apps/sim/lib/internal/vision/client.ts index 6c0b79088a2..e3cdc7703bb 100644 --- a/apps/sim/lib/internal/vision/client.ts +++ b/apps/sim/lib/internal/vision/client.ts @@ -1,6 +1,7 @@ import { GoogleGenAI } from '@google/genai' import { createLogger } from '@sim/logger' import { isRecordLike } from '@sim/utils/object' +import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { MAX_JSON_API_RESPONSE_BYTES, secureFetchWithPinnedIP, @@ -24,6 +25,7 @@ export interface VisionClientInput { model: string prompt: string remoteImageResolvedIP?: string + remoteImageProfile?: EgressProfile } export interface VisionAnalysisResult { @@ -94,7 +96,7 @@ async function fetchGeminiImage(input: VisionClientInput, signal?: AbortSignal): } const response = await secureFetchWithPinnedIP(input.imageSource, input.remoteImageResolvedIP, { - profile: 'contentFetch', + profile: input.remoteImageProfile ?? 'contentFetch', method: 'GET', maxResponseBytes: MAX_BUFFERED_TRANSFER_BYTES, signal, diff --git a/apps/sim/lib/internal/vision/operations.test.ts b/apps/sim/lib/internal/vision/operations.test.ts index 1be0b594029..f9a15a0f0c0 100644 --- a/apps/sim/lib/internal/vision/operations.test.ts +++ b/apps/sim/lib/internal/vision/operations.test.ts @@ -216,10 +216,13 @@ describe('Vision operations', () => { expect(mocks.isModelSafeWorkspaceFileKey).toHaveBeenCalledWith( 'workspace/workspace-1/image.png' ) + // A resolved internal file URL is a presigned URL against Sim's own + // storage, which on a self-hosted deployment legitimately sits on a private + // address — so it is judged as a configured endpoint, not as content. expect(mocks.validateUrlWithDNS).toHaveBeenCalledWith( 'https://storage.example.com/image.png', 'imageUrl', - 'contentFetch' + 'configuredEndpoint' ) expect(mocks.analyzeVision).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/apps/sim/lib/internal/vision/operations.ts b/apps/sim/lib/internal/vision/operations.ts index de2b744be3c..3736d321d03 100644 --- a/apps/sim/lib/internal/vision/operations.ts +++ b/apps/sim/lib/internal/vision/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' import { analyzeVision, type VisionAnalysisResult } from '@/lib/internal/vision/client' @@ -35,6 +36,7 @@ interface ResolvedImage { source: string contentType?: string resolvedIP?: string + profile?: EgressProfile } function fail(message: string, status: number, body?: Record): never { @@ -84,7 +86,8 @@ async function resolveUrlImage( if (source.startsWith('/') && !isInternalFileUrl(source)) { fail('Invalid file path. Only uploaded files are supported for internal paths.', 400) } - if (isInternalFileUrl(source)) { + const internal = isInternalFileUrl(source) + if (internal) { context.signal?.throwIfAborted() const resolution = await resolveInternalFileUrl( source, @@ -100,8 +103,13 @@ async function resolveUrlImage( } } + // A caller-supplied image URL is content; a resolved internal one is a + // presigned URL against Sim's own storage, which on a self-hosted deployment + // legitimately sits on a private address. + const profile: EgressProfile = internal ? 'configuredEndpoint' : 'contentFetch' + context.signal?.throwIfAborted() - const validation = await validateUrlWithDNS(source, 'imageUrl', 'contentFetch') + const validation = await validateUrlWithDNS(source, 'imageUrl', profile) context.signal?.throwIfAborted() if (!validation.isValid) { fail(validation.error || 'Invalid image URL', 400, { @@ -109,7 +117,7 @@ async function resolveUrlImage( error: validation.error, }) } - return { source, resolvedIP: validation.resolvedIP } + return { source, resolvedIP: validation.resolvedIP, profile } } export async function executeVisionOperation( @@ -137,6 +145,7 @@ export async function executeVisionOperation( model: input.model, prompt: input.prompt || DEFAULT_PROMPT, remoteImageResolvedIP: image.resolvedIP, + remoteImageProfile: image.profile, }, context.signal ) diff --git a/apps/sim/lib/internal/zoom/operations.ts b/apps/sim/lib/internal/zoom/operations.ts index 1068f38c779..a384efc0dba 100644 --- a/apps/sim/lib/internal/zoom/operations.ts +++ b/apps/sim/lib/internal/zoom/operations.ts @@ -69,7 +69,7 @@ export async function getZoomMeetingRecordings( const apiUrl = query.size > 0 ? `${baseUrl}?${query}` : baseUrl const validation = await validateUrlWithDNS(apiUrl, 'apiUrl', 'configuredEndpoint') context.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new ZoomOperationError(validation.error || 'Invalid Zoom API URL', 400) } diff --git a/apps/sim/lib/mcp/domain-check.test.ts b/apps/sim/lib/mcp/domain-check.test.ts index 307330f2380..a2108c7408e 100644 --- a/apps/sim/lib/mcp/domain-check.test.ts +++ b/apps/sim/lib/mcp/domain-check.test.ts @@ -18,9 +18,11 @@ vi.mock('@/executor/utils/reference-validation', () => ({ import { isMcpDomainAllowed, + MCP_EGRESS_PROFILE, McpDnsResolutionError, McpDomainNotAllowedError, McpSsrfError, + OAUTH_EGRESS_PROFILE, validateMcpDomain, validateMcpServerSsrf, } from './domain-check' @@ -475,10 +477,18 @@ describe('validateMcpServerSsrf', () => { }) it('pins public IP literals on hosted so redirects cannot escape', async () => { - await expect(validateMcpServerSsrf('http://93.184.216.34/mcp')).resolves.toBe('93.184.216.34') + await expect(validateMcpServerSsrf('https://93.184.216.34/mcp')).resolves.toBe( + '93.184.216.34' + ) expect(mockDnsLookup).not.toHaveBeenCalled() }) + it('refuses plain HTTP on hosted, where a credential would cross the wire in the clear', async () => { + await expect(validateMcpServerSsrf('http://93.184.216.34/mcp')).rejects.toThrow( + /must use https/ + ) + }) + it('still refuses loopback on hosted when a domain allowlist is configured', async () => { // The domain allowlist governs which domains may be used. It is not a // substitute for the address check, which it used to disable entirely. @@ -541,3 +551,31 @@ describe('validateMcpServerSsrf', () => { ) }) }) + +describe('the OAuth provenance', () => { + it('is contentFetch, so a hop the metadata names inherits nothing from the server', () => { + expect(OAUTH_EGRESS_PROFILE).toBe('contentFetch') + expect(MCP_EGRESS_PROFILE).toBe('selfHostedService') + }) + + it('refuses loopback that the configured-server provenance reaches', async () => { + mockDnsLookup.mockResolvedValue([{ address: '127.0.0.1', family: 4 }]) + await expect(validateMcpServerSsrf('http://localhost:3000/mcp')).resolves.toBe('127.0.0.1') + await expect( + validateMcpServerSsrf('http://localhost:3000/token', OAUTH_EGRESS_PROFILE) + ).rejects.toThrow(McpSsrfError) + }) + + it('ignores the operator allowlist that the configured-server provenance honors', async () => { + setEnvFlags({ egressAllowedIpRanges: '10.0.0.0/8' }) + try { + mockDnsLookup.mockResolvedValue([{ address: '10.0.0.9', family: 4 }]) + await expect(validateMcpServerSsrf('https://mcp.corp/mcp')).resolves.toBe('10.0.0.9') + await expect( + validateMcpServerSsrf('https://idp.corp/token', OAUTH_EGRESS_PROFILE) + ).rejects.toThrow(McpSsrfError) + } finally { + setEnvFlags({ egressAllowedIpRanges: undefined }) + } + }) +}) diff --git a/apps/sim/lib/mcp/domain-check.ts b/apps/sim/lib/mcp/domain-check.ts index cb10a27610f..8599e8ed6c6 100644 --- a/apps/sim/lib/mcp/domain-check.ts +++ b/apps/sim/lib/mcp/domain-check.ts @@ -128,9 +128,9 @@ export function validateMcpDomain(url: string | undefined): void { * `contentFetch` instead, because those URLs come out of authorization-server * metadata rather than from whoever configured the server. * - * Returns null only when the hostname still contains an unresolved env-var - * reference. That URL is checked again after resolution, at which point it takes - * the normal path. + * Returns null when there is no URL yet, or when the hostname still contains an + * unresolved env-var reference. That URL is checked again after resolution, at + * which point it takes the normal path. * * @throws McpSsrfError when the policy refuses the destination * @throws McpDnsResolutionError when the hostname cannot be resolved @@ -145,7 +145,7 @@ export async function validateMcpServerSsrf( const validation = await validateUrlWithDNS(url, 'MCP server URL', profile) if (validation.isValid) return validation.resolvedIP - const error = validation.error ?? 'MCP server URL is not reachable' + const error = validation.error if (error.includes('could not be resolved')) { let hostname = url try { diff --git a/apps/sim/lib/mcp/pinned-fetch.ts b/apps/sim/lib/mcp/pinned-fetch.ts index 6f40027a1c2..b30e2ba18e8 100644 --- a/apps/sim/lib/mcp/pinned-fetch.ts +++ b/apps/sim/lib/mcp/pinned-fetch.ts @@ -31,9 +31,9 @@ export interface GuardedMcpFetch { * against the private/reserved blocklist (validate-at-connect, the LibreChat * pattern), and redirects are followed manually with per-hop validation — an * IP-literal redirect target (which bypasses any connect-time lookup) is checked - * explicitly, and custom headers are dropped on cross-origin hops. This replaces - * the previous single-IP pin, which no reference MCP client uses and which welded - * every request to one address with no fallback. + * explicitly, and custom headers are dropped on cross-origin hops. Keeping the + * full address set rather than welding every request to one address is what lets + * the connection fall back across a server's records. * * Runs HTTP/1.1: we do not opt into undici's experimental `allowH2`, whose h2 path stalls * with headers-but-no-body on reused POST sessions (nodejs/undici #2311, #3433, #4143) — diff --git a/apps/sim/lib/media/falai.ts b/apps/sim/lib/media/falai.ts index 8ab2f35077c..037fcf934bf 100644 --- a/apps/sim/lib/media/falai.ts +++ b/apps/sim/lib/media/falai.ts @@ -203,7 +203,7 @@ export async function downloadFalMedia( } const validation = await validateUrlWithDNS(url, 'mediaUrl', 'contentFetch') - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new Error(validation.error || 'Generated media URL failed validation') } diff --git a/apps/sim/lib/uploads/contexts/workspace/fetch-external-url.ts b/apps/sim/lib/uploads/contexts/workspace/fetch-external-url.ts index f411b8f684c..abd16b59c04 100644 --- a/apps/sim/lib/uploads/contexts/workspace/fetch-external-url.ts +++ b/apps/sim/lib/uploads/contexts/workspace/fetch-external-url.ts @@ -88,7 +88,7 @@ export async function fetchExternalUrlToWorkspace( } = options const urlValidation = await validateUrlWithDNS(url, 'fileUrl', 'contentFetch') - if (!urlValidation.isValid || !urlValidation.resolvedIP) { + if (!urlValidation.isValid) { throw new ExternalUrlValidationError(urlValidation.error || 'Invalid external URL') } diff --git a/apps/sim/providers/azure-anthropic/index.test.ts b/apps/sim/providers/azure-anthropic/index.test.ts index df16b8a612c..1de56f6ed15 100644 --- a/apps/sim/providers/azure-anthropic/index.test.ts +++ b/apps/sim/providers/azure-anthropic/index.test.ts @@ -122,17 +122,4 @@ describe('azureAnthropicProvider — SSRF pinning', () => { expect(mockCreatePinnedFetch).not.toHaveBeenCalled() expect(mockExecuteAnthropic).not.toHaveBeenCalled() }) - - it('fails closed when validation passes but yields no resolvable IP to pin', async () => { - mockValidate.mockResolvedValue({ isValid: true }) - - await expect( - azureAnthropicProvider.executeRequest( - request({ azureEndpoint: 'https://rebind.attacker.tld' }) - ) - ).rejects.toThrow('could not resolve a pinnable IP address') - - expect(mockCreatePinnedFetch).not.toHaveBeenCalled() - expect(mockExecuteAnthropic).not.toHaveBeenCalled() - }) }) diff --git a/apps/sim/providers/azure-anthropic/index.ts b/apps/sim/providers/azure-anthropic/index.ts index 0a39b6040c9..dff9f1be274 100644 --- a/apps/sim/providers/azure-anthropic/index.ts +++ b/apps/sim/providers/azure-anthropic/index.ts @@ -44,9 +44,6 @@ export const azureAnthropicProvider: ProviderConfig = { }) throw new Error(`Invalid Azure Anthropic endpoint: ${validation.error}`) } - if (!validation.resolvedIP) { - throw new Error('Invalid Azure Anthropic endpoint: could not resolve a pinnable IP address') - } pinnedIP = validation.resolvedIP pinnedFetch = createPinnedFetch(pinnedIP, { profile: 'configuredEndpoint' }) } diff --git a/apps/sim/providers/azure-openai/index.test.ts b/apps/sim/providers/azure-openai/index.test.ts index 0b31d9797c5..48c7431cb36 100644 --- a/apps/sim/providers/azure-openai/index.test.ts +++ b/apps/sim/providers/azure-openai/index.test.ts @@ -175,19 +175,6 @@ describe('azureOpenAIProvider — SSRF pinning', () => { expect(mockCreatePinnedFetch).not.toHaveBeenCalled() expect(mockExecuteResponses).not.toHaveBeenCalled() }) - - it('fails closed when validation passes but yields no resolvable IP to pin', async () => { - mockValidate.mockResolvedValue({ isValid: true }) - - await expect( - azureOpenAIProvider.executeRequest( - request({ azureEndpoint: 'https://rebind.attacker.tld' }) - ) - ).rejects.toThrow('could not resolve a pinnable IP address') - - expect(mockCreatePinnedFetch).not.toHaveBeenCalled() - expect(mockExecuteResponses).not.toHaveBeenCalled() - }) }) describe('Chat Completions path', () => { diff --git a/apps/sim/providers/azure-openai/index.ts b/apps/sim/providers/azure-openai/index.ts index de3f3a612ec..72251763d99 100644 --- a/apps/sim/providers/azure-openai/index.ts +++ b/apps/sim/providers/azure-openai/index.ts @@ -686,9 +686,6 @@ export const azureOpenAIProvider: ProviderConfig = { }) throw new Error(`Invalid Azure OpenAI endpoint: ${validation.error}`) } - if (!validation.resolvedIP) { - throw new Error('Invalid Azure OpenAI endpoint: could not resolve a pinnable IP address') - } pinnedFetch = createPinnedFetch(validation.resolvedIP, { profile: 'configuredEndpoint' }) } diff --git a/apps/sim/providers/vllm/index.test.ts b/apps/sim/providers/vllm/index.test.ts index 3b6ecd20c2d..1057bb89e5e 100644 --- a/apps/sim/providers/vllm/index.test.ts +++ b/apps/sim/providers/vllm/index.test.ts @@ -234,22 +234,6 @@ describe('vllmProvider', () => { expect(openAIArgs).toHaveLength(0) expect(mockCreate).not.toHaveBeenCalled() }) - - it('rejects a validated endpoint that did not resolve to a pinnable IP', async () => { - mockValidateUrlWithDNS.mockResolvedValueOnce({ isValid: true }) - - await expect( - vllmProvider.executeRequest({ - model: 'vllm/llama-3', - messages: [{ role: 'user', content: 'hi' }], - azureEndpoint: 'https://my-vllm.example.com', - }) - ).rejects.toThrow('could not resolve a pinnable IP address') - - expect(mockCreatePinnedFetch).not.toHaveBeenCalled() - expect(openAIArgs).toHaveLength(0) - expect(mockCreate).not.toHaveBeenCalled() - }) }) it('builds a chat payload with the vllm/ prefix stripped and messages assembled in order', async () => { diff --git a/apps/sim/providers/vllm/index.ts b/apps/sim/providers/vllm/index.ts index 5e15b24e64b..3bad1cba441 100644 --- a/apps/sim/providers/vllm/index.ts +++ b/apps/sim/providers/vllm/index.ts @@ -119,7 +119,7 @@ export const vllmProvider: ProviderConfig = { * rebinding. The operator-configured `VLLM_BASE_URL` is trusted and left * unvalidated, mirroring the Azure providers. * - * The `configuredEndpoint` profile is what makes a self-hosted vLLM reachable + * The `selfHostedService` profile is what makes a self-hosted vLLM reachable * at all: over plain HTTP, which these deployments usually are, and at a * private address once the operator names it in the egress allowlist. * Anything they have not named stays blocked. @@ -139,9 +139,6 @@ export const vllmProvider: ProviderConfig = { }) throw new Error(`Invalid vLLM endpoint: ${validation.error}`) } - if (!validation.resolvedIP) { - throw new Error('Invalid vLLM endpoint: could not resolve a pinnable IP address') - } pinnedIP = validation.resolvedIP pinnedFetch = createPinnedFetch(pinnedIP, { profile: 'selfHostedService' }) } diff --git a/apps/sim/tools/bitbucket/utils.server.ts b/apps/sim/tools/bitbucket/utils.server.ts index 2178bd84a08..a82989533b2 100644 --- a/apps/sim/tools/bitbucket/utils.server.ts +++ b/apps/sim/tools/bitbucket/utils.server.ts @@ -76,7 +76,7 @@ export async function secureBitbucketRead( } = {} ): Promise { const validation = await validateUrlWithDNS(url, 'bitbucketUrl', 'configuredEndpoint') - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new Error(`Invalid Bitbucket URL: ${validation.error ?? 'DNS resolution failed'}`) } diff --git a/apps/sim/tools/github/utils.server.ts b/apps/sim/tools/github/utils.server.ts index 1ef56eaffec..f38dfb6cf50 100644 --- a/apps/sim/tools/github/utils.server.ts +++ b/apps/sim/tools/github/utils.server.ts @@ -64,7 +64,7 @@ export async function secureGitHubRequest( options: SecureGitHubRequestOptions ): Promise { const validation = await validateUrlWithDNS(url, 'githubUrl', 'configuredEndpoint') - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new Error(`Invalid GitHub URL: ${validation.error ?? 'DNS resolution failed'}`) } diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index 1df20786d5d..ba331e5c994 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -87,9 +87,12 @@ app: # EGRESS_ALLOWED_HOSTS / EGRESS_ALLOWED_IP_RANGES: destinations on your private network that # workflows may reach. Outbound requests to private, reserved, and loopback addresses are # blocked by default; naming a destination here permits it, allows plain HTTP to it, and lifts - # the blocked-port list for it. In-cluster services need the hostname form, e.g. - # "*.svc.cluster.local" or "vllm.ai.svc.cluster.local". Cloud metadata endpoints stay blocked - # however broad the list is. Pair with a NetworkPolicy that constrains what the pod can reach. + # the blocked-port list for it. In-cluster services can be named by hostname, e.g. + # "*.svc.cluster.local" or "vllm.ai.svc.cluster.local", or covered by the pod/service CIDR in + # EGRESS_ALLOWED_IP_RANGES. Cloud metadata endpoints stay blocked however broad the list is, + # and neither setting is honored for URLs harvested from content or for a proxy. Pair with a + # NetworkPolicy that constrains what the pod can reach — note networkPolicy.enabled grants + # broad egress on 443 only, so a target on another port also needs a networkPolicy.egress rule. EGRESS_ALLOWED_HOSTS: "" EGRESS_ALLOWED_IP_RANGES: "" # SOCKET_SERVER_URL: Auto-detected when realtime.enabled=true (uses internal service) diff --git a/packages/security/src/egress-hosted-posture.test.ts b/packages/security/src/egress-hosted-posture.test.ts deleted file mode 100644 index e12962f085c..00000000000 --- a/packages/security/src/egress-hosted-posture.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -/** - * @vitest-environment node - * - * Pins what the hosted platform permits. Everything here is a value comparison - * against a policy built for isHosted=true, so it needs no module mocking. - */ - -import { createEgressPolicy, evaluateAddress } from '@sim/security/egress' -import { describe, expect, it } from 'vitest' - -/** Every profile collapses to this on hosted: no allowlist, no loopback, no private. */ -const publicApi = createEgressPolicy({ insecureHttp: 'whenVouched' }) -/** ...except the self-hosted-service profile, which still tolerates plain HTTP. */ -const selfHostedService = createEgressPolicy({ insecureHttp: 'always' }) - -function decide(policy: Parameters[2], href: string, address: string) { - return evaluateAddress(new URL(href), address, policy) -} - -describe('hosted platform', () => { - it.each([ - ['https://x.example/', '10.0.0.5', 'RFC1918'], - ['https://x.example/', '127.0.0.1', 'loopback'], - ['https://x.example/', '169.254.169.254', 'metadata'], - ['https://x.example/', '192.168.1.1', 'RFC1918'], - ])('refuses %s resolving to %s — %s', (href, address) => { - expect(decide(publicApi, href, address).allowed).toBe(false) - expect(decide(selfHostedService, href, address).allowed).toBe(false) - }) - - it('refuses a service port on a public host', () => { - expect(decide(publicApi, 'https://x.example:5432/', '93.184.216.34').allowed).toBe(false) - expect(decide(selfHostedService, 'http://x.example:5432/', '93.184.216.34').allowed).toBe(false) - }) - - it('permits ordinary public HTTPS', () => { - expect(decide(publicApi, 'https://x.example/', '93.184.216.34').allowed).toBe(true) - }) - - it('keeps plain HTTP available only to the self-hosted-service profile', () => { - expect(decide(publicApi, 'http://x.example/', '93.184.216.34').allowed).toBe(false) - expect(decide(selfHostedService, 'http://x.example/', '93.184.216.34').allowed).toBe(true) - }) -}) diff --git a/packages/security/src/egress.test.ts b/packages/security/src/egress.test.ts index aeab8bc1ac7..2c9ef9b29bb 100644 --- a/packages/security/src/egress.test.ts +++ b/packages/security/src/egress.test.ts @@ -4,6 +4,8 @@ import { type EgressPolicy, evaluateAddress, evaluateUrl, + isLiftableByVouching, + policyDefersToAddress, STRICT_EGRESS_POLICY, } from './egress' @@ -388,3 +390,134 @@ describe('must not over-block', () => { ) }) }) + +describe('IPv6 forms that carry an IPv4 destination', () => { + /** Vouches for a name, which is what would otherwise carry an address past the check. */ + const vouchesByName = createEgressPolicy({ + allowedHosts: 'internal.corp', + insecureHttp: 'whenVouched', + }) + + it.each([ + ['::ffff:169.254.169.254', 'IPv4-mapped'], + ['::a9fe:a9fe', 'deprecated IPv4-compatible'], + ['64:ff9b::a9fe:a9fe', 'RFC 6052 well-known NAT64'], + ['::ffff:0:a9fe:a9fe', 'RFC 6145 IPv4-translated'], + ])('reads %s as the metadata endpoint it carries — %s', (address) => { + expect(reason(vouchesByName, 'https://internal.corp/', address)).toBe('address-metadata') + }) + + it.each([ + ['64:ff9b:1::a9fe:a9fe', 'metadata'], + ['64:ff9b:1::7f00:1', 'loopback'], + ])( + 'refuses the RFC 8215 local-use NAT64 wrapper around %s, whose offset is network-specific', + (address) => { + expect(reason(hosted, 'https://example.com/', address)).toBe('address-blocked') + expect(reason(vouchesByName, 'https://internal.corp/', address)).toBe('address-blocked') + } + ) + + it('refuses even a public IPv4 carried in a NAT64 prefix unless the policy vouches', () => { + // `ipaddr.js` classifies the whole RFC 6052 prefix as non-unicast, so an + // unvouched destination never reaches it. Fail-closed is the right default: + // a DNS64 answer is a translated route, not the destination itself. + expect(reason(hosted, 'https://example.com/', '64:ff9b::5db8:d822')).toBe('address-blocked') + expect(decide(vouchesByName, 'https://internal.corp/', '64:ff9b::5db8:d822').allowed).toBe(true) + }) +}) + +describe('an unparseable address is refused even for a vouched destination', () => { + it('does not let a hostname allowlist entry carry it past the check', () => { + const policy = createEgressPolicy({ allowedHosts: 'internal.corp' }) + expect(reason(policy, 'https://internal.corp/', 'not-an-ip')).toBe('address-blocked') + }) +}) + +describe('the loopback carve-out stops short of the port denylist', () => { + const loopbackAllowed = createEgressPolicy({ allowLoopback: true, insecureHttp: 'whenVouched' }) + + it.each([ + ['http://localhost:5432/', 'Postgres'], + ['http://127.0.0.1:6379/', 'Redis'], + ['http://localhost:22/', 'SSH'], + ])('refuses %s — %s is where Sim listens, and nobody asked for it', (href) => { + expect(reason(loopbackAllowed, href)).toBe('port-denied') + }) + + it('still permits plain HTTP to loopback on an ordinary port', () => { + expect(decide(loopbackAllowed, 'http://localhost:11434/').allowed).toBe(true) + }) + + it('lifts the port denylist once an operator names the destination', () => { + const named = createEgressPolicy({ allowedHosts: 'localhost', insecureHttp: 'whenVouched' }) + expect(decide(named, 'http://localhost:5432/').allowed).toBe(true) + }) +}) + +describe('a name that says it is loopback is refused without a lookup', () => { + it('refuses localhost before DNS when the policy does not permit loopback', () => { + expect(reason(hosted, 'https://localhost/x')).toBe('address-loopback') + }) + + it('still permits it when the policy grants the carve-out', () => { + const loopbackAllowed = createEgressPolicy({ allowLoopback: true }) + expect(decide(loopbackAllowed, 'https://localhost/x').allowed).toBe(true) + }) + + it('still permits it when an operator named it', () => { + const named = createEgressPolicy({ allowedHosts: 'localhost' }) + expect(decide(named, 'https://localhost/x').allowed).toBe(true) + }) +}) + +describe('policyDefersToAddress', () => { + it('is false for a policy whose only softenings are decided from the hostname', () => { + expect( + policyDefersToAddress(createEgressPolicy({ allowLoopback: true, allowedHosts: 'a.corp' })) + ).toBe(false) + }) + + it.each([ + [createEgressPolicy({ allowedRanges: '10.0.0.0/8' }), 'a range entry'], + [createEgressPolicy({ allowPrivate: true }), 'the blanket private grant'], + ])('is true for %#: %s', (policy) => { + expect(policyDefersToAddress(policy)).toBe(true) + }) +}) + +describe('isLiftableByVouching', () => { + it.each([['scheme-not-permitted'], ['address-metadata']] as const)( + 'reports %s as final', + (reasonCode) => { + expect(isLiftableByVouching(reasonCode)).toBe(false) + } + ) + + it.each([ + ['insecure-scheme'], + ['port-denied'], + ['address-loopback'], + ['address-blocked'], + ] as const)('reports %s as liftable', (reasonCode) => { + expect(isLiftableByVouching(reasonCode)).toBe(true) + }) +}) + +describe('createEgressPolicy validates wildcard entries too', () => { + it.each([ + ['*.foo.com/x', /expected a hostname/], + ['*.a*.com', /leading/], + ['*..com', /non-empty/], + ['*.', /non-empty/], + ['.example.com', /non-empty/], + ])('rejects %s', (entry, message) => { + expect(() => createEgressPolicy({ allowedHosts: [entry] })).toThrow(message) + }) + + it('matches a wildcard at any depth but never the bare apex', () => { + const policy = createEgressPolicy({ allowedHosts: '*.svc.cluster.local' }) + expect(decide(policy, 'https://vllm.ai.svc.cluster.local/', '10.4.2.9').allowed).toBe(true) + expect(decide(policy, 'https://svc.cluster.local/', '10.4.2.9').allowed).toBe(false) + }) +}) diff --git a/packages/security/src/egress.ts b/packages/security/src/egress.ts index 6857a64d330..4a883cabd54 100644 --- a/packages/security/src/egress.ts +++ b/packages/security/src/egress.ts @@ -31,7 +31,6 @@ import { isIpLiteral, isLoopbackIp, isPrivateIp } from './ssrf' type IpAddress = ipaddr.IPv4 | ipaddr.IPv6 /** Schemes that may ever carry an outbound request. */ -export type EgressScheme = 'http:' | 'https:' /** When a policy tolerates plain HTTP. */ export type InsecureHttpPolicy = 'never' | 'whenVouched' | 'always' @@ -170,26 +169,28 @@ function splitEntries(value: string | readonly string[] | undefined): string[] { function parseHostPattern(entry: string, sourceName: string): HostPattern { const value = unwrapIpv6Brackets(entry.toLowerCase()) - if (value.startsWith('*.')) { - const suffix = value.slice(1) - if (suffix.length < 2 || !suffix.includes('.', 1)) { - throw new Error( - `Invalid ${sourceName} entry "${entry}": a wildcard must cover at least two labels, e.g. "*.example.com"` - ) - } - return { value: suffix, wildcard: true } - } - if (value.includes('*')) { + const wildcard = value.startsWith('*.') + const host = wildcard ? value.slice(2) : value + + if (host.includes('*')) { throw new Error( `Invalid ${sourceName} entry "${entry}": a wildcard is only supported as a leading "*." label` ) } - if (value.includes('/') || /\s/.test(value)) { + if (host.includes('/') || /\s/.test(host)) { throw new Error( `Invalid ${sourceName} entry "${entry}": expected a hostname, not a URL or CIDR` ) } - return { value, wildcard: false } + if (host.length === 0 || host.split('.').some((label) => label.length === 0)) { + throw new Error(`Invalid ${sourceName} entry "${entry}": every label must be non-empty`) + } + if (wildcard && !host.includes('.')) { + throw new Error( + `Invalid ${sourceName} entry "${entry}": a wildcard must cover at least two labels, e.g. "*.example.com"` + ) + } + return wildcard ? { value: `.${host}`, wildcard: true } : { value: host, wildcard: false } } function parseCidrRange(entry: string, sourceName: string): CidrRange { @@ -256,13 +257,7 @@ function matchesRangeAllowlist(address: string, policy: EgressPolicy): boolean { ) } -/** - * Canonical form of an address, folding every IPv4-in-IPv6 spelling down to the - * IPv4 it carries. `ipaddr.process` handles the IPv4-mapped form (`::ffff:x`) - * but leaves the deprecated IPv4-compatible one (`::a.b.c.d`, which the WHATWG - * URL parser normalizes to `::a9fe:a9fe`), so comparing without this misses the - * metadata endpoint written that way. - */ +/** The IPv4 carried in an address's last 32 bits. */ function embeddedIpv4(parts: readonly number[]): string { return ipaddr .fromByteArray([ @@ -274,9 +269,26 @@ function embeddedIpv4(parts: readonly number[]): string { .toString() } -/** RFC 6052 well-known NAT64 prefix, `64:ff9b::/96`. */ -const NAT64_WELL_KNOWN_PREFIX = [0x0064, 0xff9b, 0, 0, 0, 0] as const +/** + * IPv6 prefixes whose low 32 bits are the IPv4 destination: the RFC 6052 + * well-known NAT64 prefix `64:ff9b::/96`, and the RFC 6145 IPv4-translated + * prefix `::ffff:0:0:0/96`. + */ +const IPV4_EMBEDDING_PREFIXES: readonly (readonly number[])[] = [ + [0x0064, 0xff9b, 0, 0, 0, 0], + [0, 0, 0, 0, 0xffff, 0], +] +/** RFC 8215 local-use NAT64 prefix, `64:ff9b:1::/48`. */ +const NAT64_LOCAL_USE_PREFIX = [0x0064, 0xff9b, 0x0001] as const + +/** + * Canonical form of an address, folding every IPv4-in-IPv6 spelling down to the + * IPv4 it carries. `ipaddr.process` handles the IPv4-mapped form (`::ffff:x`) + * but leaves the deprecated IPv4-compatible one (`::a.b.c.d`, which the WHATWG + * URL parser normalizes to `::a9fe:a9fe`), so comparing without this misses the + * metadata endpoint written that way. + */ function canonicalAddress(address: string): string | null { const clean = unwrapIpv6Brackets(address) if (!ipaddr.isValid(clean)) return null @@ -285,10 +297,12 @@ function canonicalAddress(address: string): string | null { if (parsed.kind() === 'ipv6') { const parts = (parsed as ipaddr.IPv6).parts - // A DNS64 resolver hands back the IPv4 destination wrapped in the well-known - // NAT64 prefix. Left unfolded, `64:ff9b::a9fe:a9fe` does not read as the - // metadata endpoint it is, and a vouched destination would reach it. - if (NAT64_WELL_KNOWN_PREFIX.every((part, index) => parts[index] === part)) { + // A DNS64 resolver hands back the IPv4 destination wrapped in a translation + // prefix. Left unfolded, `64:ff9b::a9fe:a9fe` does not read as the metadata + // endpoint it is, and a vouched destination would reach it. + if ( + IPV4_EMBEDDING_PREFIXES.some((prefix) => prefix.every((part, index) => parts[index] === part)) + ) { return embeddedIpv4(parts) } @@ -313,11 +327,19 @@ function isMetadataAddress(address: string): boolean { } /** - * Whether the policy vouches for this destination. A hostname match alone is - * enough — the operator named that host, so wherever it points is their call. - * Otherwise a resolved address inside an allowlisted range vouches for it, which - * is why this cannot be decided before DNS for a hostname destination. + * Whether the address sits in the RFC 8215 local-use NAT64 prefix, where the + * embedded IPv4 destination sits at an offset chosen by the network operator + * and so cannot be read off the address. */ +function hidesItsIpv4Destination(address: string): boolean { + const clean = unwrapIpv6Brackets(address) + if (!ipaddr.isValid(clean)) return false + const parsed = ipaddr.process(clean) + if (parsed.kind() !== 'ipv6') return false + const { parts } = parsed as ipaddr.IPv6 + return NAT64_LOCAL_USE_PREFIX.every((part, index) => parts[index] === part) +} + /** * Whether the destination names itself as loopback — `localhost`, or a loopback * IP literal. @@ -332,34 +354,56 @@ function isLoopbackDestination(host: string): boolean { return isLoopbackHostname(clean) || isLoopbackIp(clean) } -function isVouched(url: URL, address: string | undefined, policy: EgressPolicy): boolean { - if (matchesHostAllowlist(url.hostname, policy)) return true +/** + * How a destination earned its reachability, or `null` if it has not. + * + * The two are not interchangeable. `allowlist` is an operator naming a + * destination, so it carries their judgement about what is safe there, down to + * the port. `loopback` is a carve-out this policy grants on its own to any + * self-hosted deployment, which is why it stops short of exposing the service + * ports Sim's own datastores listen on. + */ +type Vouch = 'allowlist' | 'loopback' | null + +/** + * How the policy vouches for this destination. A hostname match alone is enough + * — the operator named that host, so wherever it points is their call. + * Otherwise the loopback carve-out, the legacy blanket private grant, or a + * resolved address inside an allowlisted range vouches for it, which is why the + * last two cannot be decided before DNS for a hostname destination. + */ +function isVouched(url: URL, address: string | undefined, policy: EgressPolicy): Vouch { + if (matchesHostAllowlist(url.hostname, policy)) return 'allowlist' if (policy.allowLoopback && isLoopbackDestination(url.hostname)) { // Before DNS there is no address to judge; evaluateAddress rules later. - if (address === undefined) return true + if (address === undefined) return 'loopback' // The address must land on loopback too, so a resolver answering // `localhost` with a routable address cannot borrow the carve-out — but it // may still be vouched by an allowlist entry, so this falls through rather // than refusing outright. - if (isLoopbackIp(unwrapIpv6Brackets(address))) return true + if (isLoopbackIp(unwrapIpv6Brackets(address))) return 'loopback' } - if (address === undefined) return false - if (policy.allowPrivate && isPrivateIp(unwrapIpv6Brackets(address))) return true - return matchesRangeAllowlist(address, policy) + if (address === undefined) return null + if (policy.allowPrivate && isPrivateIp(unwrapIpv6Brackets(address))) return 'allowlist' + return matchesRangeAllowlist(address, policy) ? 'allowlist' : null } -function checkSchemeAndPort(url: URL, vouched: boolean, policy: EgressPolicy): EgressDecision { +function checkSchemeAndPort(url: URL, vouch: Vouch, policy: EgressPolicy): EgressDecision { if ( url.protocol === 'http:' && policy.insecureHttp !== 'always' && - !(vouched && policy.insecureHttp === 'whenVouched') + !(vouch !== null && policy.insecureHttp === 'whenVouched') ) { return deny('insecure-scheme', `plain http to ${url.hostname}`) } - if (!vouched && url.port) { + // Only an operator naming the destination lifts the port denylist. The + // loopback carve-out deliberately does not: it is granted without anyone + // asking for it, and loopback is exactly where Sim's own Postgres and Redis + // listen, so lifting it there would hand every workflow author a route in. + if (vouch !== 'allowlist' && url.port) { const port = Number.parseInt(url.port, 10) if (DENIED_PORTS.has(port)) { return deny('port-denied', `port ${port}`) @@ -370,8 +414,8 @@ function checkSchemeAndPort(url: URL, vouched: boolean, policy: EgressPolicy): E } /** Classifies one address, assuming the vouched decision has already been made. */ -function checkAddressClass(address: string, vouched: boolean): EgressDecision { - if (vouched) return ALLOWED +function checkAddressClass(address: string, vouch: Vouch): EgressDecision { + if (vouch !== null) return ALLOWED const clean = unwrapIpv6Brackets(address) if (isLoopbackIp(clean)) { @@ -390,7 +434,8 @@ function checkAddressClass(address: string, vouched: boolean): EgressDecision { * is not known yet. * * Neither verdict is the last word on a hostname. A refusal may be liftable once - * the address is known ({@link policyCanVouch}, {@link isLiftableByVouching}), + * the address is known ({@link policyDefersToAddress}, + * {@link isLiftableByVouching}), * and an approval covers only what needs no lookup — {@link evaluateAddress} is * authoritative and must run against every resolved address before connecting. */ @@ -404,39 +449,32 @@ export function evaluateUrl(url: URL, policy: EgressPolicy): EgressDecision { return evaluateAddress(url, host, policy) } - // Judged as if unvouched, because a hostname's address is not known yet. A - // policy that could still vouch for it once resolved must not treat this - // verdict as final — see {@link policyCanVouch}. - return checkSchemeAndPort(url, isVouched(url, undefined, policy), policy) -} + const vouch = isVouched(url, undefined, policy) -/** - * Whether this policy has any way to vouch for a destination it has not already - * accepted — an allowlist entry, or a loopback carve-out. - * - * A DNS-resolving caller uses this to decide whether a refusal from - * {@link evaluateUrl} on a hostname is final, or whether it must resolve and let - * {@link evaluateAddress} rule on the addresses. Without it the pre-DNS check - * would have to either refuse destinations the policy actually permits, or wave - * through ones it does not. - */ -export function policyCanVouch(policy: EgressPolicy): boolean { - return ( - policy.allowedHosts.length > 0 || - policy.allowedRanges.length > 0 || - policy.allowLoopback || - policy.allowPrivate - ) + // A name that says it is loopback needs no lookup to be refused. Deciding it + // here rather than after DNS is what keeps the synchronous validator — the one + // that checks a value as it is saved — from accepting `https://localhost/x` on + // a deployment where loopback is Sim's own process. + if (vouch === null && isLoopbackDestination(host)) { + return deny('address-loopback', host) + } + + // Otherwise judged as if unvouched, because a hostname's address is not known + // yet. A policy that could still vouch for it once resolved must not treat + // this verdict as final — see {@link policyDefersToAddress}. + return checkSchemeAndPort(url, vouch, policy) } /** - * Whether a refusal could be reversed specifically by learning the resolved - * address — an IP-range entry, or the legacy blanket private grant. + * Whether a refusal from {@link evaluateUrl} on a hostname could still be + * reversed by learning the resolved address — an IP-range entry, or the legacy + * blanket private grant. * - * Narrower than {@link policyCanVouch}: a hostname allowlist entry and the - * loopback carve-out are both decided from the hostname, so {@link evaluateUrl} - * has already applied them. A synchronous caller must use this rather than the - * broader predicate, or it defers everything and stops refusing anything. + * A DNS-resolving caller uses this to decide whether that refusal is final. It + * is deliberately narrow: a hostname allowlist entry and the loopback carve-out + * are both decided from the hostname, so {@link evaluateUrl} has already applied + * them, and treating those as reasons to resolve would defer every refusal and + * hand a lookup to a destination the policy has already turned down. */ export function policyDefersToAddress(policy: EgressPolicy): boolean { return policy.allowedRanges.length > 0 || policy.allowPrivate @@ -475,10 +513,16 @@ export function evaluateAddress(url: URL, address: string, policy: EgressPolicy) return deny('address-blocked', `${address} is not a valid address`) } - const vouched = isVouched(url, address, policy) + // Same reasoning: a local-use NAT64 address names an IPv4 destination this + // code cannot read, so it is refused rather than judged on the wrapper. + if (hidesItsIpv4Destination(address)) { + return deny('address-blocked', `${address} hides its IPv4 destination`) + } + + const vouch = isVouched(url, address, policy) - const shape = checkSchemeAndPort(url, vouched, policy) + const shape = checkSchemeAndPort(url, vouch, policy) if (!shape.allowed) return shape - return checkAddressClass(address, vouched) + return checkAddressClass(address, vouch) } diff --git a/scripts/check-egress-boundary.ts b/scripts/check-egress-boundary.ts index 26f0fdb2ab3..f48d59cb763 100644 --- a/scripts/check-egress-boundary.ts +++ b/scripts/check-egress-boundary.ts @@ -24,6 +24,7 @@ * * Usage: bun run scripts/check-egress-boundary.ts */ +import type { Dirent } from 'node:fs' import { readdirSync, readFileSync } from 'node:fs' import path from 'node:path' import ts from '@typescript/typescript6' @@ -32,11 +33,13 @@ const ROOT = path.resolve(import.meta.dir, '..') const SCAN_DIRS = [ 'apps/sim/app', - 'apps/sim/lib', - 'apps/sim/tools', + 'apps/sim/background', + 'apps/sim/blocks', 'apps/sim/connectors', 'apps/sim/executor', + 'apps/sim/lib', 'apps/sim/providers', + 'apps/sim/tools', 'apps/sim/triggers', ] @@ -75,7 +78,13 @@ const ALLOWED = new Set([ ]) function walk(dir: string, out: string[] = []): string[] { - for (const entry of readdirSync(dir, { withFileTypes: true })) { + let entries: Dirent[] + try { + entries = readdirSync(dir, { withFileTypes: true }) + } catch { + throw new Error(`check-egress-boundary: SCAN_DIRS entry "${dir}" does not exist`) + } + for (const entry of entries) { if (SKIP_DIRS.has(entry.name)) continue const full = path.join(dir, entry.name) if (entry.isDirectory()) walk(full, out) From d3003c756add0447fd6f27b65085cf3af0406b74 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 23:40:48 -0700 Subject: [PATCH 20/20] fix(egress): judge an IPv6 address by the IPv4 it carries, whatever the wrapper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A second audit pass found the transition schemes the first one missed. Every one of these is a real route to an IPv4 destination, and the guard was judging the wrapper instead: - RFC 3056 6to4 (`2002:a9fe:a9fe::`) and RFC 5214 ISATAP (`fe80::5efe:169.254.169.254`) now fold to the IPv4 they name, so they read as the metadata endpoints they are. - RFC 4380 Teredo carries two IPv4 addresses — the client's, obfuscated, and the relay's — so rather than pick one it is refused, alongside the rest of the reserved `::/64` block, which `ipaddr.js` calls plain unicast. `::5efe:7f00:1` reached loopback under the strict hosted policy before this. - A scope id (`fd00:ec2::254%eth0`) named an interface, not a destination, and made an address a different string from the one it is. - Address classification runs on the canonical form, so the folding above reaches the private/loopback verdict and not only the metadata comparison. A 6to4 wrapper around a public address is now correctly reachable. Also from the same pass: - An explicit allowlist grant outranks the loopback carve-out. Checking the carve-out first made the policy non-monotonic: an operator who named `127.0.0.1/32` got *less* than one who named nothing, because the carve-out does not lift the port denylist and short-circuited the range match. - `localhost.` and `*.localhost` are loopback names too (RFC 6761), and a trailing dot no longer defeats the host allowlist on either side. - An operator range naming a translation prefix now matches an address inside it; only the folded spelling was being compared. - A Unicode allowlist entry is refused with a message naming the punycode form, rather than being accepted and then matching nothing — a URL hostname is always the A-label. - `proxy` is exempt from the hosted plain-HTTP cap. Its scheme is fixed by the protocol rather than by trust, and capping it left `proxyUrl` with no reachable configuration on the hosted platform and two contradictory error messages. - The egress policies are built in `instrumentation-node`, so a malformed allowlist entry stops the process at boot naming the setting, which is what the docs say and what the lazy cache had stopped doing. - `sim-setup` validates both allowlists through the same parser at the prompt, and now asks for `EGRESS_ALLOWED_IP_RANGES` as well as the hosts. Docs: the MCP OAuth rule applies only to endpoints on a different origin than the configured server; an SSO OIDC discovery URL is a configured endpoint, not content; the HTTP downgrade refusal does not apply to the two provenances that expect plain HTTP; plain HTTP is capped on Sim Cloud; the NetworkPolicy note is conditional on `networkPolicy.enabled`. Tests: `ssrf-guarded-lookup.test.ts` passes a real profile instead of relying on the unrecognized-profile fallback; the hosted block regains its positive control and a service-port case; the OAuth block sets its own posture rather than inheriting the previous describe's. --- apps/docs/content/docs/agents/mcp.mdx | 2 +- .../docs/platform/self-hosting/security.mdx | 12 +- .../platform/self-hosting/troubleshooting.mdx | 4 +- apps/sim/app/api/auth/sso/register/route.ts | 2 +- apps/sim/instrumentation-node.ts | 6 + .../lib/core/security/egress/profiles.test.ts | 23 ++- apps/sim/lib/core/security/egress/profiles.ts | 21 ++- .../core/security/input-validation.server.ts | 2 +- .../core/security/ssrf-guarded-lookup.test.ts | 95 +++++++----- .../lib/data-drains/destinations/webhook.ts | 4 +- apps/sim/lib/internal/buffer/operations.ts | 7 +- .../sim/lib/internal/microsoft-word/client.ts | 2 +- apps/sim/lib/internal/zoom/operations.ts | 2 +- apps/sim/lib/mcp/client.ts | 10 +- apps/sim/lib/mcp/domain-check.test.ts | 4 + apps/sim/lib/mcp/pinned-fetch.ts | 16 +- apps/sim/tools/bitbucket/utils.server.ts | 6 +- packages/security/src/egress.test.ts | 81 +++++++++- packages/security/src/egress.ts | 141 ++++++++++++++---- packages/sim-setup/src/steps.ts | 36 +++++ 20 files changed, 369 insertions(+), 107 deletions(-) diff --git a/apps/docs/content/docs/agents/mcp.mdx b/apps/docs/content/docs/agents/mcp.mdx index 7fb57f23627..d6d04ee0371 100644 --- a/apps/docs/content/docs/agents/mcp.mdx +++ b/apps/docs/content/docs/agents/mcp.mdx @@ -87,7 +87,7 @@ Self-hosted deployments can restrict which MCP server domains are allowed by set This governs which domains may be used. It is separate from where those domains are allowed to resolve: an MCP server on a private address is reached by naming it in `EGRESS_ALLOWED_HOSTS` or `EGRESS_ALLOWED_IP_RANGES`, described in [Security](/platform/self-hosting/security#the-ssrf-boundary). Both checks apply. -The allowlist covers the server URL itself. If the server requires OAuth, the endpoints its authorization-server metadata names are treated as content rather than as configuration, so they have to be publicly routable. +The allowlist covers the server URL itself. If the server requires OAuth, any endpoint its metadata names on a *different* origin than the server you configured is treated as content rather than as configuration, so that one has to be publicly routable. Endpoints on the server's own origin keep the server's reachability. ## Using MCP Tools in Agents diff --git a/apps/docs/content/docs/platform/self-hosting/security.mdx b/apps/docs/content/docs/platform/self-hosting/security.mdx index fee4765d84d..2f75d7e963d 100644 --- a/apps/docs/content/docs/platform/self-hosting/security.mdx +++ b/apps/docs/content/docs/platform/self-hosting/security.mdx @@ -158,7 +158,9 @@ A wildcard (`*.svc.cluster.local`) and a broad range (`10.0.0.0/8`) are accepted Both lists are validated when Sim starts, and a malformed entry stops it with a message naming the setting. `EGRESS_ALLOWED_HOSTS` takes hostnames only — a URL or a CIDR is rejected — and a wildcard has to be a leading `*.` covering at least two labels, so `*.local` is refused and `*.svc.cluster.local` matches `vllm.ai.svc.cluster.local` but not the bare `svc.cluster.local`. `EGRESS_ALLOWED_IP_RANGES` takes CIDRs and bare addresses; `0.0.0.0/0` is refused as a catch-all. -Naming a destination permits plain HTTP to it and lifts the blocked-port list for it, since those are the same decision about the same host. A database, cache, or mail connector's host carries no scheme or port of its own, so naming one of those only lifts the private-address block. Cloud metadata endpoints (`169.254.169.254` and equivalents) stay blocked no matter how broad the allowlist is, and both variables are ignored entirely on Sim Cloud. +On Sim Cloud plain HTTP is refused for every provenance, self-hosted-service ones included: nothing is vouched there, so a credential would cross the wire in the clear. + +Naming a destination permits plain HTTP to it and lifts the blocked-port list for it, since those are the same decision about the same host. The loopback carve-out does not: it is granted without being asked for, so `http://localhost:5432` stays refused until `localhost` is named. A database, cache, or mail connector's host carries no scheme or port of its own, so naming one of those only lifts the private-address block. Cloud metadata endpoints (`169.254.169.254` and equivalents) stay blocked no matter how broad the allowlist is, and both variables are ignored entirely on Sim Cloud. The allowlist reaches the four provenances marked **Yes** above. It does not reach a content fetch, and it does not reach a proxy: an HTTP block's `proxyUrl` must be a public address, because the proxy is what decides where every other request may go. Adding an internal proxy to the allowlist will not make it work. @@ -169,18 +171,18 @@ EGRESS_ALLOWED_HOSTS=host.docker.internal ``` - An allowlist widens what every workflow author on the instance can reach. Name specific hosts and narrow ranges rather than whole private networks, and pair it with a NetworkPolicy that constrains what the app can actually reach. The chart's own NetworkPolicy permits broad egress on port 443 only, so an allowlisted in-cluster target on another port also has to be added to `networkPolicy.egress`. + An allowlist widens what every workflow author on the instance can reach. Name specific hosts and narrow ranges rather than whole private networks, and pair it with a NetworkPolicy that constrains what the app can actually reach. When `networkPolicy.enabled` is true the chart permits broad egress on port 443 only, so an allowlisted in-cluster target on another port also needs a `networkPolicy.egress` rule — or `networkPolicy.allowExternalEgress: true` for unrestricted egress. ### Upgrading from an earlier release The allowlist replaces four separate escape hatches, so a few deployments that worked before now need a destination named: -- **`ALLOW_PRIVATE_DATABASE_HOSTS`** still works, but it is deprecated and logs a warning at startup. It vouches for the whole private address space for database, cache, and mail connector hosts. Replace it with `EGRESS_ALLOWED_HOSTS` or `EGRESS_ALLOWED_IP_RANGES` naming the hosts you actually use. +- **`ALLOW_PRIVATE_DATABASE_HOSTS`** still works, but it is deprecated and logs a warning at startup. It vouches for the whole private address space, loopback included, for database, cache, and mail connector hosts. Replace it with `EGRESS_ALLOWED_HOSTS` or `EGRESS_ALLOWED_IP_RANGES` naming the hosts you actually use. - **1Password Connect** on a private, non-loopback address, and an **MCP server** on a private address or reached through a DNS name that points at loopback, are no longer reachable implicitly. Name them. - **`ALLOWED_MCP_DOMAINS`** governs which domains may be used; it no longer disables the address check, so an MCP server on a private address needs the allowlist too. -- **Content fetches** — an image URL, a file imported by URL, an OIDC endpoint discovered from a provider's metadata, an MCP OAuth endpoint the server's metadata names — never use the allowlist. Those destinations have to be publicly routable. -- **Redirects** are re-judged at every hop, so a redirect that downgrades to plain HTTP or lands on a blocked port is now refused. Credentials are dropped when a redirect crosses origins, and a cross-origin redirect that would carry a request body to the new origin is refused outright rather than replayed — a POST that lands on a cross-origin redirect now fails with a message saying so. +- **Content fetches** — an image URL, a file imported by URL, an MCP OAuth endpoint on a different origin than the MCP server itself — never use the allowlist. Those destinations have to be publicly routable. (An SSO OIDC *discovery* URL is a configured endpoint and does use the allowlist; the endpoints inside the discovery document are used by the auth library and sit outside this boundary.) +- **Redirects** are re-judged at every hop under the request's own provenance, so a redirect that lands on a blocked port is refused, and one that downgrades to plain HTTP is refused for every provenance except the self-hosted-service and proxy classes, which expect plain HTTP by design. Only 301, 302, 303, 307 and 308 are followed; 300, 305 and 306 are not. Credentials are dropped when a redirect crosses origins, and a cross-origin redirect that would carry a request body to the new origin is refused outright rather than replayed — a POST that lands on a cross-origin redirect now fails with a message saying so. ## Client IP and forwarded headers diff --git a/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx b/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx index 9d5845aa1cc..8451dc7e593 100644 --- a/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx +++ b/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx @@ -36,12 +36,12 @@ EGRESS_ALLOWED_HOSTS=host.docker.internal,*.svc.cluster.local EGRESS_ALLOWED_IP_RANGES=10.0.0.0/8 ``` -Naming a destination also permits plain HTTP to it and lifts the blocked-port list for it. Cloud metadata endpoints (`169.254.169.254` and equivalents) stay blocked however broad the list is, and both variables are ignored on Sim Cloud. +Naming a destination also permits plain HTTP to it and lifts the blocked-port list for it. A database, cache, or mail connector's host carries no scheme or port of its own, so naming one of those only lifts the private-address block. Cloud metadata endpoints (`169.254.169.254` and equivalents) stay blocked however broad the list is, and both variables are ignored on Sim Cloud. Two things this does not cover: - Inside a container `localhost` is the container itself, so it will never reach a service on your host. Use `host.docker.internal` (the Compose files map it) and name it above. -- URLs harvested from content or from a third-party API response — an image URL, a file imported by URL, an OIDC or MCP OAuth endpoint discovered from a provider's metadata — never reach a private network, allowlist or not. Nor does an HTTP block's `proxyUrl`. +- URLs harvested from content or from a third-party API response — an image URL, a file imported by URL, an MCP OAuth endpoint on a different origin than the MCP server itself — never reach a private network, allowlist or not. Nor does an HTTP block's `proxyUrl`. ## LM Studio Requests Route to Ollama diff --git a/apps/sim/app/api/auth/sso/register/route.ts b/apps/sim/app/api/auth/sso/register/route.ts index fcbe0a0c082..9d3810cd700 100644 --- a/apps/sim/app/api/auth/sso/register/route.ts +++ b/apps/sim/app/api/auth/sso/register/route.ts @@ -62,7 +62,7 @@ async function fetchOIDCDiscoveryDocument(discoveryUrl: string): Promise { diff --git a/apps/sim/lib/core/security/egress/profiles.test.ts b/apps/sim/lib/core/security/egress/profiles.test.ts index d9bad1b130b..0ac86c1f75e 100644 --- a/apps/sim/lib/core/security/egress/profiles.test.ts +++ b/apps/sim/lib/core/security/egress/profiles.test.ts @@ -96,7 +96,24 @@ describe('the hosted platform ignores every softening', () => { it('caps plain HTTP, which is a self-hosted arrangement', () => { envFlagsMock.isHosted = true expect(decide('selfHostedService', 'http://vllm.example/', '93.184.216.34').allowed).toBe(false) - expect(decide('proxy', 'http://proxy.example/', '93.184.216.34').allowed).toBe(false) + }) + + it('exempts the proxy, whose scheme is fixed by the protocol rather than by trust', () => { + envFlagsMock.isHosted = true + expect(decide('proxy', 'http://proxy.example/', '93.184.216.34').allowed).toBe(true) + expect(decide('proxy', 'http://proxy.example/', '10.4.2.9').allowed).toBe(false) + }) + + it('still permits ordinary public HTTPS', () => { + envFlagsMock.isHosted = true + expect(decide('requestTarget', 'https://api.example/', '93.184.216.34').allowed).toBe(true) + }) + + it('still refuses a service port on a public host', () => { + envFlagsMock.isHosted = true + expect(decide('requestTarget', 'https://api.example:5432/', '93.184.216.34').allowed).toBe( + false + ) }) it('offers no remedy in the refusal, where the variables would do nothing', () => { @@ -112,7 +129,9 @@ describe('the deprecated ALLOW_PRIVATE_DATABASE_HOSTS', () => { it('reaches database hosts only', () => { envFlagsMock.legacyPrivateDatabaseAccess = true expect(decide('databaseHost', 'https://pg.corp/', '10.4.2.9').allowed).toBe(true) - expect(decide('configuredEndpoint', 'https://pg.corp/', '10.4.2.9').allowed).toBe(false) + for (const profile of ['configuredEndpoint', 'selfHostedService', 'requestTarget'] as const) { + expect(decide(profile, 'https://pg.corp/', '10.4.2.9').allowed).toBe(false) + } }) it('still cannot reach a metadata endpoint', () => { diff --git a/apps/sim/lib/core/security/egress/profiles.ts b/apps/sim/lib/core/security/egress/profiles.ts index a3851b4263e..38a066362f0 100644 --- a/apps/sim/lib/core/security/egress/profiles.ts +++ b/apps/sim/lib/core/security/egress/profiles.ts @@ -72,7 +72,8 @@ interface ProfileSpec { * `always` is capped at `whenVouched` when hosted, where nothing is vouched — * software served without TLS is a self-hosted arrangement, and a hosted * deployment sending a credential over cleartext to a user-supplied host is - * not one this taxonomy should permit. + * not one this taxonomy should permit. {@link ProfileSpec.schemeFixedByProtocol} + * exempts the one profile whose scheme is not a trust decision. */ readonly insecureHttp: InsecureHttpPolicy /** @@ -86,6 +87,13 @@ interface ProfileSpec { * hosted branch is reachable from a test. */ readonly allowLoopbackOffHosted: boolean + /** + * Whether this profile's scheme is fixed by the protocol rather than by how + * much the destination is trusted. Only `proxy` is: an HTTP proxy is spoken to + * over HTTP by definition, so the hosted cap below would leave it with no + * reachable configuration at all. + */ + readonly schemeFixedByProtocol?: boolean /** * Whether the deprecated `ALLOW_PRIVATE_DATABASE_HOSTS` applies. Only * `databaseHost` sets this, because that is the only thing the flag ever @@ -117,7 +125,12 @@ const PROFILE_SPECS: Record = { allowLoopbackOffHosted: false, honorsLegacyPrivateFlag: true, }, - proxy: { honorsAllowlist: false, insecureHttp: 'always', allowLoopbackOffHosted: false }, + proxy: { + honorsAllowlist: false, + insecureHttp: 'always', + allowLoopbackOffHosted: false, + schemeFixedByProtocol: true, + }, } const SOURCE_NAMES = { @@ -151,7 +164,9 @@ function buildPolicies(config: DeploymentConfig): Record 0) diff --git a/apps/sim/lib/core/security/ssrf-guarded-lookup.test.ts b/apps/sim/lib/core/security/ssrf-guarded-lookup.test.ts index 7a59c48727e..12e553a225f 100644 --- a/apps/sim/lib/core/security/ssrf-guarded-lookup.test.ts +++ b/apps/sim/lib/core/security/ssrf-guarded-lookup.test.ts @@ -21,6 +21,7 @@ declare module '@/lib/core/security/input-validation.server?ssrf-guarded-lookup- export * from '@/lib/core/security/input-validation.server' } +import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { createSsrfGuardedLookup, followRedirectsGuarded, @@ -30,10 +31,11 @@ type LookupResult = { address: string; family: number } function runLookup( hostname: string, - options: { all?: boolean } = {} + options: { all?: boolean } = {}, + profile: EgressProfile = 'contentFetch' ): Promise<{ err: Error | null; address?: string | LookupResult[]; family?: number }> { return new Promise((resolve) => { - const lookup = createSsrfGuardedLookup() + const lookup = createSsrfGuardedLookup(profile) type LookupCb = (err: Error | null, address?: string | LookupResult[], family?: number) => void // double-cast-allowed: net.LookupFunction's overloaded callback shapes collapse to this in practice ;(lookup as unknown as (h: string, o: object, cb: LookupCb) => void)( @@ -120,7 +122,7 @@ function redirectTo(location: string, status = 302): Response { describe('followRedirectsGuarded', () => { it('returns a non-redirect response as-is', async () => { const raw = vi.fn(async () => new Response('ok', { status: 200 })) - const res = await followRedirectsGuarded(raw, 'https://a.example/x', {}) + const res = await followRedirectsGuarded(raw, 'https://a.example/x', {}, 'contentFetch') expect(res.status).toBe(200) expect(raw).toHaveBeenCalledTimes(1) }) @@ -130,9 +132,12 @@ describe('followRedirectsGuarded', () => { .fn() .mockResolvedValueOnce(redirectTo('https://a.example/y')) .mockResolvedValueOnce(new Response('ok', { status: 200 })) - const res = await followRedirectsGuarded(raw, 'https://a.example/x', { - headers: { 'x-api-key': 'secret' }, - }) + const res = await followRedirectsGuarded( + raw, + 'https://a.example/x', + { headers: { 'x-api-key': 'secret' } }, + 'contentFetch' + ) expect(res.status).toBe(200) expect(raw.mock.calls[1][0]).toBe('https://a.example/y') expect(raw.mock.calls[1][1].headers).toEqual({ 'x-api-key': 'secret' }) @@ -143,39 +148,42 @@ describe('followRedirectsGuarded', () => { .fn() .mockResolvedValueOnce(redirectTo('https://b.example/harvest')) .mockResolvedValueOnce(new Response('ok', { status: 200 })) - await followRedirectsGuarded(raw, 'https://a.example/x', { - headers: { 'x-api-key': 'secret' }, - }) + await followRedirectsGuarded( + raw, + 'https://a.example/x', + { headers: { 'x-api-key': 'secret' } }, + 'contentFetch' + ) expect(raw.mock.calls[1][1].headers).toBeUndefined() }) it('blocks a redirect to a private IP literal (metadata endpoint)', async () => { const raw = vi.fn(async () => redirectTo('http://169.254.169.254/latest/meta-data/')) - await expect(followRedirectsGuarded(raw, 'https://a.example/x', {})).rejects.toThrow( - /Blocked by SSRF policy/ - ) + await expect( + followRedirectsGuarded(raw, 'https://a.example/x', {}, 'contentFetch') + ).rejects.toThrow(/Blocked by SSRF policy/) expect(raw).toHaveBeenCalledTimes(1) }) it('blocks a redirect to a bracketed private IPv6 literal', async () => { const raw = vi.fn(async () => redirectTo('http://[::1]/admin')) - await expect(followRedirectsGuarded(raw, 'https://a.example/x', {})).rejects.toThrow( - /Blocked by SSRF policy/ - ) + await expect( + followRedirectsGuarded(raw, 'https://a.example/x', {}, 'contentFetch') + ).rejects.toThrow(/Blocked by SSRF policy/) }) it('blocks non-http(s) redirect protocols', async () => { const raw = vi.fn(async () => redirectTo('file:///etc/passwd')) - await expect(followRedirectsGuarded(raw, 'https://a.example/x', {})).rejects.toThrow( - /unsupported protocol/ - ) + await expect( + followRedirectsGuarded(raw, 'https://a.example/x', {}, 'contentFetch') + ).rejects.toThrow(/unsupported protocol/) }) it('caps the number of hops', async () => { const raw = vi.fn(async () => redirectTo('https://a.example/loop')) - await expect(followRedirectsGuarded(raw, 'https://a.example/x', {})).rejects.toThrow( - /more than \d+ redirects/ - ) + await expect( + followRedirectsGuarded(raw, 'https://a.example/x', {}, 'contentFetch') + ).rejects.toThrow(/more than \d+ redirects/) }) it('switches POST to a bodyless GET on 303', async () => { @@ -183,7 +191,12 @@ describe('followRedirectsGuarded', () => { .fn() .mockResolvedValueOnce(redirectTo('https://a.example/next', 303)) .mockResolvedValueOnce(new Response('ok', { status: 200 })) - await followRedirectsGuarded(raw, 'https://a.example/x', { method: 'POST', body: 'data' }) + await followRedirectsGuarded( + raw, + 'https://a.example/x', + { method: 'POST', body: 'data' }, + 'contentFetch' + ) expect(raw.mock.calls[1][1].method).toBe('GET') expect(raw.mock.calls[1][1].body).toBeUndefined() }) @@ -193,7 +206,7 @@ describe('followRedirectsGuarded', () => { .fn() .mockResolvedValueOnce(redirectTo('https://a.example/next', 303)) .mockResolvedValueOnce(new Response(null, { status: 200 })) - await followRedirectsGuarded(raw, 'https://a.example/x', { method: 'HEAD' }) + await followRedirectsGuarded(raw, 'https://a.example/x', { method: 'HEAD' }, 'contentFetch') expect(raw.mock.calls[1][1].method).toBe('HEAD') }) @@ -202,7 +215,12 @@ describe('followRedirectsGuarded', () => { .fn() .mockResolvedValueOnce(redirectTo('https://a.example/next', 307)) .mockResolvedValueOnce(new Response('ok', { status: 200 })) - await followRedirectsGuarded(raw, 'https://a.example/x', { method: 'POST', body: 'data' }) + await followRedirectsGuarded( + raw, + 'https://a.example/x', + { method: 'POST', body: 'data' }, + 'contentFetch' + ) expect(raw.mock.calls[1][1].method).toBe('POST') expect(raw.mock.calls[1][1].body).toBe('data') }) @@ -212,7 +230,7 @@ describe('followRedirectsGuarded — hardening', () => { it('blocks a private IP-literal as the INITIAL url (guard is self-contained)', async () => { const raw = vi.fn(async () => new Response('ok')) await expect( - followRedirectsGuarded(raw, 'http://169.254.169.254/latest/meta-data/', {}) + followRedirectsGuarded(raw, 'http://169.254.169.254/latest/meta-data/', {}, 'contentFetch') ).rejects.toThrow(/Blocked by SSRF policy/) expect(raw).not.toHaveBeenCalled() }) @@ -222,11 +240,16 @@ describe('followRedirectsGuarded — hardening', () => { .fn() .mockResolvedValueOnce(redirectTo('https://a.example/next', 303)) .mockResolvedValueOnce(new Response('ok', { status: 200 })) - await followRedirectsGuarded(raw, 'https://a.example/x', { - method: 'POST', - body: '{"a":1}', - headers: { 'content-type': 'application/json', 'content-length': '7', 'x-keep': 'yes' }, - }) + await followRedirectsGuarded( + raw, + 'https://a.example/x', + { + method: 'POST', + body: '{"a":1}', + headers: { 'content-type': 'application/json', 'content-length': '7', 'x-keep': 'yes' }, + }, + 'contentFetch' + ) const hopHeaders = new Headers(raw.mock.calls[1][1].headers) expect(hopHeaders.get('content-type')).toBeNull() expect(hopHeaders.get('content-length')).toBeNull() @@ -238,10 +261,12 @@ describe('followRedirectsGuarded — cross-origin body protection', () => { it('refuses a cross-origin 307 that would forward a request body', async () => { const raw = vi.fn(async () => redirectTo('https://b.example/steal', 307)) await expect( - followRedirectsGuarded(raw, 'https://a.example/token', { - method: 'POST', - body: 'client_secret=shh', - }) + followRedirectsGuarded( + raw, + 'https://a.example/token', + { method: 'POST', body: 'client_secret=shh' }, + 'contentFetch' + ) ).rejects.toThrow(/cross-origin redirect would forward a request body/) }) @@ -250,7 +275,7 @@ describe('followRedirectsGuarded — cross-origin body protection', () => { .fn() .mockResolvedValueOnce(redirectTo('https://b.example/next', 302)) .mockResolvedValueOnce(new Response('ok', { status: 200 })) - const res = await followRedirectsGuarded(raw, 'https://a.example/x', {}) + const res = await followRedirectsGuarded(raw, 'https://a.example/x', {}, 'contentFetch') expect(res.status).toBe(200) }) }) diff --git a/apps/sim/lib/data-drains/destinations/webhook.ts b/apps/sim/lib/data-drains/destinations/webhook.ts index 1403bd3bd34..54c480f6dd0 100644 --- a/apps/sim/lib/data-drains/destinations/webhook.ts +++ b/apps/sim/lib/data-drains/destinations/webhook.ts @@ -47,8 +47,8 @@ const HEADER_INJECTION_PATTERN = /[\r\n\0]/ async function resolvePublicTarget(url: string): Promise { const result = await validateUrlWithDNS(url, 'url', 'configuredEndpoint') - if (!result.isValid || !result.resolvedIP) { - throw new Error(result.error ?? 'Webhook URL failed SSRF validation') + if (!result.isValid) { + throw new Error(result.error) } return result.resolvedIP } diff --git a/apps/sim/lib/internal/buffer/operations.ts b/apps/sim/lib/internal/buffer/operations.ts index 1bce11a0b6b..77c833e468e 100644 --- a/apps/sim/lib/internal/buffer/operations.ts +++ b/apps/sim/lib/internal/buffer/operations.ts @@ -71,8 +71,11 @@ async function resolveMediaKind(args: { const extensionKind = mediaKindFromExtension(pathOrName) if (extensionKind) return extensionKind - // `fileUrl` is minted by `resolveFileInputToUrl` against Sim's own storage, - // which on a self-hosted deployment legitimately sits on a private address. + // A `fileUrl` minted by `resolveFileInputToUrl` is a presigned URL against + // Sim's own storage, which on a self-hosted deployment legitimately sits on a + // private address. A caller-supplied URL passes through that helper unchanged + // and was already judged as content there, so widening here re-permits + // nothing it refused. try { const validation = await validateUrlWithDNS(fileUrl, 'media', 'configuredEndpoint') context.signal?.throwIfAborted() diff --git a/apps/sim/lib/internal/microsoft-word/client.ts b/apps/sim/lib/internal/microsoft-word/client.ts index 28cf2ab189a..b3948e9f26e 100644 --- a/apps/sim/lib/internal/microsoft-word/client.ts +++ b/apps/sim/lib/internal/microsoft-word/client.ts @@ -322,7 +322,7 @@ async function uploadSessionBytes( const response = await graphFetch( uploadUrl, - 'documentUploadUrl', + 'documentUploadSessionUrl', { method: 'PUT', headers: { diff --git a/apps/sim/lib/internal/zoom/operations.ts b/apps/sim/lib/internal/zoom/operations.ts index a384efc0dba..1860b06e667 100644 --- a/apps/sim/lib/internal/zoom/operations.ts +++ b/apps/sim/lib/internal/zoom/operations.ts @@ -107,7 +107,7 @@ export async function getZoomMeetingRecordings( 'downloadUrl', 'contentFetch' ) - if (!fileValidation.isValid || !fileValidation.resolvedIP) continue + if (!fileValidation.isValid) continue const downloadResponse = await secureFetchWithPinnedIP( file.download_url, fileValidation.resolvedIP, diff --git a/apps/sim/lib/mcp/client.ts b/apps/sim/lib/mcp/client.ts index 5aefff7fc08..8d9622cbed7 100644 --- a/apps/sim/lib/mcp/client.ts +++ b/apps/sim/lib/mcp/client.ts @@ -104,11 +104,11 @@ export class McpClient { throw new McpError('OAuth MCP server requires an authProvider') } const useOauth = this.config.authType === 'oauth' - // `resolvedIP` non-null signals the SSRF policy is active for this server (it is null in - // allowlist mode / localhost-on-self-hosted); the guard validates addresses per-connect. - // A private/loopback resolvedIP only reaches here on self-hosted (where the policy - // permits it) — the guarded lookup would filter it, so that case keeps the legacy pin - // to the validated address (old behavior + its anti-rebinding property). + // `resolvedIP` is null only when the hostname still carries an unresolved env-var + // reference, which is checked again once it resolves. Otherwise the guard validates + // addresses per-connect. A private/loopback resolvedIP only reaches here on a + // self-hosted deployment whose policy permits it, and that case pins to the address + // that was validated rather than to whatever the name resolves to next. const guarded = resolvedIP ? isPrivateIp(resolvedIP) ? createPinnedPrivateMcpFetch(resolvedIP) diff --git a/apps/sim/lib/mcp/domain-check.test.ts b/apps/sim/lib/mcp/domain-check.test.ts index a2108c7408e..f68f9de7b0c 100644 --- a/apps/sim/lib/mcp/domain-check.test.ts +++ b/apps/sim/lib/mcp/domain-check.test.ts @@ -553,6 +553,10 @@ describe('validateMcpServerSsrf', () => { }) describe('the OAuth provenance', () => { + beforeEach(() => { + setEnvFlags({ isHosted: false }) + }) + it('is contentFetch, so a hop the metadata names inherits nothing from the server', () => { expect(OAUTH_EGRESS_PROFILE).toBe('contentFetch') expect(MCP_EGRESS_PROFILE).toBe('selfHostedService') diff --git a/apps/sim/lib/mcp/pinned-fetch.ts b/apps/sim/lib/mcp/pinned-fetch.ts index b30e2ba18e8..c36eba0fdb2 100644 --- a/apps/sim/lib/mcp/pinned-fetch.ts +++ b/apps/sim/lib/mcp/pinned-fetch.ts @@ -93,11 +93,12 @@ function capResponseBody(response: Response, maxBytes: number): Response { } /** - * Legacy single-IP pin, kept ONLY for self-hosted private/loopback resolutions - * (a DNS alias the policy explicitly permits): the guarded lookup would filter - * the address and strand the connect, while an unguarded fallback would reopen - * rebinding/redirect escape. Pinning to the validated address preserves the old - * behavior and its security property for exactly this carve-out. + * Single-IP pin, used for the self-hosted private/loopback resolutions the + * policy explicitly permits. The guarded transport would reach them too, but a + * destination vouched by hostname is vouched for wherever it points, so on this + * one path pinning to the address that was actually validated is the stricter + * choice: it holds the connection to that address rather than to whatever the + * name resolves to next. */ export function createPinnedPrivateMcpFetch(resolvedIP: string): GuardedMcpFetch { const { fetch: pinnedFetch, dispatcher } = createPinnedFetchWithDispatcher(resolvedIP, { @@ -322,8 +323,9 @@ export function createSsrfGuardedMcpFetch( throw new McpSsrfError('MCP OAuth request URL could not be validated') } logger.info('OAuth guarded fetch: requesting', { host, configured: sameAsConfigured }) - // A private address only survives validation on the configured first hop, - // and the guarded lookup would filter it, so that case pins instead. + // A private address only survives validation on the configured first hop. + // Pinning it holds the connection to the address that was validated, which + // a hostname-vouched destination would otherwise not be held to. const transport = isPrivateIp(resolvedIP) ? createPinnedFetchWithDispatcher(resolvedIP, { profile, diff --git a/apps/sim/tools/bitbucket/utils.server.ts b/apps/sim/tools/bitbucket/utils.server.ts index a82989533b2..c1b64fdda11 100644 --- a/apps/sim/tools/bitbucket/utils.server.ts +++ b/apps/sim/tools/bitbucket/utils.server.ts @@ -143,10 +143,8 @@ export async function resolveBitbucketPullRequestRedirect( 'bitbucketPullRequestUrl', 'configuredEndpoint' ) - if (!initialValidation.isValid || !initialValidation.resolvedIP) { - throw new Error( - `Invalid Bitbucket pull request URL: ${initialValidation.error ?? 'DNS resolution failed'}` - ) + if (!initialValidation.isValid) { + throw new Error(`Invalid Bitbucket pull request URL: ${initialValidation.error}`) } const { fetch: pinnedFetch, dispatcher } = createPinnedFetchWithDispatcher( diff --git a/packages/security/src/egress.test.ts b/packages/security/src/egress.test.ts index 2c9ef9b29bb..39b39d41888 100644 --- a/packages/security/src/egress.test.ts +++ b/packages/security/src/egress.test.ts @@ -418,12 +418,25 @@ describe('IPv6 forms that carry an IPv4 destination', () => { } ) - it('refuses even a public IPv4 carried in a NAT64 prefix unless the policy vouches', () => { - // `ipaddr.js` classifies the whole RFC 6052 prefix as non-unicast, so an - // unvouched destination never reaches it. Fail-closed is the right default: - // a DNS64 answer is a translated route, not the destination itself. - expect(reason(hosted, 'https://example.com/', '64:ff9b::5db8:d822')).toBe('address-blocked') - expect(decide(vouchesByName, 'https://internal.corp/', '64:ff9b::5db8:d822').allowed).toBe(true) + it('judges a public IPv4 carried in a translation prefix as that IPv4', () => { + expect(decide(hosted, 'https://example.com/', '64:ff9b::5db8:d822').allowed).toBe(true) + expect(decide(hosted, 'https://example.com/', '2002:5db8:d822::').allowed).toBe(true) + }) + + it.each([ + ['2002:a9fe:a9fe::', 'RFC 3056 6to4'], + ['fe80::5efe:169.254.169.254', 'RFC 5214 ISATAP'], + ])('reads %s as the metadata endpoint it carries — %s', (address) => { + expect(reason(vouchesByName, 'https://internal.corp/', address)).toBe('address-metadata') + }) + + it.each([ + ['2001:0:a9fe:a9fe::', 'Teredo, which carries two IPv4 addresses'], + ['::1:7f00:1', 'the reserved ::/64 block'], + ['::ffff:1:a9fe:a9fe', 'one group outside the IPv4-translated prefix'], + ])('refuses %s — %s', (address) => { + expect(reason(hosted, 'https://example.com/', address)).toBe('address-blocked') + expect(reason(vouchesByName, 'https://internal.corp/', address)).toBe('address-blocked') }) }) @@ -509,7 +522,7 @@ describe('createEgressPolicy validates wildcard entries too', () => { ['*.foo.com/x', /expected a hostname/], ['*.a*.com', /leading/], ['*..com', /non-empty/], - ['*.', /non-empty/], + ['*.', /leading/], ['.example.com', /non-empty/], ])('rejects %s', (entry, message) => { expect(() => createEgressPolicy({ allowedHosts: [entry] })).toThrow(message) @@ -521,3 +534,57 @@ describe('createEgressPolicy validates wildcard entries too', () => { expect(decide(policy, 'https://svc.cluster.local/', '10.4.2.9').allowed).toBe(false) }) }) + +describe('a scope id does not change what an address is', () => { + it('still reads a zoned metadata address as metadata', () => { + const policy = createEgressPolicy({ allowPrivate: true }) + expect(reason(policy, 'https://pg.corp/', 'fd00:ec2::254%eth0')).toBe('address-metadata') + }) +}) + +describe('an explicit allowlist grant outranks the loopback carve-out', () => { + it('lifts the port denylist that the carve-out alone leaves in place', () => { + const named = createEgressPolicy({ + allowedRanges: '127.0.0.1/32', + allowLoopback: true, + insecureHttp: 'whenVouched', + }) + const carveOutOnly = createEgressPolicy({ allowLoopback: true, insecureHttp: 'whenVouched' }) + expect(decide(named, 'http://localhost:5432/', '127.0.0.1').allowed).toBe(true) + expect(reason(carveOutOnly, 'http://localhost:5432/', '127.0.0.1')).toBe('port-denied') + }) +}) + +describe('loopback names beyond the bare label', () => { + it.each([ + ['https://localhost./x', 'a trailing dot'], + ['https://foo.localhost/x', 'the RFC 6761 suffix'], + ])('refuses %s pre-DNS — %s', (href) => { + expect(reason(hosted, href)).toBe('address-loopback') + }) +}) + +describe('a trailing dot does not defeat the host allowlist', () => { + it('matches an entry written without one', () => { + const policy = createEgressPolicy({ allowedHosts: 'api.example.com' }) + expect(decide(policy, 'https://api.example.com./', '10.0.0.1').allowed).toBe(true) + }) +}) + +describe('an operator range naming a translation prefix still matches', () => { + it('vouches for an address inside it', () => { + const policy = createEgressPolicy({ allowedRanges: '64:ff9b::/96' }) + expect(decide(policy, 'https://svc.internal/', '64:ff9b::a00:1').allowed).toBe(true) + }) +}) + +describe('an internationalized allowlist entry names the form that works', () => { + it('refuses the Unicode spelling rather than silently never matching', () => { + expect(() => createEgressPolicy({ allowedHosts: ['*.exämple.com'] })).toThrow(/punycode/) + }) + + it('accepts the A-label form', () => { + const policy = createEgressPolicy({ allowedHosts: '*.xn--exmple-cua.com' }) + expect(decide(policy, 'https://api.xn--exmple-cua.com/', '10.0.0.1').allowed).toBe(true) + }) +}) diff --git a/packages/security/src/egress.ts b/packages/security/src/egress.ts index 4a883cabd54..4ad359677ae 100644 --- a/packages/security/src/egress.ts +++ b/packages/security/src/egress.ts @@ -30,8 +30,6 @@ import { isIpLiteral, isLoopbackIp, isPrivateIp } from './ssrf' type IpAddress = ipaddr.IPv4 | ipaddr.IPv6 -/** Schemes that may ever carry an outbound request. */ - /** When a policy tolerates plain HTTP. */ export type InsecureHttpPolicy = 'never' | 'whenVouched' | 'always' @@ -167,8 +165,13 @@ function splitEntries(value: string | readonly string[] | undefined): string[] { return parts.map((entry) => entry.trim()).filter((entry) => entry.length > 0) } +/** A hostname as the allowlist compares it: lower-cased, with no trailing dot. */ +function normalizeHost(host: string): string { + return unwrapIpv6Brackets(host.toLowerCase()).replace(/\.$/, '') +} + function parseHostPattern(entry: string, sourceName: string): HostPattern { - const value = unwrapIpv6Brackets(entry.toLowerCase()) + const value = normalizeHost(entry) const wildcard = value.startsWith('*.') const host = wildcard ? value.slice(2) : value @@ -185,6 +188,15 @@ function parseHostPattern(entry: string, sourceName: string): HostPattern { if (host.length === 0 || host.split('.').some((label) => label.length === 0)) { throw new Error(`Invalid ${sourceName} entry "${entry}": every label must be non-empty`) } + // A URL hostname is always the A-label form, so a Unicode entry could never + // match and would present as an unexplained connection failure. Refusing it + // says so, and names the form that works. + // biome-ignore lint/suspicious/noControlCharactersInRegex: rejecting control characters is the point + if (/[^\x21-\x7e]/.test(host)) { + throw new Error( + `Invalid ${sourceName} entry "${entry}": use the punycode form of an internationalized name` + ) + } if (wildcard && !host.includes('.')) { throw new Error( `Invalid ${sourceName} entry "${entry}": a wildcard must cover at least two labels, e.g. "*.example.com"` @@ -238,7 +250,7 @@ export const STRICT_EGRESS_POLICY: EgressPolicy = createEgressPolicy() function matchesHostAllowlist(host: string, policy: EgressPolicy): boolean { if (policy.allowedHosts.length === 0) return false - const clean = unwrapIpv6Brackets(host.toLowerCase()) + const clean = normalizeHost(host) return policy.allowedHosts.some((pattern) => pattern.wildcard ? clean.endsWith(pattern.value) : clean === pattern.value ) @@ -246,15 +258,21 @@ function matchesHostAllowlist(host: string, policy: EgressPolicy): boolean { function matchesRangeAllowlist(address: string, policy: EgressPolicy): boolean { if (policy.allowedRanges.length === 0) return false - // Canonical form, so an operator's IPv4 CIDR still matches a resolver that - // answered with the IPv4-compatible IPv6 spelling of the same address. - const clean = canonicalAddress(address) - if (clean === null) return false - const parsed = ipaddr.process(clean) - return policy.allowedRanges.some( - (range) => - range.address.kind() === parsed.kind() && parsed.match(range.address, range.prefixLength) - ) + + // Both spellings are compared: the canonical one so an operator's IPv4 CIDR + // matches a resolver that answered with an IPv4-in-IPv6 form, and the literal + // one so an entry naming the wrapping IPv6 prefix itself still matches. + const literal = unwrapIpv6Brackets(address).split('%')[0] + const candidates = [canonicalAddress(address), ipaddr.isValid(literal) ? literal : null] + + return candidates.some((candidate) => { + if (candidate === null) return false + const parsed = ipaddr.process(candidate) + return policy.allowedRanges.some( + (range) => + range.address.kind() === parsed.kind() && parsed.match(range.address, range.prefixLength) + ) + }) } /** The IPv4 carried in an address's last 32 bits. */ @@ -282,6 +300,53 @@ const IPV4_EMBEDDING_PREFIXES: readonly (readonly number[])[] = [ /** RFC 8215 local-use NAT64 prefix, `64:ff9b:1::/48`. */ const NAT64_LOCAL_USE_PREFIX = [0x0064, 0xff9b, 0x0001] as const +/** + * The IPv4 an IPv6 address carries under one of the transition schemes, or null + * when it carries none. Each of these is a real route to the IPv4 address they + * name, so judging the wrapper instead of the destination is how an IPv6 + * spelling of a metadata endpoint gets through. + */ +function transitionIpv4(parts: readonly number[]): string | null { + // RFC 3056 6to4, `2002:a.b.c.d::/48`. + if (parts[0] === 0x2002) { + return ipaddr + .fromByteArray([ + (parts[1] >> 8) & 0xff, + parts[1] & 0xff, + (parts[2] >> 8) & 0xff, + parts[2] & 0xff, + ]) + .toString() + } + // RFC 5214 ISATAP, `:0:5efe:a.b.c.d`, and its `0:200:5efe` variant. + if (parts[4] === 0x0000 && parts[5] === 0x5efe) return embeddedIpv4(parts) + if (parts[4] === 0x0200 && parts[5] === 0x5efe) return embeddedIpv4(parts) + return null +} + +/** + * Whether the address is RFC 4380 Teredo, `2001:0::/32`, which carries the + * client's IPv4 obfuscated (bitwise-inverted) in its low 32 bits and the relay + * server's in the middle. Both are IPv4 destinations, so rather than pick one to + * canonicalize, the address is refused outright. + */ +function isTeredo(parts: readonly number[]): boolean { + return parts[0] === 0x2001 && parts[1] === 0x0000 +} + +/** + * Whether the address sits in `::/64` without being one of the forms folded + * above. That block is reserved, nothing routes there, and an address in it + * carries something this code cannot read — `::` and `::1` are excluded because + * they are the unspecified and loopback addresses, which are classified + * normally. + */ +function isUnreadableReservedLowBlock(parts: readonly number[]): boolean { + if (!parts.slice(0, 4).every((part) => part === 0)) return false + const trailing = (((parts[4] << 16) >>> 0) + parts[5]) | (((parts[6] << 16) >>> 0) + parts[7]) + return trailing !== 0 && !(parts.slice(0, 7).every((part) => part === 0) && parts[7] === 1) +} + /** * Canonical form of an address, folding every IPv4-in-IPv6 spelling down to the * IPv4 it carries. `ipaddr.process` handles the IPv4-mapped form (`::ffff:x`) @@ -290,7 +355,9 @@ const NAT64_LOCAL_USE_PREFIX = [0x0064, 0xff9b, 0x0001] as const * metadata endpoint written that way. */ function canonicalAddress(address: string): string | null { - const clean = unwrapIpv6Brackets(address) + // A scope id names an interface, not a destination, and leaving it on would + // make `fd00:ec2::254%eth0` a different string from the metadata address it is. + const clean = unwrapIpv6Brackets(address).split('%')[0] if (!ipaddr.isValid(clean)) return null const parsed = ipaddr.process(clean) @@ -313,6 +380,9 @@ function canonicalAddress(address: string): string | null { if (parts.slice(0, 6).every((part) => part === 0) && embedded > 1) { return embeddedIpv4(parts) } + + const transition = transitionIpv4(parts) + if (transition !== null) return transition } return parsed.toString() } @@ -327,17 +397,24 @@ function isMetadataAddress(address: string): boolean { } /** - * Whether the address sits in the RFC 8215 local-use NAT64 prefix, where the - * embedded IPv4 destination sits at an offset chosen by the network operator - * and so cannot be read off the address. + * Whether the address carries an IPv4 destination this code cannot read: the + * RFC 8215 local-use NAT64 prefix, whose embedded IPv4 sits at an offset the + * network operator chooses; Teredo, which carries two; and the rest of the + * reserved `::/64` block. */ function hidesItsIpv4Destination(address: string): boolean { - const clean = unwrapIpv6Brackets(address) + const clean = unwrapIpv6Brackets(address).split('%')[0] if (!ipaddr.isValid(clean)) return false const parsed = ipaddr.process(clean) if (parsed.kind() !== 'ipv6') return false + // An address whose IPv4 the folding above could read is judged as that IPv4. + if (canonicalAddress(address) !== parsed.toString()) return false const { parts } = parsed as ipaddr.IPv6 - return NAT64_LOCAL_USE_PREFIX.every((part, index) => parts[index] === part) + return ( + NAT64_LOCAL_USE_PREFIX.every((part, index) => parts[index] === part) || + isTeredo(parts) || + isUnreadableReservedLowBlock(parts) + ) } /** @@ -350,8 +427,10 @@ function hidesItsIpv4Destination(address: string): boolean { * route to the deployment's own loopback services. */ function isLoopbackDestination(host: string): boolean { - const clean = unwrapIpv6Brackets(host) - return isLoopbackHostname(clean) || isLoopbackIp(clean) + // RFC 6761 reserves `localhost` and everything under it, and a fully qualified + // name may carry a trailing dot the URL parser keeps. + const clean = unwrapIpv6Brackets(host).replace(/\.$/, '') + return isLoopbackHostname(clean) || clean.endsWith('.localhost') || isLoopbackIp(clean) } /** @@ -375,19 +454,23 @@ type Vouch = 'allowlist' | 'loopback' | null function isVouched(url: URL, address: string | undefined, policy: EgressPolicy): Vouch { if (matchesHostAllowlist(url.hostname, policy)) return 'allowlist' + // The operator's explicit grants are checked first, so a deployment that named + // a loopback range gets more than one that named nothing — the carve-out below + // is the weaker of the two. + if (address !== undefined) { + if (policy.allowPrivate && isPrivateIp(unwrapIpv6Brackets(address))) return 'allowlist' + if (matchesRangeAllowlist(address, policy)) return 'allowlist' + } + if (policy.allowLoopback && isLoopbackDestination(url.hostname)) { // Before DNS there is no address to judge; evaluateAddress rules later. if (address === undefined) return 'loopback' // The address must land on loopback too, so a resolver answering - // `localhost` with a routable address cannot borrow the carve-out — but it - // may still be vouched by an allowlist entry, so this falls through rather - // than refusing outright. + // `localhost` with a routable address cannot borrow the carve-out. if (isLoopbackIp(unwrapIpv6Brackets(address))) return 'loopback' } - if (address === undefined) return null - if (policy.allowPrivate && isPrivateIp(unwrapIpv6Brackets(address))) return 'allowlist' - return matchesRangeAllowlist(address, policy) ? 'allowlist' : null + return null } function checkSchemeAndPort(url: URL, vouch: Vouch, policy: EgressPolicy): EgressDecision { @@ -417,7 +500,9 @@ function checkSchemeAndPort(url: URL, vouch: Vouch, policy: EgressPolicy): Egres function checkAddressClass(address: string, vouch: Vouch): EgressDecision { if (vouch !== null) return ALLOWED - const clean = unwrapIpv6Brackets(address) + // Classified on the canonical form, so an IPv6 wrapper around an IPv4 address + // is judged as the destination it carries rather than as the wrapper. + const clean = canonicalAddress(address) ?? unwrapIpv6Brackets(address) if (isLoopbackIp(clean)) { return deny('address-loopback', address) } diff --git a/packages/sim-setup/src/steps.ts b/packages/sim-setup/src/steps.ts index ed61a4d7df5..b01a0bcaa33 100644 --- a/packages/sim-setup/src/steps.ts +++ b/packages/sim-setup/src/steps.ts @@ -1,3 +1,5 @@ +import { createEgressPolicy } from '@sim/security/egress' +import { getErrorMessage } from '@sim/utils/errors' import { KNOWLEDGE_EMBEDDINGS_SETUP } from './capability-config' import { type CapabilitySetupContext, @@ -50,6 +52,26 @@ export function collectSecrets(existing: EnvFile): Record { return secrets } +/** + * Runs an allowlist answer through the same parser the app uses, so a malformed + * entry is caught at the prompt rather than at the first outbound request. + */ +function validateEgressEntries(spec: { + allowedHosts?: string + allowedRanges?: string +}): string | undefined { + try { + createEgressPolicy({ + allowedHosts: spec.allowedHosts?.trim() || undefined, + allowedRanges: spec.allowedRanges?.trim() || undefined, + sourceNames: { hosts: 'EGRESS_ALLOWED_HOSTS', ranges: 'EGRESS_ALLOWED_IP_RANGES' }, + }) + return undefined + } catch (error) { + return getErrorMessage(error, 'invalid entry') + } +} + export async function promptCopilotKey(existing?: string): Promise { if (existing) { const keep = await p.confirm({ @@ -243,11 +265,25 @@ export async function promptSecurity(vars: Map): Promise validateEgressEntries({ allowedHosts: value }), }) if (typeof egressHosts === 'string' && egressHosts.trim()) { sim.EGRESS_ALLOWED_HOSTS = egressHosts.trim() } + const existingEgressRanges = vars.get('EGRESS_ALLOWED_IP_RANGES') + const egressRanges = await p.text({ + message: + 'Address ranges on your private network that workflows may reach? (CIDRs, comma-separated)', + placeholder: '10.0.0.0/8,192.168.65.254/32', + initialValue: existingEgressRanges ?? '', + defaultValue: '', + validate: (value) => validateEgressEntries({ allowedRanges: value }), + }) + if (typeof egressRanges === 'string' && egressRanges.trim()) { + sim.EGRESS_ALLOWED_IP_RANGES = egressRanges.trim() + } + const existingAdminKey = vars.get('ADMIN_API_KEY') if (!existingAdminKey || isPlaceholder(existingAdminKey)) { const wantsAdmin = await p.confirm({