fix(credentials): age-gate 403 token eviction and tidy the client-credential paths (#9918)

* fix(credentials): age-gate 403 token eviction and stop memoizing transient mint failures

An internal operation's 403 now drops this instance's cached client-credential
token only once it is older than a minute, matching the external-request path,
so a 403 from a file-access refusal or per-record permission no longer re-mints
on every call. The coordinated token's failure memo keeps only refusals of the
credentials themselves; provider_unavailable propagates with no memo row.

* chore(credentials): drop redundant redirect options and correct SailPoint and HubSpot docs

fetchProvider already refuses redirects by default. Zendesk's hidden token param
uses savedCredentialTokenParam, the SailPoint minter takes its token URL from
resolveSailPointHosts, and the SailPoint host and HubSpot fallback TSDoc now
match the code.

* fix(credentials): keep the coordinated token's 30-second backoff for every failed exchange

Remembering only credential refusals in the shared row left the per-instance
mint cache and the Vanta chain still backing off on outages, so the policy
differed by layer. Keep the long-standing 30-second backoff everywhere and
document it.

Author: waleedlatif1

apps/sim/lib/credentials/client-credential-accounts/descriptors.ts

@@ -413,7 +413,8 @@ export function resolveZohoDeskDataCenter(rawDataCenter?: string): {
 
 /**
  * A SailPoint Identity Security Cloud API host: one tenant label under `api.identitynow.com`, or
- * under `api.identitynowgov.com` for government tenants (the same hosts the legacy block accepts).
+ * under `api.identitynowgov.com` for government tenants. Stricter than the legacy block's inline
+ * tenant, which also admits a dotted prefix under either domain.
  */
 const SAILPOINT_API_HOST_REGEX =
   /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.api\.identitynow(?:gov)?\.com$/

apps/sim/lib/credentials/client-credential-accounts/minters/datadog.ts

@@ -67,7 +67,6 @@ export async function mintDatadogServiceAccountToken(
     datadogApiUrl(site, '/api/v1/validate'),
     {
       method: 'GET',
-      redirect: 'error',
       signal: options?.signal,
       headers: { Accept: 'application/json', 'DD-API-KEY': apiKey },
     },

apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.ts

@@ -11,7 +11,11 @@ import {
   requireClientSecret,
   TokenServiceAccountValidationError,
 } from '@/lib/credentials/token-service-accounts/errors'
-import { requestSailPointToken, SailPointAuthError } from '@/lib/internal/sailpoint/client'
+import {
+  requestSailPointToken,
+  resolveSailPointHosts,
+  SailPointAuthError,
+} from '@/lib/internal/sailpoint/client'
 
 const STEP = 'sailpoint_token_mint'
 /** Bounds the whole exchange, including its 429 backoff retries. */
@@ -45,11 +49,12 @@ export async function mintSailPointServiceAccountToken(
     })
   }
 
+  const { tokenUrl } = resolveSailPointHosts(host)
   const timeout = AbortSignal.timeout(TOKEN_EXCHANGE_TIMEOUT_MS)
   let minted: { token: string; expiresInSeconds: number }
   try {
     minted = await requestSailPointToken(
-      `https://${host}/oauth/token`,
+      tokenUrl,
       fields.clientId,
       clientSecret,
       options?.signal ? AbortSignal.any([options.signal, timeout]) : timeout

apps/sim/lib/credentials/client-credential-accounts/minters/zendesk.ts

@@ -94,7 +94,6 @@ async function exchangeClientCredentials(
         scope,
         expires_in: REQUESTED_TOKEN_LIFETIME_SECONDS,
       }),
-      redirect: 'error',
       cache: 'no-store',
       signal,
     },
@@ -135,7 +134,6 @@ async function verifyStaffUser(
     {
       method: 'GET',
       headers: { Accept: 'application/json', Authorization: authorization },
-      redirect: 'error',
       cache: 'no-store',
       signal,
     },

apps/sim/lib/credentials/client-credentials-grant.ts

@@ -70,7 +70,6 @@ export async function exchangeClientCredentialsGrant(
         client_secret: request.clientSecret,
         ...extraFields,
       }).toString(),
-      redirect: 'error',
       cache: 'no-store',
       signal,
     },

apps/sim/lib/credentials/coordinated-client-token.ts

@@ -154,9 +154,11 @@ function isServable(expiresAt: Date): boolean {
  * to reuse one token rather than accumulate them. Minting happens only under a
  * transaction-scoped advisory lock keyed by the application, so two workers never race two
  * tokens into existence; a token minted with a different secret is never served, and a token the
- * provider rejects is expired through {@link invalidateCoordinatedClientToken}. A rejected
- * exchange is remembered for 30 seconds so a hot workflow on a revoked secret does not hammer the
- * token endpoint. Tokens are encrypted at rest.
+ * provider rejects is expired through {@link invalidateCoordinatedClientToken}. An exchange that
+ * fails validation, whether refused for the credentials or answered with an outage, is remembered
+ * for 30 seconds, the same backoff the per-instance mint cache applies, so a hot workflow neither
+ * hammers the token endpoint with a revoked secret nor floods a struggling one. Tokens are
+ * encrypted at rest.
  */
 export async function resolveCoordinatedClientToken(
   input: CoordinatedClientTokenInput

apps/sim/lib/credentials/token-service-accounts/validators/hubspot.ts

@@ -101,11 +101,12 @@ const HUBSPOT_KEY_PREFIX = /^pat-/
  * tokens, so one code path covers both regions.
  *
  * That endpoint is documented only for private app tokens, and live probing
- * shows it returns a bare 404 (HTML) for a token it does not recognize. So a
- * 400, 403 or 404, a 401 for a `pat-` key (which may be a service key), or a
- * 200 without a portal falls back to the Account Information API, and a key
- * that route refuses is settled by a contacts read, the route HubSpot documents
- * for service keys. A token-info 401 for any other token is a rejection.
+ * shows it returns a bare 404 (HTML) for a token it does not recognize. So any
+ * non-transient 4xx other than 401 (408 and 429 are transient), a 401 for a
+ * `pat-` key (which may be a service key), or a 200 without a portal falls back
+ * to the Account Information API, and a key that route refuses is settled by a
+ * contacts read, the route HubSpot documents for service keys. A token-info 401
+ * for any other token is a rejection.
  *
  * The display name is `HubSpot portal {hubId}` when the portal is known — the
  * account-info `uiDomain` is the shared regional host (e.g. `app.hubspot.com`),

apps/sim/lib/credentials/token-service-accounts/validators/linear.ts

@@ -82,7 +82,6 @@ export async function validateLinearServiceAccount(
         'Content-Type': 'application/json',
       },
       body: JSON.stringify({ query: VIEWER_QUERY }),
-      redirect: 'error',
     },
     'viewer'
   )

apps/sim/lib/credentials/token-service-accounts/validators/shopify.ts

@@ -95,7 +95,6 @@ export async function validateShopifyServiceAccount(
         'X-Shopify-Access-Token': fields.apiToken,
       },
       body: JSON.stringify({ query: SHOP_QUERY }),
-      redirect: 'error',
     },
     'shop_query',
     { dnsFailureCode: 'site_not_found', dnsFailureReason: 'store domain does not resolve' }

apps/sim/lib/oauth/credential-service-cancellation.test.ts

@@ -125,4 +125,30 @@ describe('client credential token cache', () => {
     await expect(resolve()).resolves.toEqual({ accessToken: 'replacement-token' })
     expect(mint).toHaveBeenCalledTimes(2)
   })
+  it('keeps a token minted within the minimum age, and drops it once it is older', async () => {
+    vi.useFakeTimers({ toFake: ['Date'] })
+    try {
+      const mint = vi.fn<clientAccounts.ClientCredentialAccountMinter>()
+      mint
+        .mockResolvedValueOnce({ ...token, accessToken: 'refused-token' })
+        .mockResolvedValueOnce({ ...token, accessToken: 'replacement-token' })
+      vi.spyOn(clientAccounts, 'getClientCredentialAccountMinter').mockReturnValue(mint)
+      const resolve = () => {
+        queueTableRows(credential, [storedCredential])
+        return resolveServiceAccountToken('ramp-refused-aged', RAMP_SERVICE_ACCOUNT_PROVIDER_ID)
+      }
+
+      await expect(resolve()).resolves.toEqual({ accessToken: 'refused-token' })
+      vi.advanceTimersByTime(30_000)
+      forgetRejectedClientCredentialToken('refused-token', { minimumAgeMs: 60_000 })
+      await expect(resolve()).resolves.toEqual({ accessToken: 'refused-token' })
+
+      vi.advanceTimersByTime(31_000)
+      forgetRejectedClientCredentialToken('refused-token', { minimumAgeMs: 60_000 })
+      await expect(resolve()).resolves.toEqual({ accessToken: 'replacement-token' })
+      expect(mint).toHaveBeenCalledTimes(2)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
 })