fix(credentials): accept scoped keys on save, isolate the retiring signing key, surface Databricks token errors

Author: waleedlatif1

apps/sim/lib/credentials/token-service-accounts/validators/token-checks.test.ts

@@ -3,6 +3,8 @@ import { describe, expect, it, vi } from 'vitest'
 import {
   validateBrexServiceAccount,
   validateFlintServiceAccount,
+  validateInstantlyServiceAccount,
+  validateYouTubeServiceAccount,
 } from '@/lib/credentials/token-service-accounts/validators/token-checks'
 
 describe('Flint saved API key verification', () => {
@@ -92,3 +94,42 @@ describe('Brex saved token verification', () => {
     })
   })
 })
+
+describe('saved key verification for vendors with scoped keys', () => {
+  it('accepts a valid Instantly key that lacks the campaigns scope', async () => {
+    vi.stubGlobal(
+      'fetch',
+      vi.fn().mockResolvedValue(jsonResponse({ message: 'Insufficient scope' }, 403))
+    )
+
+    await expect(validateInstantlyServiceAccount({ apiToken: 'leads-only' })).resolves.toBeTruthy()
+  })
+
+  it('accepts a valid YouTube key that is out of quota for today', async () => {
+    vi.stubGlobal(
+      'fetch',
+      vi
+        .fn()
+        .mockResolvedValue(
+          jsonResponse({ error: { code: 403, errors: [{ reason: 'quotaExceeded' }] } }, 403)
+        )
+    )
+
+    await expect(validateYouTubeServiceAccount({ apiToken: 'AIza-valid' })).resolves.toBeTruthy()
+  })
+
+  it('rejects a YouTube key whose project has the API turned off', async () => {
+    vi.stubGlobal(
+      'fetch',
+      vi
+        .fn()
+        .mockResolvedValue(
+          jsonResponse({ error: { code: 403, errors: [{ reason: 'accessNotConfigured' }] } }, 403)
+        )
+    )
+
+    await expect(validateYouTubeServiceAccount({ apiToken: 'AIza-off' })).rejects.toMatchObject({
+      code: 'invalid_credentials',
+    })
+  })
+})

apps/sim/lib/credentials/token-service-accounts/validators/token-checks.ts

@@ -225,13 +225,17 @@ export const validateDubServiceAccount = tokenCheckValidator({
   forbiddenMeansScoped: true,
 })
 
-/** ElevenLabs returns the key owner's account for a valid key and 401 otherwise. */
+/**
+ * ElevenLabs returns the key owner's account for a valid key and 400/401 for a bad one. A key
+ * restricted to other endpoints gets 403, which still proves it authenticated.
+ */
 export const validateElevenLabsServiceAccount = tokenCheckValidator({
   serviceLabel: 'ElevenLabs',
   request: ({ apiToken }) => ({
     url: 'https://api.elevenlabs.io/v1/user',
     headers: { 'xi-api-key': apiToken, Accept: 'application/json' },
   }),
+  forbiddenMeansScoped: true,
 })
 
 /** Enrich's credit-balance endpoint costs no credits and succeeds only for a valid key. */
@@ -355,13 +359,17 @@ export const validateIncidentioServiceAccount = tokenCheckValidator({
   }),
 })
 
-/** Instantly's campaign list (the one the Instantly tools call) answers 200 for a valid key and 401 otherwise. */
+/**
+ * Instantly's campaign list (the one the Instantly tools call) answers 200 for a valid key and
+ * 401 for a bad one; a valid key without the campaigns scope gets 403.
+ */
 export const validateInstantlyServiceAccount = tokenCheckValidator({
   serviceLabel: 'Instantly',
   request: ({ apiToken }) => ({
     url: 'https://api.instantly.ai/api/v2/campaigns?limit=1',
     headers: { Authorization: `Bearer ${apiToken}`, Accept: 'application/json' },
   }),
+  forbiddenMeansScoped: true,
 })
 
 /** Intercom's identify-admin endpoint answers 200 with the token's admin and 401 to an unknown token. */
@@ -597,22 +605,30 @@ export const validateSportmonksServiceAccount = tokenCheckValidator({
   }),
 })
 
-/** Square's list-locations read answers 200 for a valid production access token and 401 otherwise. */
+/**
+ * Square's list-locations read answers 200 for a valid production access token and 401 for a bad
+ * one; a token without the merchant-profile scope gets 403.
+ */
 export const validateSquareServiceAccount = tokenCheckValidator({
   serviceLabel: 'Square',
   request: ({ apiToken }) => ({
     url: 'https://connect.squareup.com/v2/locations',
     headers: { Authorization: `Bearer ${apiToken}`, Accept: 'application/json' },
   }),
+  forbiddenMeansScoped: true,
 })
 
-/** Listing devices in the token's default tailnet (`-`) answers 200 for a valid token and 401 otherwise. */
+/**
+ * Listing devices in the token's default tailnet (`-`) answers 200 for a valid token and 401 for a
+ * bad one; an OAuth token without the devices scope gets 403.
+ */
 export const validateTailscaleServiceAccount = tokenCheckValidator({
   serviceLabel: 'Tailscale',
   request: ({ apiToken }) => ({
     url: 'https://api.tailscale.com/api/v2/tailnet/-/devices',
     headers: { Authorization: `Bearer ${apiToken}`, Accept: 'application/json' },
   }),
+  forbiddenMeansScoped: true,
 })
 
 /** Tavily's usage endpoint answers 200 for a valid key and 401 otherwise. */
@@ -651,13 +667,28 @@ export const validateYouComServiceAccount = tokenCheckValidator({
   }),
 })
 
-/** Listing video categories costs one quota unit; Google answers 400/403 for an invalid or restricted key. */
+/** Google's 403 reasons that a valid key gets when it is only out of quota for now. */
+const YOUTUBE_QUOTA_REASONS = new Set(['quotaExceeded', 'dailyLimitExceeded', 'rateLimitExceeded'])
+
+/**
+ * Listing video categories costs one quota unit. Google answers 400 for an invalid key and 403
+ * for a restricted one or a project without the YouTube API; a 403 for spent quota still proves
+ * the key, so only that one is accepted.
+ */
 export const validateYouTubeServiceAccount = tokenCheckValidator({
   serviceLabel: 'YouTube',
   request: ({ apiToken }) => ({
     url: `https://www.googleapis.com/youtube/v3/videoCategories?part=snippet&regionCode=US&key=${encodeURIComponent(apiToken)}`,
     headers: { Accept: 'application/json' },
   }),
+  forbiddenMeansScoped: true,
+  rejectsKey: (errorBody) => {
+    const error = toRecord(toRecord(errorBody).error)
+    if (error.code !== 400 && error.code !== 403) return false
+    return !toArray(error.errors).some((entry) =>
+      YOUTUBE_QUOTA_REASONS.has(String(toRecord(entry).reason))
+    )
+  },
 })
 
 /** Listing one thread needs a valid project key; Zep answers 401 for any other. */

apps/sim/lib/credentials/workload-identity/issuer.test.ts

@@ -101,6 +101,20 @@ describe('workload identity issuer', () => {
     expect((await workloadIdentityJwksResponse()).status).toBe(404)
   })
 
+  it('keeps signing with the current key when the retiring key is malformed', async () => {
+    setEnv('WORKLOAD_IDENTITY_PREVIOUS_KEY', 'not a PEM key')
+    const issuer = await loadIssuer()
+    const token = await issuer.mintWorkloadIdentityToken({
+      subject: 'sim:workspace:ws-1:credential:abc123def456',
+      audience: 'sts.amazonaws.com',
+      owner: { workspaceId: 'ws-1' },
+    })
+    const jwks = await issuer.getWorkloadIdentityJwks()
+
+    expect(await issuer.isWorkloadIdentityConfigured()).toBe(true)
+    await expect(jwtVerify(token, createLocalJWKSet(jwks))).resolves.toBeTruthy()
+  })
+
   it('publishes nothing and refuses to sign when no signing key is configured', async () => {
     setEnv('WORKLOAD_IDENTITY_SIGNING_KEY', undefined)
     const issuer = await loadIssuer()

apps/sim/lib/credentials/workload-identity/issuer.ts

@@ -55,6 +55,21 @@ async function toPublicJwk(key: KeyObject): Promise<JWK> {
   return { ...exported, kid, alg: WORKLOAD_IDENTITY_ALGORITHM, use: 'sig' }
 }
 
+/**
+ * The retiring key's public JWK, or null when it is unset or unusable. A bad
+ * retiring key only drops that key from the JWKS; the current key keeps signing.
+ */
+async function loadPreviousPublicJwk(): Promise<JWK | null> {
+  const previousKey = env.WORKLOAD_IDENTITY_PREVIOUS_KEY
+  if (!previousKey) return null
+  try {
+    return await toPublicJwk(createPublicKey(normalizePem(previousKey)))
+  } catch (error) {
+    logger.error('Workload identity previous key is set but unusable', { error })
+    return null
+  }
+}
+
 let signingMaterial: Promise<SigningMaterial> | undefined
 
 /**
@@ -70,11 +85,8 @@ function loadSigningMaterial(): Promise<SigningMaterial> {
       const privateKey = createPrivateKey(normalizePem(signingKey))
       const current = await toPublicJwk(privateKey)
       const keys = [current]
-      const previousKey = env.WORKLOAD_IDENTITY_PREVIOUS_KEY
-      if (previousKey) {
-        const previous = await toPublicJwk(createPublicKey(normalizePem(previousKey)))
-        if (previous.kid !== current.kid) keys.push(previous)
-      }
+      const previous = await loadPreviousPublicJwk()
+      if (previous && previous.kid !== current.kid) keys.push(previous)
       return { privateKey, kid: current.kid as string, jwks: { keys } }
     })()
     signingMaterial.catch((error: unknown) => {

apps/sim/lib/oauth/token-resolution.ts

@@ -435,6 +435,11 @@ export async function resolveCredentialToken(
             }
           }
         }
+        // Its messages name the trust, secret, or host problem and carry no secrets. Matched by
+        // name because the Databricks module is loaded lazily, off this module's graph.
+        if (error instanceof Error && error.name === 'DatabricksCredentialError') {
+          return { ok: false, status: 401, error: error.message }
+        }
         return { ok: false, status: 401, error: 'Failed to get service account token' }
       }
     }