Add OAuth 2.1 sign-in to the hosted MCP server

.env.example

@@ -1,3 +1,17 @@
 SCRAPER_API_TOKEN=<token>
 
-ENABLE_MCPS_LOGGER=false
+ENABLE_MCPS_LOGGER=false
+PORT=3000
+PUBLIC_URL=http://localhost:3000
+OAUTH_STATE_SECRET=<random_32_bytes>
+OAUTH_BACKEND=decodo
+DASHBOARD_AUTHORIZE_URL=https://dashboard.decodo.com/scraper/token-exchange/decision
+TOKEN_EXCHANGE_BASE_URL=https://dashboard.decodo.com/subscription-api/v1/api/scraper/apikey/token-exchange
+TOKEN_EXCHANGE_CLIENT_ID=scrapper-mcp
+TOKEN_EXCHANGE_KID=<kid_registered_in_subscription_api>
+TOKEN_EXCHANGE_PRIVATE_KEY_FILE=/path/to/private.pem
+OAUTH_ALLOWED_REDIRECT_URIS=
+MOCK_SCRAPER_API_KEY=<scraping_api_key>
+TRUST_PROXY=false
+DECODO_SAPI_HOST=
+DECODO_DATA_API_HOST=

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@decodo/mcp-server",
-  "version": "1.2.5",
+  "version": "1.3.0",
   "description": "Decodo MCP Server",
   "bin": {
     "decodo-mcp": "./build/index.js"
@@ -27,7 +27,7 @@
     "build"
   ],
   "dependencies": {
-    "@decodo/sdk-ts": "^2.3.0",
+    "@decodo/sdk-ts": "^2.4.0",
     "@modelcontextprotocol/sdk": "^1.12.3",
     "cors": "^2.8.5",
     "dotenv": "^16.5.0",

server.json

@@ -7,13 +7,13 @@
     "url": "https://github.com/Decodo/mcp-server",
     "source": "github"
   },
-  "version": "1.2.5",
+  "version": "1.3.0",
   "packages": [
     {
       "registryType": "npm",
       "registryBaseUrl": "https://registry.npmjs.org",
       "identifier": "@decodo/mcp-server",
-      "version": "1.2.5",
+      "version": "1.3.0",
       "transport": {
         "type": "stdio"
       }

src/clients/__tests__/hosts.test.ts

@@ -0,0 +1,19 @@
+import { scrapingApiHostsFromEnv } from '../hosts';
+
+describe('scrapingApiHostsFromEnv', () => {
+  it('reads both hosts from env', () => {
+    expect(
+      scrapingApiHostsFromEnv({
+        DECODO_SAPI_HOST: 'https://stage-scraper-api.decodo.com',
+        DECODO_DATA_API_HOST: 'https://stage-scraper-platform-api.cyberbutis.io',
+      })
+    ).toEqual({
+      scraperApi: 'https://stage-scraper-api.decodo.com',
+      dataApi: 'https://stage-scraper-platform-api.cyberbutis.io',
+    });
+  });
+
+  it('leaves unset or blank hosts undefined so the SDK defaults apply', () => {
+    expect(scrapingApiHostsFromEnv({ DECODO_SAPI_HOST: '  ' })).toEqual({ scraperApi: undefined, dataApi: undefined });
+  });
+});

src/clients/__tests__/scraper-api-client.test.ts

@@ -111,6 +111,38 @@ describe('ScraperApiClient', () => {
     });
   });
 
+  describe('scrape - hosts', () => {
+    beforeEach(() => {
+      respondWith(() => scrapeResponse('<html></html>'));
+    });
+
+    it('sends api keys to the configured data API host', async () => {
+      const staged = new ScraperApiClient({ hosts: { dataApi: 'https://stage-data.example' } });
+
+      await staged.scrape({ auth: apiKeyAuth, scrapingParams: { url: 'https://example.com' } });
+
+      expect(lastRequest().url).toBe('https://stage-data.example/v1/scrape');
+    });
+
+    it('sends basic tokens to the configured scraper API host', async () => {
+      const staged = new ScraperApiClient({ hosts: { scraperApi: 'https://stage-scraper-api.example' } });
+
+      await staged.scrape(defaultArgs);
+
+      expect(lastRequest().url).toBe('https://stage-scraper-api.example/v2/scrape');
+    });
+
+    it('keeps the SDK defaults when no host is configured', async () => {
+      const bare = new ScraperApiClient({ hosts: {} });
+
+      await bare.scrape(defaultArgs);
+      expect(lastRequest().url).toBe('https://scraper-api.decodo.com/v2/scrape');
+
+      await bare.scrape({ auth: apiKeyAuth, scrapingParams: { url: 'https://example.com' } });
+      expect(lastRequest().url).toBe('https://data.decodo.com/v1/scrape');
+    });
+  });
+
   describe('transformScrapingParams', () => {
     it('maps jsRender to headless html', () => {
       expect(client.transformScrapingParams({ scrapingParams: { jsRender: true } })).toMatchObject({
@@ -165,6 +197,19 @@ describe('ScraperApiClient', () => {
       );
     });
 
+    it('reports the rejected credential on 401 and only then', async () => {
+      const onAuthenticationError = jest.fn();
+      const reporting = new ScraperApiClient({ maxRetries: 0, onAuthenticationError });
+
+      respondOnceWith(() => jsonResponse({ status: 401, body: { message: 'Unauthorized' } }));
+      await expect(reporting.scrape(defaultArgs)).rejects.toThrow('(401)');
+      expect(onAuthenticationError).toHaveBeenCalledWith(auth);
+
+      respondOnceWith(() => jsonResponse({ status: 502, body: { message: 'Bad gateway' } }));
+      await expect(reporting.scrape(defaultArgs)).rejects.toThrow('(502)');
+      expect(onAuthenticationError).toHaveBeenCalledTimes(1);
+    });
+
     it('surfaces the server message on 429', async () => {
       respondWith(() => jsonResponse({ status: 429, body: { message: 'Rate limit exceeded' } }));
 

src/clients/hosts.ts

@@ -0,0 +1,13 @@
+export type ScrapingApiHosts = {
+  scraperApi?: string;
+  dataApi?: string;
+};
+
+type Env = Record<string, string | undefined>;
+
+const hostFromEnv = (env: Env, key: string): string | undefined => env[key]?.trim() || undefined;
+
+export const scrapingApiHostsFromEnv = (env: Env = process.env): ScrapingApiHosts => ({
+  scraperApi: hostFromEnv(env, 'DECODO_SAPI_HOST'),
+  dataApi: hostFromEnv(env, 'DECODO_DATA_API_HOST'),
+});

src/clients/scraper-api-client.ts

@@ -8,7 +8,9 @@ import {
 import type { ScrapeRequest, SyncResponse } from '@decodo/sdk-ts';
 import { ScrapingMCPParams } from 'types';
 import { AUTH_TYPE } from '../auth';
-import type { AuthCredential, AuthType } from '../auth';
+import { scrapingApiHostsFromEnv } from './hosts';
+import type { ScrapingApiHosts } from './hosts';
+import type { AuthCredential } from '../auth';
 import { ProgressNotifier, ProgressExtra } from '../utils';
 import { log } from '../logger';
 import {
@@ -31,20 +33,32 @@ const API_PARAM_ALIASES = new Map([
   ['deliveryZip', 'delivery_zip'],
 ]);
 
+export type AuthenticationErrorListener = (auth: AuthCredential) => void;
+
 export class ScraperApiClient {
   maxRetries: number;
 
   delayMs: number;
 
+  onAuthenticationError?: AuthenticationErrorListener;
+
+  hosts: ScrapingApiHosts;
+
   constructor({
     maxRetries = MAX_RETRIES,
     delayMs = BASE_RETRY_DELAY_MS,
+    onAuthenticationError,
+    hosts = scrapingApiHostsFromEnv(),
   }: {
     maxRetries?: number;
     delayMs?: number;
+    onAuthenticationError?: AuthenticationErrorListener;
+    hosts?: ScrapingApiHosts;
   } = {}) {
     this.maxRetries = maxRetries;
     this.delayMs = delayMs;
+    this.onAuthenticationError = onAuthenticationError;
+    this.hosts = hosts;
   }
 
   transformScrapingParams = ({
@@ -83,21 +97,26 @@ export class ScraperApiClient {
   private sdkError = ({
     error,
     target,
-    authType,
+    auth,
     startMs,
   }: {
     error: unknown;
     target: string;
-    authType: AuthType;
+    auth: AuthCredential;
     startMs: number;
   }): unknown => {
     const latencyMs = Date.now() - startMs;
+    const authType = auth.type;
     const message = error instanceof Error ? error.message : String(error);
 
     if (error instanceof DecodoError) {
       const sdkMessage =
         error instanceof AuthenticationError ? 'Authentication failed.' : error.message;
 
+      if (error instanceof AuthenticationError) {
+        this.onAuthenticationError?.(auth);
+      }
+
       log('error', 'tool_call', {
         outcome: 'error',
         target,
@@ -139,8 +158,10 @@ export class ScraperApiClient {
     return error;
   };
 
-  private sdkCredentials = (auth: AuthCredential) =>
-    auth.type === AUTH_TYPE.API_KEY ? { apiKey: auth.value } : { token: auth.value };
+  private sdkTransport = (auth: AuthCredential) =>
+    auth.type === AUTH_TYPE.API_KEY
+      ? { apiKey: auth.value, baseUrl: this.hosts.dataApi }
+      : { token: auth.value, baseUrl: this.hosts.scraperApi };
 
   scrape = async <T = string>({
     auth,
@@ -164,7 +185,7 @@ export class ScraperApiClient {
 
       const { webScrapingApi } = new DecodoClient({
         webScrapingApi: {
-          ...this.sdkCredentials(auth),
+          ...this.sdkTransport(auth),
           integrationHeader: INTEGRATION_HEADER,
         },
         timeoutMs: REQUEST_TIMEOUT_MS,
@@ -222,7 +243,7 @@ export class ScraperApiClient {
         }
       }
 
-      throw this.sdkError({ error: lastError, target, authType: auth.type, startMs });
+      throw this.sdkError({ error: lastError, target, auth, startMs });
     } finally {
       notifier.stopWaitingNotifications();
     }

src/oauth/__tests__/bounded-body.test.ts

@@ -0,0 +1,40 @@
+import { readBoundedText } from '../bounded-body';
+
+const streamOf = (chunks: string[], { endless = false, pulls = { count: 0 } } = {}) =>
+  new ReadableStream<Uint8Array>({
+    pull(controller) {
+      pulls.count += 1;
+      const next = chunks.shift();
+
+      if (next !== undefined) {
+        controller.enqueue(new TextEncoder().encode(next));
+      } else if (endless) {
+        controller.enqueue(new Uint8Array(1024));
+      } else {
+        controller.close();
+      }
+    },
+  });
+
+describe('readBoundedText', () => {
+  it('returns the whole body when it fits', async () => {
+    await expect(readBoundedText(new Response(streamOf(['{"a":', '1}'])), 64)).resolves.toBe('{"a":1}');
+  });
+
+  it('returns an empty string for a body-less response', async () => {
+    await expect(readBoundedText(new Response(null, { status: 204 }), 64)).resolves.toBe('');
+  });
+
+  it('rejects a declared length over the limit without reading', async () => {
+    const response = new Response('x'.repeat(10), { headers: { 'content-length': '100000' } });
+
+    await expect(readBoundedText(response, 64)).resolves.toBeUndefined();
+  });
+
+  it('stops reading an endless body once the limit is passed', async () => {
+    const pulls = { count: 0 };
+
+    await expect(readBoundedText(new Response(streamOf([], { endless: true, pulls })), 4096)).resolves.toBeUndefined();
+    expect(pulls.count).toBeLessThan(10);
+  });
+});