From 68cb157b6525b5691969171d80bc02cec6c5df00 Mon Sep 17 00:00:00 2001 From: Mirko Poloni Date: Wed, 12 Aug 2026 13:04:01 +0200 Subject: [PATCH 1/3] fix(adapters): resolve empty credentials to empty, not placeholder resolveString() used `credentials[key] || '{{key}}'`, so an explicitly-supplied empty credential fell back to the literal placeholder instead of resolving. Any API that needs a credential header to be present but blank would receive the raw string "{{VAR_NAME}}" as its value. Only an *absent* key now keeps its placeholder, so the existing "import without credentials, fill them in later" flow is unchanged. Surfaced by the Destatis GENESIS connector, which expects `password: ""` when identifying via a personal API token. Co-Authored-By: Claude Opus 5 (1M context) --- packages/backend/src/adapters/adapters.service.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/adapters/adapters.service.ts b/packages/backend/src/adapters/adapters.service.ts index 212f5fc6..7467ec94 100644 --- a/packages/backend/src/adapters/adapters.service.ts +++ b/packages/backend/src/adapters/adapters.service.ts @@ -152,7 +152,15 @@ export class AdaptersService { credentials?: Record, ): string { if (!credentials) return str; - return str.replace(/\{\{(\w+)\}\}/g, (_, key) => credentials[key] || `{{${key}}}`); + // An explicitly-supplied empty value must resolve to empty, not fall back + // to the literal placeholder — some APIs require a credential header to be + // present but blank (e.g. Destatis GENESIS wants `password: ""` when + // identifying via API token). A `||` fallback here would send the string + // "{{DESTATIS_PASSWORD}}" as the password. Only an *absent* key keeps its + // placeholder, so the operator can still fill it in later. + return str.replace(/\{\{(\w+)\}\}/g, (_, key) => + key in credentials ? credentials[key] : `{{${key}}}`, + ); } /** Deep-replace {{VAR}} placeholders in an object/value */ From 528e01584d776bf24e54274aee04142257439cd6 Mon Sep 17 00:00:00 2001 From: Mirko Poloni Date: Wed, 12 Aug 2026 13:04:13 +0200 Subject: [PATCH 2/3] feat(adapters): support optionalEnvVars in the install modal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The install modal derived its fields from requiredEnvVars and disabled "Import with credentials" while any of them was blank. An adapter whose credential is legitimately optional could therefore not be installed: the only way past the gate was to type something into a field that has to stay empty. Adapters can now declare `optionalEnvVars`. These are prompted with an "(optional)" label but never gate submission, and they are seeded to '' so an untouched optional field still reaches the backend as an empty value — without the key, {{VAR}} would survive resolution and be sent to the target API verbatim. Co-Authored-By: Claude Opus 5 (1M context) --- packages/backend/src/adapters/catalog.ts | 7 ++++ .../src/app/connectors/store/page.tsx | 37 +++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/adapters/catalog.ts b/packages/backend/src/adapters/catalog.ts index cd68ad88..66b70bd5 100644 --- a/packages/backend/src/adapters/catalog.ts +++ b/packages/backend/src/adapters/catalog.ts @@ -200,6 +200,12 @@ export interface AdapterMeta { icon: string; docsUrl: string; requiredEnvVars: string[]; + /** Env vars the connector references but that may legitimately be left blank + * (e.g. Destatis GENESIS needs no password when an API token is used). The + * install modal prompts for these without blocking submission, and submits + * them as an empty string so the placeholder resolves instead of leaking + * into the request verbatim. */ + optionalEnvVars?: string[]; toolCount: number; /** Content-addressed version of the adapter's installable content (tools + * connector meta + instructions). Stamped onto a connector at install @@ -490,6 +496,7 @@ export function listAdapters(): AdapterMeta[] { icon: adapter.icon, docsUrl: adapter.docsUrl, requiredEnvVars: adapter.requiredEnvVars, + optionalEnvVars: adapter.optionalEnvVars, toolCount: adapter.tools.length, version: adapter.version, authType: adapter.connector.authType, diff --git a/packages/frontend/src/app/connectors/store/page.tsx b/packages/frontend/src/app/connectors/store/page.tsx index 21a18097..fcc0dd37 100644 --- a/packages/frontend/src/app/connectors/store/page.tsx +++ b/packages/frontend/src/app/connectors/store/page.tsx @@ -147,10 +147,27 @@ interface AdapterItem { icon: string; docsUrl: string; requiredEnvVars: string[]; + // Env vars that may legitimately stay blank (e.g. Destatis GENESIS needs no + // password when an API token is used). Prompted, but never block Import. + optionalEnvVars?: string[]; toolCount: number; authType?: string; } +/** + * Pre-fill every optional env var with an empty string. An optional var the + * user never touches must still reach the backend as '' — otherwise the + * {{VAR}} placeholder survives resolution and is sent to the target API as a + * literal string (e.g. a Destatis password header of "{{DESTATIS_PASSWORD}}"). + */ +function seedOptionalCredentials(adapter: { + optionalEnvVars?: string[]; +}): Record { + return Object.fromEntries( + (adapter.optionalEnvVars || []).map((v) => [v, '']), + ); +} + interface AdapterDetail extends AdapterItem { // Long-form, Markdown-formatted help authored on the adapter JSON. // Rendered inside the install modal so users see "where to find your @@ -240,7 +257,10 @@ function AdapterStoreContent() { try { const detail = await adapters.get(adapter.slug, token); setConfigAdapter(detail); - setCredentialValues({}); + // Seed optional vars to '' so leaving one blank still submits an empty + // value. Without the key the backend keeps the literal {{VAR}} + // placeholder and sends it to the API verbatim. + setCredentialValues(seedOptionalCredentials(detail)); setRevealedCredentials({}); } catch { // Fallback: use list data @@ -248,7 +268,7 @@ function AdapterStoreContent() { ...adapter, connector: { name: adapter.name, type: 'REST', baseUrl: '', authType: 'API_KEY' }, } as AdapterDetail); - setCredentialValues({}); + setCredentialValues(seedOptionalCredentials(adapter)); setRevealedCredentials({}); } finally { setConfigLoading(false); @@ -531,7 +551,13 @@ function AdapterStoreContent() { )}
- {configAdapter.requiredEnvVars.map((envVar) => { + {[ + ...configAdapter.requiredEnvVars, + ...(configAdapter.optionalEnvVars || []), + ].map((envVar) => { + const isOptional = ( + configAdapter.optionalEnvVars || [] + ).includes(envVar); const isSecret = envVar.toLowerCase().includes('secret') || envVar.toLowerCase().includes('password') || @@ -545,6 +571,11 @@ function AdapterStoreContent() { className="mb-1 block text-sm font-medium" > {formatEnvVarLabel(envVar)} + {isOptional && ( + + (optional) + + )}
Date: Wed, 12 Aug 2026 13:04:31 +0200 Subject: [PATCH 3/3] fix(adapters): migrate Destatis GENESIS to POST/header auth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Destatis permanently shut down GET requests carrying credentials as URL parameters on 30 June 2025. Since then the GENESIS API is reachable only via POST, with credentials as HTTP header fields and all other parameters in an application/x-www-form-urlencoded body. The adapter was still configured for the old mechanism (QUERY_AUTH + GET), so every tool was broken: a GET now returns the GENESIS HTML web interface instead of JSON. The marketplace label "Query Param Auth" was accurate — the configuration behind it was what had gone stale. Changes: - authType QUERY_AUTH -> API_KEY with headerName "username" plus an extraHeaders password field (Destatis expects header fields named literally `username` / `password`, not Authorization: Basic). - All tools: GET + queryParams -> POST + bodyMapping with bodyEncoding "form-urlencoded". - baseUrl -> https://genesis.destatis.de/... (the www-genesis host now answers with a 307). - Personal API token support: the token replaces the username and needs no password, so DESTATIS_USERNAME becomes DESTATIS_USERNAME_OR_TOKEN (required) and DESTATIS_PASSWORD becomes optional. - New destatis_login_check tool + instructions documenting the token's read-only limitation (job=true needs a password), the 3-parallel- request cap and Code 98 on oversized tables. Tool names are unchanged, so existing installs keep working once credentials are re-entered. Also documents a trap found while verifying: unauthenticated find/find does not fail. It answers Code 0 as the GAST guest account with every result list null — indistinguishable from a genuine no-hits search. The data and metadata services fail loudly with Code 15 instead. Tests: destatis-genesis.spec.ts adds 7 always-on static guards pinning POST + header auth + form-urlencoded bodies + the canonical host (5 of them fail against the previous configuration). destatis-genesis.live.spec.ts adds opt-in live coverage via RUN_DESTATIS_LIVE. Co-Authored-By: Claude Opus 5 (1M context) --- content/guides/de/destatis-genesis-to-mcp.mdx | 12 +- content/guides/en/destatis-genesis-to-mcp.mdx | 15 ++- content/guides/it/destatis-genesis-to-mcp.mdx | 14 +- .../src/adapters/de/destatis-genesis.json | 59 +++++++-- .../adapters/de/destatis-genesis.live.spec.ts | 125 ++++++++++++++++++ .../src/adapters/de/destatis-genesis.spec.ts | 118 +++++++++++++++++ scripts/validate-adapters.mjs | 2 +- 7 files changed, 318 insertions(+), 27 deletions(-) create mode 100644 packages/backend/src/adapters/de/destatis-genesis.live.spec.ts create mode 100644 packages/backend/src/adapters/de/destatis-genesis.spec.ts diff --git a/content/guides/de/destatis-genesis-to-mcp.mdx b/content/guides/de/destatis-genesis-to-mcp.mdx index 26c20b4d..616c4759 100644 --- a/content/guides/de/destatis-genesis-to-mcp.mdx +++ b/content/guides/de/destatis-genesis-to-mcp.mdx @@ -20,13 +20,17 @@ git clone https://github.com/HelpCode-ai/anythingmcp.git cd anythingmcp && docker compose up -d ``` -### Schritt 2: Destatis-Adapter importieren +### Schritt 2: GENESIS-Zugangsdaten erhalten -Öffnen Sie `http://localhost:3000/connectors/store` und klicken Sie auf **Import**. Geben Sie Ihren GENESIS-Benutzernamen und Passwort ein. +Registrieren Sie sich kostenlos auf [genesis.destatis.de](https://genesis.destatis.de/). -### Schritt 3: GENESIS-Zugangsdaten erhalten +Destatis empfiehlt die Identifizierung über einen **persönlichen API-Token** anstelle des Passworts: Nach dem Login finden Sie die 32-stellige Zeichenkette im Modal **„Webservice-Schnittstelle (API)"**. Der Token lässt sich unabhängig von Ihren Zugangsdaten zurücksetzen — praktisch bei gemeinsam genutzten Projekten. Beachten Sie: Mit Token sind nur lesende Zugriffe möglich; sehr große Tabellenabfragen, die Destatis in die Warteschlange schreibt (`job=true`), erfordern weiterhin Benutzername und Passwort. -Registrieren Sie sich kostenlos auf [www-genesis.destatis.de](https://www-genesis.destatis.de/). +### Schritt 3: Destatis-Adapter importieren + +Öffnen Sie `http://localhost:3000/connectors/store` und klicken Sie auf **Import**. Tragen Sie **entweder** Ihren API-Token **oder** Ihre Nutzerkennung in das Feld *Username or Token* ein. Das Feld *Password* ist optional — bei Verwendung eines Tokens lassen Sie es leer. + +> **Hinweis zur Authentifizierung:** Destatis hat GET-Anfragen mit Zugangsdaten in der URL zum **30.06.2025** endgültig abgeschaltet. Die API ist seitdem ausschließlich per POST mit Zugangsdaten im HTTP-Header erreichbar; der Adapter ist entsprechend konfiguriert. Kommt eine Tabellensuche leer zurück, rufen Sie zuerst `destatis_login_check` auf: Lassen sich die Zugangsdaten nicht auflösen, antwortet `find/find` weiterhin mit `Code: 0` als Gast-Konto `GAST` — alle Trefferlisten sind dann `null`, was von einer echten Nulltreffer-Suche nicht zu unterscheiden ist. Die Daten- und Metadaten-Dienste scheitern dagegen deutlich mit `Code: 15`. ### Schritt 4: Mit KI-Agent verbinden diff --git a/content/guides/en/destatis-genesis-to-mcp.mdx b/content/guides/en/destatis-genesis-to-mcp.mdx index ab3c7e4a..914c3d58 100644 --- a/content/guides/en/destatis-genesis-to-mcp.mdx +++ b/content/guides/en/destatis-genesis-to-mcp.mdx @@ -20,13 +20,17 @@ git clone https://github.com/HelpCode-ai/anythingmcp.git cd anythingmcp && docker compose up -d ``` -### Step 2: Import the Destatis Adapter +### Step 2: Get GENESIS Credentials -Open `http://localhost:3000/connectors/store` and click **Import** on the Destatis GENESIS Statistics adapter. Enter your GENESIS username and password when prompted. +Register for a free account at [genesis.destatis.de](https://genesis.destatis.de/). -### Step 3: Get GENESIS Credentials +Destatis recommends identifying with a **personal API token** rather than your password: log in, open the **"Webservice-Schnittstelle (API)"** modal and copy the 32-character token. It can be regenerated independently of your password, which is useful for shared projects. Note that a token allows read access only — very large table queries that Destatis pushes into its queue (`job=true`) still require username and password. -Register for a free account at [www-genesis.destatis.de](https://www-genesis.destatis.de/) to get API access credentials. +### Step 3: Import the Destatis Adapter + +Open `http://localhost:3000/connectors/store` and click **Import** on the Destatis GENESIS Statistics adapter. Put **either** your API token **or** your Nutzerkennung into the *Username or Token* field. The *Password* field is optional — leave it blank when using a token. + +> **Authentication note:** Destatis permanently shut down GET requests with credentials in the URL on **30 June 2025**. The API is now reachable only via POST with credentials in the HTTP header, and this adapter is configured accordingly. If a table search comes back empty, run `destatis_login_check` first: when credentials fail to resolve, `find/find` still answers `Code: 0` as the `GAST` guest account with all result lists `null`, which is indistinguishable from a genuine no-hits search. The data and metadata services are stricter and fail loudly with `Code: 15`. ### Step 4: Connect to Your AI Agent @@ -44,9 +48,10 @@ Register for a free account at [www-genesis.destatis.de](https://www-genesis.des | Tool | Description | |------|-------------| +| `destatis_login_check` | Verify credentials and confirm which account the API sees | | `destatis_search_tables` | Search for statistical tables by keyword | | `destatis_get_table` | Retrieve data from a specific statistical table | -| `destatis_get_metadata` | Get metadata and definitions for a table | +| `destatis_get_table_metadata` | Get metadata and definitions for a table | | `destatis_list_statistics` | List all available statistics catalogs | ## AI Agent Use Cases diff --git a/content/guides/it/destatis-genesis-to-mcp.mdx b/content/guides/it/destatis-genesis-to-mcp.mdx index 41356ec6..4b5ad9fd 100644 --- a/content/guides/it/destatis-genesis-to-mcp.mdx +++ b/content/guides/it/destatis-genesis-to-mcp.mdx @@ -20,11 +20,19 @@ git clone https://github.com/HelpCode-ai/anythingmcp.git cd anythingmcp && docker compose up -d ``` -### Passo 2: Importa l'Adattatore Destatis +### Passo 2: Ottieni le Credenziali GENESIS -Apri `http://localhost:3000/connectors/store` e clicca **Import**. Inserisci username e password GENESIS. +Registrati gratuitamente su [genesis.destatis.de](https://genesis.destatis.de/). -### Passo 3: Collega al Tuo Agente AI +Destatis raccomanda di identificarsi con un **token API personale** anziché con la password: dopo il login, copia la stringa di 32 caratteri dal modale **"Webservice-Schnittstelle (API)"**. Il token può essere rigenerato indipendentemente dalle credenziali. Attenzione: con il token è possibile solo l'accesso in lettura; le query su tabelle molto grandi che Destatis accoda (`job=true`) richiedono ancora username e password. + +### Passo 3: Importa l'Adattatore Destatis + +Apri `http://localhost:3000/connectors/store` e clicca **Import**. Inserisci **il token API oppure** la tua Nutzerkennung nel campo *Username or Token*. Il campo *Password* è opzionale — lascialo vuoto se usi un token. + +> **Nota sull'autenticazione:** Destatis ha disattivato definitivamente le richieste GET con credenziali nell'URL il **30 giugno 2025**. L'API è ora raggiungibile solo via POST con le credenziali nell'header HTTP, e l'adattatore è configurato di conseguenza. Se una ricerca di tabelle torna vuota, esegui prima `destatis_login_check`: quando le credenziali non vengono risolte, `find/find` risponde comunque con `Code: 0` come account ospite `GAST` e con tutte le liste di risultati a `null`, indistinguibile da una ricerca senza risultati. I servizi dati e metadati invece falliscono in modo esplicito con `Code: 15`. + +### Passo 4: Collega al Tuo Agente AI ```json { diff --git a/packages/backend/src/adapters/de/destatis-genesis.json b/packages/backend/src/adapters/de/destatis-genesis.json index cf6f1b72..c6ddf4dd 100644 --- a/packages/backend/src/adapters/de/destatis-genesis.json +++ b/packages/backend/src/adapters/de/destatis-genesis.json @@ -5,22 +5,49 @@ "region": "de", "category": "government", "icon": "destatis", - "docsUrl": "https://www-genesis.destatis.de/genesis/misc/GENESIS-Webservices_Einfuehrung.pdf", + "docsUrl": "https://genesis.destatis.de/datenbank/online/docs/GENESIS-Webservices_Einfuehrung.pdf", + "instructions": "This connector uses the GENESIS-Online RESTful/JSON web service (Anwenderdokumentation Version 5.1, 01.06.2026).\n\n**IMPORTANT — POST only.** Destatis permanently shut down GET requests with credentials in the URL on **30 June 2025** (\"Zur Verbesserung des Schutzes Ihrer Nutzerdaten wird die bisher angebotene Möglichkeit der API Nutzung mittels GET Requests am 30. Juni 2025 endgültig abgeschaltet\"). Since then the API is reachable **only via POST**, with credentials sent as HTTP **header** fields and all other parameters in a `application/x-www-form-urlencoded` request body. Every tool here is mapped that way. Do not convert any tool back to GET — a GET now returns the GENESIS HTML web interface instead of JSON.\n\n**Setup**:\n1. Register free at https://genesis.destatis.de (Registrierung).\n2. Recommended: log in and open the modal **\"Webservice-Schnittstelle (API)\"** to copy your personal **API token** (32 characters). Destatis explicitly recommends the token: it can be reset independently of your password and contains no characters that need escaping.\n3. Set `DESTATIS_USERNAME_OR_TOKEN` to **either** your Nutzerkennung/e-mail **or** the API token. When using a token, leave `DESTATIS_PASSWORD` empty — a password is not required. When using a Nutzerkennung, set `DESTATIS_PASSWORD` as well.\n\n**Token limitation**: a token identifies but cannot authenticate write access. Requests with `job=true` (used automatically for very large tables), `profile/password` and `profile/removeresult` require Nutzerkennung + password. All tools in this connector are read-only, so a token covers them — except when a table is large enough to be pushed into the queue.\n\n**Silent guest fallback — watch out**: if credentials are missing or unresolved, `find/find` does **not** return an auth error. It runs as the **`GAST`** guest account and answers `Status.Code = 0` / \"erfolgreich\" with every result list `null` — indistinguishable from a genuine no-hits search. The data, metadata and catalogue services are stricter and fail loudly with `Code = 15` (\"Sie sind nicht berechtigt diesen Service aufzurufen...\"). So: an empty `destatis_search_tables` result, or any `Code = 15`, most likely means the credentials never arrived — call `destatis_login_check` and confirm the returned `Username` is yours and not `GAST`.\n\n**Resource limits** (chapter 1.7 of the docs): a maximum of **3 parallel requests** per account; requests running longer than 15 minutes are terminated. Tables with more than ~40,000 values cannot be downloaded in the dialog and return `Status.Code = 98` — split the query by time range or narrow the classifying variables.\n\n**Reading responses**: every response carries a `Status` block. `Code: 0` means success; a non-zero `Code` carries the reason in `Content`. Table codes look like `12411-0001` (population) or `61111-0001` (consumer price index) — use `destatis_search_tables` to find them before calling `destatis_get_table`.", "requiredEnvVars": [ - "DESTATIS_USERNAME", + "DESTATIS_USERNAME_OR_TOKEN" + ], + "optionalEnvVars": [ "DESTATIS_PASSWORD" ], "connector": { "name": "Destatis GENESIS", "type": "REST", - "baseUrl": "https://www-genesis.destatis.de/genesisWS/rest/2020", - "authType": "QUERY_AUTH", + "baseUrl": "https://genesis.destatis.de/genesisWS/rest/2020", + "authType": "API_KEY", "authConfig": { - "username": "{{DESTATIS_USERNAME}}", - "password": "{{DESTATIS_PASSWORD}}" + "headerName": "username", + "apiKey": "{{DESTATIS_USERNAME_OR_TOKEN}}", + "extraHeaders": { + "password": "{{DESTATIS_PASSWORD}}" + } } }, "tools": [ + { + "name": "destatis_login_check", + "description": "Verify that the configured GENESIS credentials work and report which account the API sees. Use this first when results look wrong — if the returned Username is 'GAST' the credentials did not resolve and you are getting guest-level data.", + "parameters": { + "type": "object", + "properties": { + "language": { + "type": "string", + "description": "Language for status messages: 'de' for German or 'en' for English. Default: 'de'" + } + } + }, + "endpointMapping": { + "method": "POST", + "path": "/helloworld/logincheck", + "bodyEncoding": "form-urlencoded", + "bodyMapping": { + "language": "$language" + } + } + }, { "name": "destatis_search_tables", "description": "Search for statistical tables in the Destatis GENESIS database by keyword. Returns table codes, titles, and metadata. Use this to find the right table before retrieving data.", @@ -41,9 +68,10 @@ ] }, "endpointMapping": { - "method": "GET", + "method": "POST", "path": "/find/find", - "queryParams": { + "bodyEncoding": "form-urlencoded", + "bodyMapping": { "term": "$searchterm", "category": "tables", "language": "$language" @@ -78,9 +106,10 @@ ] }, "endpointMapping": { - "method": "GET", + "method": "POST", "path": "/data/table", - "queryParams": { + "bodyEncoding": "form-urlencoded", + "bodyMapping": { "name": "$name", "startyear": "$startyear", "endyear": "$endyear", @@ -108,9 +137,10 @@ ] }, "endpointMapping": { - "method": "GET", + "method": "POST", "path": "/metadata/table", - "queryParams": { + "bodyEncoding": "form-urlencoded", + "bodyMapping": { "name": "$name", "language": "$language" } @@ -133,9 +163,10 @@ } }, "endpointMapping": { - "method": "GET", + "method": "POST", "path": "/catalogue/statistics", - "queryParams": { + "bodyEncoding": "form-urlencoded", + "bodyMapping": { "selection": "$searchterm", "language": "$language" } diff --git a/packages/backend/src/adapters/de/destatis-genesis.live.spec.ts b/packages/backend/src/adapters/de/destatis-genesis.live.spec.ts new file mode 100644 index 00000000..c2688e02 --- /dev/null +++ b/packages/backend/src/adapters/de/destatis-genesis.live.spec.ts @@ -0,0 +1,125 @@ +import { RestEngine } from '../../connectors/engines/rest.engine'; +import { OAuth2TokenService } from '../../connectors/engines/oauth2-token.service'; +import { LoginTokenService } from '../../connectors/engines/login-token.service'; + +/** + * Live reachability check for the Destatis GENESIS adapter. Skipped unless + * RUN_DESTATIS_LIVE is set AND DESTATIS_USERNAME_OR_TOKEN is provided. + * + * This is the layer that proves what static tests cannot: that the real API + * accepts credentials as HTTP *header* fields and that each endpoint accepts + * its parameters as *body* fields (they used to be query params). + * + * Run with a personal API token (password stays empty): + * RUN_DESTATIS_LIVE=1 DESTATIS_USERNAME_OR_TOKEN=<32-char-token> \ + * npx jest src/adapters/de/destatis-genesis.live.spec.ts + * + * ...or with Nutzerkennung + password: + * RUN_DESTATIS_LIVE=1 DESTATIS_USERNAME_OR_TOKEN= \ + * DESTATIS_PASSWORD= \ + * npx jest src/adapters/de/destatis-genesis.live.spec.ts + * + * NOTE: importing RestEngine currently drags in + * connectors/engines/unblocker-proxy-agent.ts, which does not type-check + * against the installed https-proxy-agent typings. That breakage is + * pre-existing (oxomi.live.spec.ts fails the same way) and blocks this suite + * from running until it is fixed. The static guards in + * destatis-genesis.spec.ts are unaffected. + */ + +const live = + process.env.RUN_DESTATIS_LIVE && process.env.DESTATIS_USERNAME_OR_TOKEN + ? describe + : describe.skip; + +live('destatis-genesis adapter — live GENESIS API reachability', () => { + const oauth = {} as unknown as OAuth2TokenService; + const login = {} as unknown as LoginTokenService; + const engine = new RestEngine(oauth, login); + + const config = { + baseUrl: 'https://genesis.destatis.de/genesisWS/rest/2020', + authType: 'API_KEY', + authConfig: { + headerName: 'username', + apiKey: process.env.DESTATIS_USERNAME_OR_TOKEN as string, + // Empty string when identifying via API token — matching Destatis' + // own Python example (`'password': ""`). + extraHeaders: { password: process.env.DESTATIS_PASSWORD || '' }, + }, + }; + + const call = ( + path: string, + bodyMapping: Record, + params: Record, + ) => + engine.execute( + config, + { method: 'POST', path, bodyEncoding: 'form-urlencoded', bodyMapping }, + params, + ); + + it('logincheck authenticates and does NOT fall back to the GAST guest account', async () => { + const res = (await call( + '/helloworld/logincheck', + { language: '$language' }, + { language: 'de' }, + )) as { Status?: string; Username?: string }; + expect(res).toBeDefined(); + // A missing/unresolved credential does not error — GENESIS silently logs + // the caller in as GAST and returns guest-level data. Assert we are not it. + expect(res.Username).not.toBe('GAST'); + expect(res.Status).toContain('erfolgreich'); + }, 30000); + + it('find/find accepts term/category as body fields', async () => { + const res = (await call( + '/find/find', + { term: '$searchterm', category: 'tables', language: '$language' }, + { searchterm: 'Bevölkerung', language: 'de' }, + )) as { Status?: { Code?: number }; Tables?: unknown[] }; + expect(res.Status?.Code).toBe(0); + expect(Array.isArray(res.Tables)).toBe(true); + }, 30000); + + it('data/table returns a table for a known code', async () => { + const res = (await call( + '/data/table', + { + name: '$name', + startyear: '$startyear', + endyear: '$endyear', + language: '$language', + }, + { + name: '12411-0001', + startyear: '2020', + endyear: '2024', + language: 'de', + }, + )) as { Status?: { Code?: number }; Object?: unknown }; + expect(res.Status?.Code).toBe(0); + expect(res.Object).toBeDefined(); + }, 30000); + + it('metadata/table returns table metadata', async () => { + const res = (await call( + '/metadata/table', + { name: '$name', language: '$language' }, + { name: '12411-0001', language: 'de' }, + )) as { Status?: { Code?: number } }; + expect(res.Status?.Code).toBe(0); + }, 30000); + + it('catalogue/statistics lists statistics with the selection filter omitted', async () => { + // `selection` maps from an optional tool param — when the caller omits it + // the key must drop out of the body rather than be sent as "$searchterm". + const res = (await call( + '/catalogue/statistics', + { selection: '$searchterm', language: '$language' }, + { language: 'de' }, + )) as { Status?: { Code?: number } }; + expect(res.Status?.Code).toBe(0); + }, 30000); +}); diff --git a/packages/backend/src/adapters/de/destatis-genesis.spec.ts b/packages/backend/src/adapters/de/destatis-genesis.spec.ts new file mode 100644 index 00000000..71b0e5df --- /dev/null +++ b/packages/backend/src/adapters/de/destatis-genesis.spec.ts @@ -0,0 +1,118 @@ +import * as adapter from './destatis-genesis.json'; + +/** + * Static conformance guards for the Destatis GENESIS adapter — always run, no + * network and no credentials needed. + * + * These pin the migration off the GET/query-param auth that Destatis + * permanently shut down on 30 June 2025 ("Die GET-Methoden mit Credentials + * wurden durch die bisher parallel angebotenen POST-Methoden der + * RESTful/JSON-Schnittstelle ersetzt" — Anwenderdokumentation 5.1, + * 01.06.2026). Since that date a GET returns the GENESIS HTML web interface + * instead of JSON, so a regression to GET or to QUERY_AUTH silently breaks + * every tool in this adapter. + * + * Live reachability against the real API lives in destatis-genesis.live.spec.ts + * (opt-in, needs an account). + */ + +describe('destatis-genesis adapter — static spec conformance', () => { + const a = adapter as unknown as { + connector: { + baseUrl: string; + authType: string; + authConfig: Record; + }; + requiredEnvVars: string[]; + optionalEnvVars?: string[]; + tools: Array<{ + name: string; + endpointMapping: { + method: string; + path: string; + bodyEncoding?: string; + bodyMapping?: Record; + queryParams?: Record; + }; + }>; + }; + + it('targets the canonical GENESIS host (not the redirecting www- host)', () => { + expect(a.connector.baseUrl).toBe( + 'https://genesis.destatis.de/genesisWS/rest/2020', + ); + // www-genesis.destatis.de now answers with a 307 to the above. + expect(a.connector.baseUrl).not.toContain('www-genesis'); + }); + + it('sends credentials as HTTP headers, never as query params', () => { + // Regression guard: QUERY_AUTH is exactly the shut-down mechanism. + expect(a.connector.authType).not.toBe('QUERY_AUTH'); + expect(a.connector.authType).toBe('API_KEY'); + // Destatis expects header fields literally named `username` / `password` + // — not Authorization: Basic. + expect(a.connector.authConfig.headerName).toBe('username'); + expect(a.connector.authConfig.apiKey).toBe( + '{{DESTATIS_USERNAME_OR_TOKEN}}', + ); + expect(a.connector.authConfig.extraHeaders).toEqual({ + password: '{{DESTATIS_PASSWORD}}', + }); + }); + + it('requires only the username-or-token var; password is optional', () => { + // The API token is placed in the `username` field and needs no password, + // so gating the install on a password would block token-only setups. + expect(a.requiredEnvVars).toEqual(['DESTATIS_USERNAME_OR_TOKEN']); + expect(a.optionalEnvVars).toEqual(['DESTATIS_PASSWORD']); + }); + + it('uses POST with a form-urlencoded body for every tool', () => { + for (const tool of a.tools) { + expect(tool.endpointMapping.method).toBe('POST'); + expect(tool.endpointMapping.bodyEncoding).toBe('form-urlencoded'); + expect(tool.endpointMapping.bodyMapping).toBeDefined(); + // Parameters moved from the query string into the body — a leftover + // queryParams block would put them back in the URL. + expect(tool.endpointMapping.queryParams).toBeUndefined(); + } + }); + + it('never sends credentials inside a tool body or path', () => { + const serialized = JSON.stringify(a.tools); + expect(serialized).not.toContain('DESTATIS_USERNAME'); + expect(serialized).not.toContain('DESTATIS_PASSWORD'); + }); + + it('maps the documented RESTful/JSON endpoints', () => { + const byName = Object.fromEntries( + a.tools.map((t) => [t.name, t.endpointMapping]), + ); + expect(byName['destatis_login_check']).toMatchObject({ + method: 'POST', + path: '/helloworld/logincheck', + }); + expect(byName['destatis_search_tables']).toMatchObject({ + method: 'POST', + path: '/find/find', + }); + expect(byName['destatis_get_table']).toMatchObject({ + method: 'POST', + path: '/data/table', + }); + expect(byName['destatis_get_table_metadata']).toMatchObject({ + method: 'POST', + path: '/metadata/table', + }); + expect(byName['destatis_list_statistics']).toMatchObject({ + method: 'POST', + path: '/catalogue/statistics', + }); + }); + + it('prefixes every tool name with destatis_', () => { + for (const tool of a.tools) { + expect(tool.name.startsWith('destatis_')).toBe(true); + } + }); +}); diff --git a/scripts/validate-adapters.mjs b/scripts/validate-adapters.mjs index 7daa1f0d..b3644192 100644 --- a/scripts/validate-adapters.mjs +++ b/scripts/validate-adapters.mjs @@ -55,7 +55,7 @@ const ALLOWED_AUTH_TYPES = new Set([ 'OAUTH2', 'OAUTH1', // OAuth 1.0a HMAC-SHA1 request signing (e.g. ImmobilienScout24) 'LOGIN_TOKEN', - 'QUERY_AUTH', // existing adapters (destatis, here-geocoding, oxomi) pass the API key as a query string parameter + 'QUERY_AUTH', // existing adapters (here-geocoding, oxomi) pass the API key as a query string parameter ]); const REQUIRED_TOP_LEVEL = [