From a4238ba54d1d35c77c24bbe0b3c1bd237db4d9d8 Mon Sep 17 00:00:00 2001 From: esgn <5435148+esgn@users.noreply.github.com> Date: Fri, 2 Oct 2026 17:43:04 +0200 Subject: [PATCH] test: remove the e2e agent tests, now in geocontext-test End-to-end tests with an LLM agent live in https://github.com/ignfab/geocontext-test. Remove test/integration/level2-agent, its helpers and vitest.e2e.config.mts, the test:e2e and verify:full scripts, the LangChain dev dependencies only they used, their documentation and the .env.e2e entry of .gitignore, which is regrouped by purpose. --- .gitignore | 14 +- docs/config/corporate-proxy.md | 1 - docs/dev.md | 53 +- package-lock.json | 504 ------------------ package.json | 9 - test/integration/config/level2-agent.ts | 79 --- test/integration/helpers/level2-agent.ts | 53 -- test/integration/helpers/level2-assertions.ts | 153 ------ test/integration/helpers/level2-scenarios.ts | 133 ----- .../level2-agent/level2-agent.test.ts | 128 ----- vitest.e2e.config.mts | 24 - 11 files changed, 15 insertions(+), 1136 deletions(-) delete mode 100644 test/integration/config/level2-agent.ts delete mode 100644 test/integration/helpers/level2-agent.ts delete mode 100644 test/integration/helpers/level2-assertions.ts delete mode 100644 test/integration/helpers/level2-scenarios.ts delete mode 100644 test/integration/level2-agent/level2-agent.test.ts delete mode 100644 vitest.e2e.config.mts diff --git a/.gitignore b/.gitignore index d422f388..63e01807 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,16 @@ +# Dependencies and build output /node_modules/ +/dist/ /coverage/ + +# Local scratch files and logs +/tmp/ /*.log -/dist/ + +# Editor and tool settings /.vscode/ -.npmrc .codex -.env.e2e + +# Secrets and local configuration .env -/tmp/ +.npmrc diff --git a/docs/config/corporate-proxy.md b/docs/config/corporate-proxy.md index d5c75f48..59f2c305 100644 --- a/docs/config/corporate-proxy.md +++ b/docs/config/corporate-proxy.md @@ -12,7 +12,6 @@ Le support du proxy est activé par l'environnement dans les principaux contexte - En exécution locale, le serveur démarre avec `node --use-env-proxy`. - Les tests d'intégration propagent `NODE_USE_ENV_PROXY=1` au sous-processus MCP lancé en `stdio`. -- Les tests E2E démarrent les workers Vitest avec `--use-env-proxy`. Il suffit ensuite de définir les variables d'environnement standard selon votre contexte réseau : diff --git a/docs/dev.md b/docs/dev.md index 3c80d322..40b5ed45 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -112,13 +112,14 @@ npm run inspect:mcp:cli # mode CLI ## Tests -Le projet distingue trois niveaux de tests : +Le projet distingue deux niveaux de tests : - **Unitaires** : pas de réseau, exécutés par défaut. - **Intégration niveau 1** (`test/integration/level1-protocol`) : appels MCP directs vers les tools, avec de vrais appels réseau vers la Géoplateforme. -- **E2E niveau 2** (`test/integration/level2-agent`) : un agent LangChain branché au serveur MCP local avec un vrai modèle LLM. -Les niveaux 1 et 2 nécessitent un build à jour (`npm run build`) et un accès réseau aux services appelés. Les deux suites s'exécutent séquentiellement pour limiter la charge sur les services externes et éviter de démarrer plusieurs serveurs MCP en parallèle. +Les tests d'intégration nécessitent un build à jour (`npm run build`) et un accès réseau aux services appelés. Ils s'exécutent séquentiellement pour limiter la charge sur les services externes et éviter de démarrer plusieurs serveurs MCP en parallèle. + +Les tests de bout en bout, avec un agent et un vrai modèle LLM, sont dans le dépôt [geocontext-test](https://github.com/ignfab/geocontext-test). ### Vue d'ensemble des commandes @@ -129,12 +130,10 @@ Les niveaux 1 et 2 nécessitent un build à jour (`npm run build`) et un accès | `npm test` / `test:unit` | Tests unitaires | | `npm run test:perf` | Tests chronométrés (`*.perf.test.ts`), un fichier à la fois | | `npm run test:integration` | Tests d'intégration niveau 1 | -| `npm run test:e2e` | Tests E2E agent niveau 2 | | `npm run test:coverage` | Tests unitaires avec couverture | | `npm run bench` | Benchmark du calcul de `intersection_area` | | `npm run verify:fast` | `typecheck` + `typecheck:test` + `build` + `test:unit` + `test:perf` | | `npm run verify` | `verify:fast` + `test:integration` | -| `npm run verify:full` | `verify` + `test:e2e` | ### Tests unitaires @@ -159,13 +158,6 @@ npm run build npm run test:integration ``` -### Tests E2E agent (niveau 2) - -```bash -npm run build -npm run test:e2e -``` - ### Couverture ```bash @@ -177,12 +169,11 @@ npm run test:coverage ```bash npm run verify:fast # typecheck + build + tests unitaires et chronométrés npm run verify # verify:fast + tests d'intégration niveau 1 -npm run verify:full # verify + tests E2E niveau 2 ``` ### Variables d'environnement -Communes aux suites d'intégration (niveaux 1 et 2) : +Pour les tests d'intégration : | Variable | Description | | ----------------------------------------- | ------------------------------------------------------------------- | @@ -190,43 +181,9 @@ Communes aux suites d'intégration (niveaux 1 et 2) : | `GEOCONTEXT_LOG_LEVEL` | Niveau de log du serveur lancé par les tests | | `HTTP_PROXY` / `HTTPS_PROXY` / `NO_PROXY` | Configuration proxy réseau | -Spécifiques aux tests E2E agent (`test:e2e`) : - -| Variable | Description | -| ------------------- | ------------------------------------------------------------------- | -| `MODEL_NAME` | Modèle LangChain à utiliser (défaut : `anthropic:claude-haiku-4-5`) | -| `ANTHROPIC_API_KEY` | Clé API Anthropic | -| `OPENAI_API_KEY` | Clé API OpenAI | -| `GOOGLE_API_KEY` | Clé API Google | -| `MISTRAL_API_KEY` | Clé API Mistral | - -La clé API requise dépend du provider indiqué dans `MODEL_NAME`. - -### Exemples de lancement des tests E2E - -Avec Anthropic : - -```bash -export MODEL_NAME=anthropic:claude-haiku-4-5 -export ANTHROPIC_API_KEY=... -npm run build -npm run test:e2e -``` - -Avec Ollama en local : - -```bash -export MODEL_NAME=ollama:llama3.1 -export OLLAMA_BASE_URL=http://127.0.0.1:11434 -npm run build -npm run test:e2e -``` - ## Dépannage - Si `test:integration` échoue immédiatement : vérifier que `dist/index.js` existe (`npm run build`). -- Si `test:e2e` est ignoré : vérifier que la clé API attendue par `MODEL_NAME` est définie. -- Si un provider local (Ollama, etc.) est utilisé derrière un proxy : ajouter `NO_PROXY=localhost,127.0.0.1`. ## Commandes utiles diff --git a/package-lock.json b/package-lock.json index aeb74494..06be280a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,12 +32,6 @@ "geocontext-proxy": "dist/proxy/index.js" }, "devDependencies": { - "@langchain/anthropic": "^1.5.11", - "@langchain/core": "^1.2.14", - "@langchain/google-genai": "^2.3.2", - "@langchain/langgraph": "^1.4.18", - "@langchain/mcp-adapters": "^2.0.0", - "@langchain/ollama": "^1.3.0", "@modelcontextprotocol/inspector": "^2.9.0", "@turf/intersect": "^7.4.0", "@types/geojson": "^7946.0.16", @@ -46,7 +40,6 @@ "@vitest/coverage-v8": "^5.0.3", "ajv": "^8.20.0", "ajv-formats": "^3.0.1", - "langchain": "^1.5.15", "supertest": "^7.3.0", "typescript": "^7.0.2", "vitest": "^5.0.3" @@ -69,28 +62,6 @@ "node": ">=18" } }, - "node_modules/@anthropic-ai/sdk": { - "version": "0.122.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.122.0.tgz", - "integrity": "sha512-GGPNftt0caaz9MDlmNQGHX8855Ojaduyy5pm9Sm1h7HalCn0cWNb5/bweadJF+4yzbal+QL6ztBa09WAAOzLmQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "json-schema-to-ts": "^3.1.1", - "standardwebhooks": "^1.0.0" - }, - "bin": { - "anthropic-ai-sdk": "bin/cli" - }, - "peerDependencies": { - "zod": "^3.25.0 || ^4.0.0" - }, - "peerDependenciesMeta": { - "zod": { - "optional": true - } - } - }, "node_modules/@babel/helper-string-parser": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", @@ -127,16 +98,6 @@ "node": ">=6.0.0" } }, - "node_modules/@babel/runtime": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", - "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/@babel/types": { "version": "7.29.8", "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.8.tgz", @@ -192,13 +153,6 @@ "node": ">=14.0.0" } }, - "node_modules/@cfworker/json-schema": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", - "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", - "devOptional": true, - "license": "MIT" - }, "node_modules/@colors/colors": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", @@ -262,16 +216,6 @@ "lodash.escaperegexp": "^4.1.2" } }, - "node_modules/@google/generative-ai": { - "version": "0.24.1", - "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.24.1.tgz", - "integrity": "sha512-MqO+MLfM6kjxcKoy0p1wRzG3b4ZZXtPI+z2IE26UogS2Cm/XHO+7gGRBh6gcJsOiIVoH93UwKvW4HdgiOZCy9Q==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@hono/node-server": { "version": "2.0.12", "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.0.12.tgz", @@ -329,214 +273,6 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@langchain/anthropic": { - "version": "1.5.11", - "resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-1.5.11.tgz", - "integrity": "sha512-NunWC6u8A/h5VnRk/WvyFkvN3RtdONUWV16u2i/EcEvhtjL2ZhJBPjjSFhm/7vyjtwdTyDhp7Uny75FHB7JXEQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@anthropic-ai/sdk": "^0.122.0", - "zod": "^3.25.76 || ^4" - }, - "engines": { - "node": ">=20" - }, - "peerDependencies": { - "@langchain/core": "^1.2.12" - } - }, - "node_modules/@langchain/core": { - "version": "1.2.14", - "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.2.14.tgz", - "integrity": "sha512-NP+FZMDO8kzVf79kiVHByveErVbTc1kUm0OvJuASxy5dt2xo/m7mi4O0npskU71A9h0vke3YUDYEciiAsXdCoA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@cfworker/json-schema": "^4.0.2", - "@standard-schema/spec": "^1.1.0", - "js-tiktoken": "^1.0.12", - "langsmith": ">=0.5.0 <1.0.0", - "mustache": "^4.2.0", - "p-queue": "^6.6.2", - "zod": "^3.25.76 || ^4" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/@langchain/google-genai": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/@langchain/google-genai/-/google-genai-2.3.2.tgz", - "integrity": "sha512-wiiHx3aBJmWJcKmYs/4xZo5KNRICYjdbwegtmH8ega7liYFKBY6e1F4ocMNL+ylApaRIYeO54ceAHNbKmRVHeQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@google/generative-ai": "^0.24.1" - }, - "engines": { - "node": ">=20" - }, - "peerDependencies": { - "@langchain/core": "^1.2.11" - } - }, - "node_modules/@langchain/langgraph": { - "version": "1.4.18", - "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-1.4.18.tgz", - "integrity": "sha512-yrMMJ9hk2NVMD2xU2WoVrgFawAU6s/RzEl/dX9BhlyxZHazo1wHY35z4K2BIBzTUh4z3giCzrUoqRAqW2CWpWg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@langchain/langgraph-checkpoint": "^1.1.5", - "@langchain/langgraph-sdk": "~1.12.0", - "@langchain/protocol": "^0.0.19", - "@standard-schema/spec": "1.1.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@langchain/core": "^1.1.48", - "zod": "^3.25.32 || ^4.2.0" - } - }, - "node_modules/@langchain/langgraph-checkpoint": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@langchain/langgraph-checkpoint/-/langgraph-checkpoint-1.1.5.tgz", - "integrity": "sha512-BwDwl5VeTOh6CVuiIPgsUgfK51vTJDMSbFcSCUfjJWsl8/DPdK/mbv+ejxJstkSk/BlSPMP4JfXWcN6jD2ea2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@langchain/core": "^1.1.48" - } - }, - "node_modules/@langchain/langgraph-sdk": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-1.12.0.tgz", - "integrity": "sha512-F3AOZjKZRUGmE3FzY+RaVZI6WzXOkwnuF7jqgto6rDPSnO9OdVdYGvwGDi3jjRGf5xLoDtX2dpsR9z5KptU+5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@langchain/protocol": "^0.0.19", - "@types/json-schema": "^7.0.15", - "p-queue": "^9.0.1", - "p-retry": "^7.1.1" - }, - "peerDependencies": { - "@langchain/core": "^1.1.48", - "react": "^18 || ^19", - "react-dom": "^18 || ^19" - }, - "peerDependenciesMeta": { - "react": { - "optional": true - }, - "react-dom": { - "optional": true - } - } - }, - "node_modules/@langchain/langgraph-sdk/node_modules/eventemitter3": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", - "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@langchain/langgraph-sdk/node_modules/p-queue": { - "version": "9.3.3", - "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.3.3.tgz", - "integrity": "sha512-NXAOdnEe5FsZJfT4oK84lE1Y5cFFdWlRuOo5tww8DyNMxyRXwn39fIkUtNLKppcPC+UYU/bXujNCUGDv01y7CA==", - "dev": true, - "license": "MIT", - "dependencies": { - "eventemitter3": "^5.0.4", - "p-timeout": "^7.0.0" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@langchain/langgraph-sdk/node_modules/p-timeout": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-7.0.2.tgz", - "integrity": "sha512-prbX4Z3YszrFNgH+MW5Zoeq3baXrMtP/MQnFeET90UB/GtGcGDQ5Usg9OCy6ETjTTntOw1SL2z9fMPUppN3Guw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@langchain/mcp-adapters": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@langchain/mcp-adapters/-/mcp-adapters-2.0.0.tgz", - "integrity": "sha512-0h5KzZVqSORz7JAQ0XlA8L+jvN+6mealM+60AdNs62i7vQBCkCTXG/PL7gLuiiPz72mIEzlAb662NL8serAQ+g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@modelcontextprotocol/client": "^2.2.0", - "@modelcontextprotocol/core": "^2.2.0", - "zod": "^4.4.3" - }, - "engines": { - "node": ">=20.10.0" - }, - "peerDependencies": { - "@langchain/core": "^1.2.6", - "@langchain/langgraph": "^1.4.13" - }, - "peerDependenciesMeta": { - "@langchain/core": { - "optional": false - }, - "@langchain/langgraph": { - "optional": false - } - } - }, - "node_modules/@langchain/mcp-adapters/node_modules/zod": { - "version": "4.6.5", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", - "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/@langchain/ollama": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@langchain/ollama/-/ollama-1.3.0.tgz", - "integrity": "sha512-uzLkZVTXN0SI5zAoJbJzL4y/QN7gbmJ1txBgFfqJ5M0Qf+Yu2Zu8q98L3jIrfM+akWMaNmRU4R4APhcfQmeIsw==", - "dev": true, - "license": "MIT", - "dependencies": { - "ollama": "^0.6.3" - }, - "engines": { - "node": ">=20" - }, - "peerDependencies": { - "@langchain/core": "^1.0.0" - } - }, - "node_modules/@langchain/protocol": { - "version": "0.0.19", - "resolved": "https://registry.npmjs.org/@langchain/protocol/-/protocol-0.0.19.tgz", - "integrity": "sha512-9hKcRrH7cBX6gfutdfXPoft1OCchHe4FEpALoDJMl5Qu+n/YG5ynZmyu8+8cxORlPwHBoKTxggvXz+76M1yX1Q==", - "dev": true, - "license": "MIT" - }, "node_modules/@modelcontextprotocol/client": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@modelcontextprotocol/client/-/client-2.2.0.tgz", @@ -1373,13 +1109,6 @@ "text-hex": "1.0.x" } }, - "node_modules/@stablelib/base64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz", - "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", - "dev": true, - "license": "MIT" - }, "node_modules/@standard-schema/spec": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", @@ -1631,13 +1360,6 @@ "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", "license": "MIT" }, - "node_modules/@types/json-schema": { - "version": "7.0.15", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", - "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/jsonwebtoken": { "version": "9.0.10", "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.10.tgz", @@ -2336,27 +2058,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, "node_modules/bignumber.js": { "version": "9.3.1", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", @@ -3020,13 +2721,6 @@ "node": ">= 0.6" } }, - "node_modules/eventemitter3": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", - "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", - "dev": true, - "license": "MIT" - }, "node_modules/eventsource": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", @@ -3158,13 +2852,6 @@ "dev": true, "license": "MIT" }, - "node_modules/fast-sha256": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", - "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", - "dev": true, - "license": "Unlicense" - }, "node_modules/fast-uri": { "version": "3.1.8", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", @@ -3751,19 +3438,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/is-network-error": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.2.tgz", - "integrity": "sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/is-plain-obj": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", @@ -3837,16 +3511,6 @@ "url": "https://github.com/sponsors/panva" } }, - "node_modules/js-tiktoken": { - "version": "1.0.21", - "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", - "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", - "dev": true, - "license": "MIT", - "dependencies": { - "base64-js": "^1.5.1" - } - }, "node_modules/js-tokens": { "version": "10.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", @@ -3876,20 +3540,6 @@ "js-yaml": "bin/js-yaml.mjs" } }, - "node_modules/json-schema-to-ts": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", - "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/runtime": "^7.18.3", - "ts-algebra": "^2.0.0" - }, - "engines": { - "node": ">=16" - } - }, "node_modules/json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -3997,59 +3647,6 @@ "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==", "license": "MIT" }, - "node_modules/langchain": { - "version": "1.5.15", - "resolved": "https://registry.npmjs.org/langchain/-/langchain-1.5.15.tgz", - "integrity": "sha512-AlXdwsNf9e3e9QbGkW+qgII3QNdr/tntQPijobQAt5rNG/4O6ggOpVkH79+CegFDPFpR0/8LYAJbP54jCHaqmA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@langchain/langgraph": "^1.4.13", - "@langchain/langgraph-checkpoint": "^1.1.5", - "langsmith": ">=0.5.0 <1.0.0", - "zod": "^3.25.76 || ^4" - }, - "engines": { - "node": ">=20" - }, - "peerDependencies": { - "@langchain/core": "^1.2.14" - } - }, - "node_modules/langsmith": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.6.0.tgz", - "integrity": "sha512-GGaj5IMRfLv2HXXFzGk9diISMYLTpSTh6fzCZGKxWYW/NqEztIFtnXLq6G/RVhzFRmCykLap1fuC67LVKoQLcg==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-queue": "6.6.2" - }, - "peerDependencies": { - "@opentelemetry/api": "*", - "@opentelemetry/exporter-trace-otlp-proto": "*", - "@opentelemetry/sdk-trace-base": "*", - "openai": "*", - "ws": ">=7" - }, - "peerDependenciesMeta": { - "@opentelemetry/api": { - "optional": true - }, - "@opentelemetry/exporter-trace-otlp-proto": { - "optional": true - }, - "@opentelemetry/sdk-trace-base": { - "optional": true - }, - "openai": { - "optional": true - }, - "ws": { - "optional": true - } - } - }, "node_modules/lightningcss": { "version": "1.33.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", @@ -4638,16 +4235,6 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, - "node_modules/mustache": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", - "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", - "dev": true, - "license": "MIT", - "bin": { - "mustache": "bin/mustache" - } - }, "node_modules/nanoid": { "version": "3.3.18", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", @@ -4789,16 +4376,6 @@ "node": ">=12.20.0" } }, - "node_modules/ollama": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/ollama/-/ollama-0.6.3.tgz", - "integrity": "sha512-KEWEhIqE5wtfzEIZbDCLH51VFZ6Z3ZSa6sIOg/E/tBV8S51flyqBOXi+bRxlOYKDf8i327zG9eSTb8IJxvm3Zg==", - "dev": true, - "license": "MIT", - "dependencies": { - "whatwg-fetch": "^3.6.20" - } - }, "node_modules/on-exit-leak-free": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", @@ -4874,16 +4451,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/p-finally": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/p-limit": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-4.0.0.tgz", @@ -4914,52 +4481,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/p-queue": { - "version": "6.6.2", - "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", - "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "eventemitter3": "^4.0.4", - "p-timeout": "^3.2.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-retry": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-7.1.1.tgz", - "integrity": "sha512-J5ApzjyRkkf601HpEeykoiCvzHQjWxPAHhyjFcEUP2SWq0+35NKh8TLhpLw+Dkq5TZBFvUM6UigdE9hIVYTl5w==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-network-error": "^1.1.0" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-timeout": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", - "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-finally": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/parse-ms": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-4.0.0.tgz", @@ -5714,17 +5235,6 @@ "node": ">=10" } }, - "node_modules/standardwebhooks": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.1.1.tgz", - "integrity": "sha512-bCbX9ZEyFkWPsRz7Bl3NuQUJohmwGSev/yhr7vhaGPlc4AfIrspIRa6cPTBuI1ItmrTDJ4d/S2hCsfe4+vQGnQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@stablelib/base64": "^1.0.0", - "fast-sha256": "^1.3.0" - } - }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -5955,13 +5465,6 @@ "node": ">= 14.0.0" } }, - "node_modules/ts-algebra": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", - "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", - "dev": true, - "license": "MIT" - }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -6272,13 +5775,6 @@ "node": ">= 8" } }, - "node_modules/whatwg-fetch": { - "version": "3.6.20", - "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz", - "integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==", - "dev": true, - "license": "MIT" - }, "node_modules/when-exit": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/when-exit/-/when-exit-2.1.5.tgz", diff --git a/package.json b/package.json index 02bc3865..8c28df62 100644 --- a/package.json +++ b/package.json @@ -43,12 +43,10 @@ "test:unit": "vitest run", "test:perf": "vitest run --config vitest.perf.config.mts", "test:integration": "vitest run --config vitest.integration.config.mts", - "test:e2e": "vitest run --config vitest.e2e.config.mts", "test:coverage": "vitest run --coverage", "bench": "vitest bench --run --reporter=verbose", "verify:fast": "npm run typecheck && npm run typecheck:test && npm run build && npm run test:unit && npm run test:perf", "verify": "npm run verify:fast && npm run test:integration", - "verify:full": "npm run verify && npm run test:e2e", "fresh": "npm run reset:local && npm ci && npm run verify", "deps:outdated": "npm outdated", "deps:audit": "npm audit", @@ -75,12 +73,6 @@ "zod": "^3.25.76" }, "devDependencies": { - "@langchain/anthropic": "^1.5.11", - "@langchain/core": "^1.2.14", - "@langchain/google-genai": "^2.3.2", - "@langchain/langgraph": "^1.4.18", - "@langchain/mcp-adapters": "^2.0.0", - "@langchain/ollama": "^1.3.0", "@modelcontextprotocol/inspector": "^2.9.0", "@turf/intersect": "^7.4.0", "@types/geojson": "^7946.0.16", @@ -89,7 +81,6 @@ "@vitest/coverage-v8": "^5.0.3", "ajv": "^8.20.0", "ajv-formats": "^3.0.1", - "langchain": "^1.5.15", "supertest": "^7.3.0", "typescript": "^7.0.2", "vitest": "^5.0.3" diff --git a/test/integration/config/level2-agent.ts b/test/integration/config/level2-agent.ts deleted file mode 100644 index ec1a1af1..00000000 --- a/test/integration/config/level2-agent.ts +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Configuration for Level 2 (Agent E2E) integration tests. - * - * Equivalent of the Python config.py from geocontext-test. - * - * Environment variables: - * - MODEL_NAME: model identifier (default: "anthropic:claude-haiku-4-5") - * - Provider key depending on MODEL_NAME prefix (ex: ANTHROPIC_API_KEY, GOOGLE_API_KEY, OPENAI_API_KEY) - * - HTTPS_PROXY / HTTP_PROXY: proxy used for model provider calls - */ - -import { initChatModel } from "langchain"; -import { MCPAdapter } from "@langchain/mcp-adapters"; -import { INTEGRATION_CONFIG, MILLISECONDS } from "./shared.js"; - -/** Model name for the LLM */ -export const MODEL_NAME = process.env.MODEL_NAME ?? "anthropic:claude-haiku-4-5"; - -function requiredApiKeyEnvFromModelName(modelName: string): string | undefined { - const [provider] = modelName.split(":", 1); - - if (provider === "anthropic") return "ANTHROPIC_API_KEY"; - if (provider === "google_genai" || provider === "google") return "GOOGLE_API_KEY"; - if (provider === "openai") return "OPENAI_API_KEY"; - if (provider === "mistralai" || provider === "mistral") return "MISTRAL_API_KEY"; - - return undefined; -} - -/** System prompt for the agent */ -export const SYSTEM_PROMPT = [ - "Use the tools to answer question about geospatial data.", -].join("\n"); - -/** Timeout for agent E2E tests (2 minutes) */ -export const E2E_TIMEOUT = 120 * MILLISECONDS; - -/** Timeout for server connection */ -export const CONNECT_TIMEOUT = 30 * MILLISECONDS; - -export const MODEL_PROVIDER_API_KEY_ENV = requiredApiKeyEnvFromModelName(MODEL_NAME); -export const HAS_MODEL_PROVIDER_API_KEY = MODEL_PROVIDER_API_KEY_ENV - ? Boolean(process.env[MODEL_PROVIDER_API_KEY_ENV]) - : true; - -/** - * Create a provider-agnostic LangChain chat model instance. - */ -export async function createModel() { - if (!HAS_MODEL_PROVIDER_API_KEY) { - throw new Error( - `${MODEL_PROVIDER_API_KEY_ENV} is required to run level2-agent integration tests with MODEL_NAME=\"${MODEL_NAME}\".`, - ); - } - - return initChatModel(MODEL_NAME, { - temperature: 0, - }); -} - -/** - * Create an MCPAdapter connected to the geocontext server via stdio. - * - * Uses the local built geocontext server entrypoint. Tool names are not prefixed - * with the server name (`geocode`, not `geocontext__geocode`), as in `requiredToolCalls`. - */ -export function createMcpClient(): MCPAdapter { - return new MCPAdapter({ - prefixToolNameWithServerName: false, - servers: { - geocontext: { - transport: "stdio", - command: INTEGRATION_CONFIG.serverCommand, - args: [...INTEGRATION_CONFIG.serverArgs], - env: INTEGRATION_CONFIG.serverEnv(), - }, - }, - }); -} diff --git a/test/integration/helpers/level2-agent.ts b/test/integration/helpers/level2-agent.ts deleted file mode 100644 index ecb7c067..00000000 --- a/test/integration/helpers/level2-agent.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { createAgent } from "langchain"; -import { HumanMessage } from "@langchain/core/messages"; -import type { Callbacks } from "@langchain/core/callbacks/manager"; -import { createModel } from "../config/level2-agent.js"; - -type AgentTools = Parameters[0]["tools"]; -type AgentInstance = ReturnType; -type AgentInvokeResult = Awaited>; -type AgentInvokeInput = Parameters[0]; - -/** - * Defines options for invoking a LangChain agent in integration E2E tests. - */ -export interface InvokeAgentOptions { - /** End-user question sent as a single human message. */ - userInput: string; - /** Toolset exposed to the agent for this test scenario. */ - tools: AgentTools; - /** Optional system prompt to steer behavior. */ - systemPrompt?: string; - /** Optional callback handlers (for example tool call tracking). */ - callbacks?: Callbacks; -} - -/** - * Builds a model-backed agent and invokes it with one user message. - * - * This helper centralizes the repeated `createModel` + `createAgent` + `invoke` - * sequence used by level2-agent tests. - * - * @param options Invocation options (user input, tools, optional prompt/callbacks). - * @returns Agent invoke result containing the conversation messages. - */ -export async function invokeAgent(options: InvokeAgentOptions): Promise { - const model = await createModel(); - const agent = createAgent({ - model, - tools: options.tools, - systemPrompt: options.systemPrompt, - }); - - // LangChain's current createAgent typing rejects the message object literal - // shape at the call site, even though the runtime API accepts it. - const input = { - messages: [new HumanMessage(options.userInput)], - } as AgentInvokeInput; - - if (options.callbacks) { - return agent.invoke(input, { callbacks: options.callbacks }); - } - - return agent.invoke(input); -} diff --git a/test/integration/helpers/level2-assertions.ts b/test/integration/helpers/level2-assertions.ts deleted file mode 100644 index c234a5a0..00000000 --- a/test/integration/helpers/level2-assertions.ts +++ /dev/null @@ -1,153 +0,0 @@ -import { BaseCallbackHandler } from "@langchain/core/callbacks/base"; -import type { Serialized } from "@langchain/core/load/serializable"; -import { expect } from "vitest"; - -interface SerializedToolCandidate { - name?: unknown; - id?: unknown; -} - -/** - * Extracts a human-readable tool name from LangChain serialized metadata. - * - * LangChain callbacks may expose the runnable name as the callback `runName`, - * as `name` on serialized metadata, or only as an `id` array where the last - * item corresponds to the runnable/tool class name. - * - * @param tool Serialized LangChain runnable metadata. - * @param runName Optional callback run name supplied by LangChain. - * @returns Extracted tool name, or `undefined` when not found. - */ -export function extractToolName(tool: Serialized, runName?: string): string | undefined { - if (typeof runName === "string" && runName.length > 0) { - return runName; - } - - const candidate = tool as unknown as SerializedToolCandidate; - - if (typeof candidate.name === "string" && candidate.name.length > 0) { - return candidate.name; - } - - if (Array.isArray(candidate.id) && candidate.id.length > 0) { - const last = candidate.id[candidate.id.length - 1]; - if (typeof last === "string" && last.length > 0) { - return last; - } - } - - return undefined; -} - -/** - * Normalizes text for resilient assertions by replacing non-breaking spaces, - * lowercasing content, and removing diacritics. - * - * @param text Raw text to normalize. - * @returns Normalized text suitable for robust `contains` assertions. - */ -export function normalizeText(text: string): string { - return text - .replace(/\u202f/g, " ") - .replace(/\xa0/g, " ") - .toLowerCase() - .normalize("NFD") - .replace(/\p{M}/gu, ""); -} - -/** Minimal message shape required by this helper module. */ -export type MessageLike = { - content: unknown; -}; - -/** - * Returns the normalized content of the final agent message. - * - * @param messages Agent message list. - * @returns Normalized text content of the last message. - * @throws {Error} When the message list is empty. - */ -export function normalizedFinalMessage(messages: readonly MessageLike[]): string { - if (messages.length === 0) { - throw new Error("Agent returned no messages."); - } - - return normalizeText(String(messages[messages.length - 1].content)); -} - -/** - * Asserts that all expected fragments are present in the input text. - * - * @param text Text to inspect. - * @param expectedFragments Fragments that must all be present. - * @returns Nothing. - */ -export function expectContainsAll(text: string, expectedFragments: readonly string[]): void { - const normalizedText = normalizeText(text); - - for (const fragment of expectedFragments) { - expect(normalizedText).toContain(normalizeText(fragment)); - } -} - -/** - * Normalizes the final message, asserts all expected fragments, and returns - * the normalized message for additional assertions. - * - * @param messages Agent message list. - * @param expectedFragments Fragments that must all be present. - * @returns Normalized final message text. - */ -export function expectNormalizedFinalMessageContainsAll( - messages: readonly MessageLike[], - expectedFragments: readonly string[], -): string { - const text = normalizedFinalMessage(messages); - expectContainsAll(text, expectedFragments); - return text; -} - -/** - * Tracks tool calls observed through LangChain callbacks. - */ -export class ToolCallTracker extends BaseCallbackHandler { - name = "tool-call-tracker"; - private readonly toolCalls = new Set(); - - /** - * Records a tool call when LangChain starts a tool. - * - * @param tool Serialized tool metadata. - * @param _input Serialized tool input. - * @param _runId LangChain run identifier. - * @param _parentRunId Optional parent run identifier. - * @param _tags Callback tags. - * @param _metadata Callback metadata. - * @param runName Callback run name, usually the concrete tool name. - * @returns Nothing. - */ - handleToolStart( - tool: Serialized, - _input?: string, - _runId?: string, - _parentRunId?: string, - _tags?: string[], - _metadata?: Record, - runName?: string, - ): void { - const toolName = extractToolName(tool, runName); - if (toolName) { - this.toolCalls.add(toolName); - } - } - - /** - * Checks whether a tool name was observed at least once. - * - * @param toolName Tool name to check. - * @returns `true` when the tool call has been observed, else `false`. - */ - hasToolCall(toolName: string): boolean { - return this.toolCalls.has(toolName); - } -} diff --git a/test/integration/helpers/level2-scenarios.ts b/test/integration/helpers/level2-scenarios.ts deleted file mode 100644 index 4f365cf4..00000000 --- a/test/integration/helpers/level2-scenarios.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { expect } from "vitest"; -import { SYSTEM_PROMPT } from "../config/level2-agent.js"; -import { invokeAgent } from "./level2-agent.js"; -import { - expectNormalizedFinalMessageContainsAll, - ToolCallTracker, -} from "./level2-assertions.js"; - -type AgentTools = Parameters[0]["tools"]; - -/** Context passed to a scenario-specific assertion hook. */ -export interface Level2ScenarioContext { - /** Raw LangChain result returned by the agent invocation. */ - result: Awaited>; - /** Normalized final agent message, ready for string checks. */ - normalizedFinalMessage: string; - /** Tool call tracker, available when the scenario uses tracking. */ - toolCallTracker?: ToolCallTracker; -} - -interface BaseLevel2AgentScenario { - /** Vitest test name shown in the suite output. */ - testName: string; - /** End-user prompt sent to the agent. */ - userInput: string; - /** Fragments that must appear in the final response. */ - expectedResponseFragments: string[]; - /** Tool names that must have been called at least once. */ - requiredToolCalls?: string[]; - /** Optional extra assertions for cases that need more than string checks. */ - assertScenarioResult?: (context: Level2ScenarioContext) => void | Promise; -} - -export interface NoToolsLevel2AgentScenario extends BaseLevel2AgentScenario { - /** Run the scenario without MCP tools. */ - toolMode: "none"; -} - -export interface McpLevel2AgentScenario extends BaseLevel2AgentScenario { - /** Run the scenario with the shared MCP toolset and default system prompt. */ - toolMode: "mcp"; -} - -export type Level2AgentScenario = NoToolsLevel2AgentScenario | McpLevel2AgentScenario; - -interface RunLevel2ScenarioOptions { - tools?: AgentTools; -} - -function resolveScenarioTools( - scenario: Level2AgentScenario, - options: RunLevel2ScenarioOptions, -): AgentTools { - if (scenario.toolMode === "none") { - return []; - } - - if (!options.tools) { - throw new Error(`Scenario "${scenario.testName}" requires MCP tools.`); - } - - if (options.tools.length === 0) { - throw new Error(`Scenario "${scenario.testName}": MCP tools list is empty.`); - } - - return options.tools; -} - -export async function runLevel2Scenario( - scenario: Level2AgentScenario, - options: RunLevel2ScenarioOptions = {}, -): Promise { - const tools = resolveScenarioTools(scenario, options); - const shouldTrackTools = - (scenario.requiredToolCalls?.length ?? 0) > 0 || Boolean(scenario.assertScenarioResult); - const toolCallTracker = shouldTrackTools ? new ToolCallTracker() : undefined; - - const result = await invokeAgent({ - userInput: scenario.userInput, - tools, - systemPrompt: scenario.toolMode === "mcp" ? SYSTEM_PROMPT : undefined, - callbacks: toolCallTracker ? [toolCallTracker] : undefined, - }); - - // DEBUG: show only the LLM's own output — its text and the tool calls it made - // (name + args) — skipping tool RESULTS and the human turn. Printed BEFORE any - // assertion so a failing check does not hide what the agent actually did. - // TODO: remove once the level2 count scenarios are diagnosed. - console.log(`\n===== [${scenario.testName}] agent trace =====`); - for (const message of result.messages) { - // Only AI (LLM) turns; skip HumanMessage and ToolMessage (results). - if (message.getType?.() !== "ai") { - continue; - } - const text = typeof message.content === "string" - ? message.content - : JSON.stringify(message.content); - if (text.trim()) { - console.log(`[llm] ${text.trim()}`); - } - const toolCalls = (message as { tool_calls?: { name: string; args: unknown }[] }).tool_calls ?? []; - for (const call of toolCalls) { - console.log(`[tool call] ${call.name}(${JSON.stringify(call.args)})`); - } - } - console.log(`===== [${scenario.testName}] end trace =====\n`); - - const normalizedFinalMessage = expectNormalizedFinalMessageContainsAll( - result.messages, - scenario.expectedResponseFragments, - ); - - const requiredToolCalls = scenario.requiredToolCalls ?? []; - - if (requiredToolCalls.length > 0 && !toolCallTracker) { - throw new Error(`Scenario "${scenario.testName}": tool-call tracking was not initialized.`); - } - - const trackedToolCallTracker = requiredToolCalls.length > 0 ? toolCallTracker : undefined; - - for (const toolName of requiredToolCalls) { - expect( - trackedToolCallTracker!.hasToolCall(toolName), - `Scenario "${scenario.testName}": expected tool "${toolName}" to have been called.`, - ).toBe(true); - } - - await scenario.assertScenarioResult?.({ - result, - normalizedFinalMessage, - toolCallTracker, - }); -} diff --git a/test/integration/level2-agent/level2-agent.test.ts b/test/integration/level2-agent/level2-agent.test.ts deleted file mode 100644 index b5b3421b..00000000 --- a/test/integration/level2-agent/level2-agent.test.ts +++ /dev/null @@ -1,128 +0,0 @@ -/** - * E2E tests for level 2 agent scenarios. - * - * Ports of the Python level 2 scenarios from geocontext-test. - */ - -import type { MCPAdapter } from "@langchain/mcp-adapters"; -import { afterAll, beforeAll, describe, it, expect } from "vitest"; -import { - CONNECT_TIMEOUT, - E2E_TIMEOUT, - HAS_MODEL_PROVIDER_API_KEY, - createMcpClient, -} from "../config/level2-agent.js"; -import type { Level2AgentScenario } from "../helpers/level2-scenarios.js"; -import { runLevel2Scenario } from "../helpers/level2-scenarios.js"; - -const describeIfProvider = HAS_MODEL_PROVIDER_API_KEY ? describe : describe.skip; - -/** - * Test if the given text contains a number in the specified range. - * The function looks for numbers containing up to 11 characters, including optional spaces. - */ -function containsNumberInRange(text: string, min: number, max: number): boolean { - const values = [...text.matchAll(/\b\d[\d ]{0,10}\b/g)] - .map(([value]) => Number(value.replace(/\s+/g, ""))) - .filter((value) => Number.isFinite(value)); - - return values.some((value) => value >= min && value <= max); -} - -const noToolScenarios = [ - { - testName: "should answer that Paris is the capital of France", - userInput: "Quelle est la capitale de la France?", - expectedResponseFragments: ["paris"], - toolMode: "none", - }, -] satisfies Level2AgentScenario[]; - -const mcpScenarios = [ - { - testName: "should find ERP-related tables for schools", - userInput: "Dans quelle table peut-on trouver des informations sur les écoles?", - expectedResponseFragments: ["bdtopo_v3:zone_d_activite_ou_d_interet"], - toolMode: "mcp", - }, - { - testName: "should mention building-related GPF tables", - userInput: "Dans quelle table peut-on trouver des informations sur les bâtiments?", - expectedResponseFragments: [ - "bdtopo_v3:batiment", - "cadastralparcels.parcellaire_express:batiment", - ], - toolMode: "mcp", - requiredToolCalls: ["gpf_search_types"], - }, - { - testName: "should chain geocode and altitude tools to answer the question", - userInput: "Quelle est l'altitude de la mairie de Chamonix?", - expectedResponseFragments: ["chamonix"], - toolMode: "mcp", - requiredToolCalls: ["geocode", "altitude"], - assertScenarioResult: ({ normalizedFinalMessage }) => { - expect(containsNumberInRange(normalizedFinalMessage, 1000, 1100)).toBe(true); - }, - }, - { - testName: "should answer the question about 14 lycées near the chateau de Vincennes", - userInput: "Combien de lycées sont situés à 2km du chateau de vincennes?", - expectedResponseFragments: ["14", "lycées"], - toolMode: "mcp", - requiredToolCalls: ["geocode", "gpf_search_types", "gpf_describe_type", "gpf_get_features"], - }, - { - testName: "should find about 1 741 batiments in the commune of Saint Mandé", - userInput: "Combien y a t il de batiments sur la commune de saint mandé?", - toolMode: "mcp", - requiredToolCalls: ["geocode", "gpf_search_types", "gpf_count_features"], - expectedResponseFragments: ["batiments"], - assertScenarioResult: ({ normalizedFinalMessage }) => { - expect(containsNumberInRange(normalizedFinalMessage, 1700, 1800)).toBe(true); - }, - }, - { - testName: "should find 19 batiments of more than 30 meters in Angouleme", - userInput: "Combien y a t il de batiments de plus de 30 mètres sur la commune d'Angoulême?", - toolMode: "mcp", - requiredToolCalls: ["geocode", "gpf_search_types", "gpf_describe_type", "gpf_count_features"], - // assuming that it won't often change and that the number is correct at the moment of writing - // (switch to assertScenarioResult with a range if needed in the future) - expectedResponseFragments: ["19", "batiments"] - }, - { - testName: "should find that Sivom swimming pool in Mondeville is the nearest pool to the LUX cinema in Caen, and that the walking distance is 29 minutes", - userInput: "Quel est le temps de marche exact entre le cinéma LUX, situé au sud-est de Caen, et la piscine la plus proche ?", - toolMode: "mcp", - requiredToolCalls: ["geocode", "gpf_search_types", "gpf_describe_type", "gpf_get_features", "distance"], - expectedResponseFragments: ["Sivom", "Mondeville"], - assertScenarioResult: ({ normalizedFinalMessage }) => { - expect(containsNumberInRange(normalizedFinalMessage, 25, 35)).toBe(true); - }, - } -] satisfies Level2AgentScenario[]; - -describeIfProvider("Agent E2E: basic questions (no tools)", () => { - it.each(noToolScenarios)("$testName", async (scenario) => { - await runLevel2Scenario(scenario); - }, E2E_TIMEOUT); -}); - -describeIfProvider("Agent E2E: MCP-backed scenarios", () => { - let client: MCPAdapter | undefined; - let tools: Awaited> | undefined; - - beforeAll(async () => { - client = createMcpClient(); - tools = await client.listTools(); - }, CONNECT_TIMEOUT); - - afterAll(async () => { - await client?.close(); - }); - - it.each(mcpScenarios)("$testName", async (scenario) => { - await runLevel2Scenario(scenario, { tools }); - }, E2E_TIMEOUT); -}); diff --git a/vitest.e2e.config.mts b/vitest.e2e.config.mts deleted file mode 100644 index e04a498d..00000000 --- a/vitest.e2e.config.mts +++ /dev/null @@ -1,24 +0,0 @@ -import { defineConfig } from "vitest/config"; - -const MILLISECONDS = 1000; - -export default defineConfig({ - resolve: { - alias: [ - { - find: /^(\.{1,2}\/.*)\.js$/, - replacement: "$1", - }, - ], - }, - test: { - globals: false, - environment: "node", - include: ["test/integration/level2-agent/**/*.test.ts"], - testTimeout: 120 * MILLISECONDS, - // Run sequentially: each test starts its own MCP server - pool: "forks", - fileParallelism: false, - execArgv: ["--use-env-proxy"], - }, -});