diff --git a/.coverage b/.coverage index 5b04d9ff..c3abd146 100644 Binary files a/.coverage and b/.coverage differ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b46d09c..4901e7f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,14 +39,14 @@ jobs: run: | pytest tests/test_categories_roadmap.py tests/test_report_categories_golden.py \ tests/test_categories_coverage.py tests/test_indexation_coverage.py tests/test_crawl_segments.py \ - tests/test_terminology.py \ + tests/test_terminology.py tests/test_compare_payload.py \ --cov=website_profiling.reporting --cov-config=.coveragerc.reporting \ --cov-report=term-missing --cov-fail-under=100 -q -o addopts= - name: Pytest (tools coverage gate) run: | pytest tests/test_alert_checker.py tests/test_schedule_runner.py tests/test_export_audit.py \ tests/test_export_audit_coverage.py tests/test_audit_tools.py tests/test_audit_tools_expanded.py \ - tests/test_compare_payload.py tests/test_audit_tools_coverage.py \ + tests/test_audit_tools_coverage.py \ tests/test_mcp_registry.py tests/test_mcp_resources.py \ --cov=website_profiling.tools --cov-config=.coveragerc.tools \ --cov-report=term-missing --cov-fail-under=95 -q -o addopts= diff --git a/local-test.ps1 b/local-test.ps1 new file mode 100644 index 00000000..f7180beb --- /dev/null +++ b/local-test.ps1 @@ -0,0 +1,2 @@ +# Wrapper — run from repo root: .\local-test.ps1 [command] +& "$PSScriptRoot\scripts\local-test.ps1" @args diff --git a/scripts/local-run.ps1 b/scripts/local-run.ps1 index 057feb41..95b26265 100644 --- a/scripts/local-run.ps1 +++ b/scripts/local-run.ps1 @@ -301,7 +301,7 @@ Environment overrides (optional): After start, open: http://localhost:3000/home Run audits via sidebar "Run audit" (bottom-right FAB). -Run CI-style tests: ./local-test (bash/Git Bash/WSL) or see scripts/local-test.sh. +Run CI-style tests: .\local-test.ps1 or ./local-test (bash/Git Bash/WSL). "@ } diff --git a/scripts/local-test.ps1 b/scripts/local-test.ps1 new file mode 100644 index 00000000..a04218d9 --- /dev/null +++ b/scripts/local-test.ps1 @@ -0,0 +1,379 @@ +# Local test runner — mirrors .github/workflows/ci.yml on Windows. +# Usage: .\scripts\local-test.ps1 [command] [-NoCov] +# (default) all — Postgres + migrations + Python + web checks +# python — DB + pytest + CLI smoke only +# reporting — reporting module 100% coverage gate (CI step) +# tools — tools module coverage gate +# web — typecheck, lint, vitest (no Postgres) +# quick — pytest -NoCov + web (DB must already be running) +# help — show commands +# Requires: PowerShell 5.1+ (PowerShell 7+ recommended for reliable exit codes) + +$ErrorActionPreference = "Stop" + +$ROOT = Split-Path -Parent $PSScriptRoot +Set-Location $ROOT + +$PG_CONTAINER = if ($env:WP_PG_CONTAINER) { $env:WP_PG_CONTAINER } else { "wp-pg" } +$PG_IMAGE = if ($env:WP_PG_IMAGE) { $env:WP_PG_IMAGE } else { "postgres:16-alpine" } +$PG_PORT = if ($env:WP_PG_PORT) { $env:WP_PG_PORT } else { "5432" } +$PG_USER = if ($env:WP_PG_USER) { $env:WP_PG_USER } else { "postgres" } +$PG_PASSWORD = if ($env:WP_PG_PASSWORD) { $env:WP_PG_PASSWORD } else { "dev" } +$PG_DB = if ($env:WP_PG_DB) { $env:WP_PG_DB } else { "website_profiling" } + +if (-not $env:DATABASE_URL) { + $env:DATABASE_URL = "postgres://${PG_USER}:${PG_PASSWORD}@127.0.0.1:${PG_PORT}/${PG_DB}" +} +if (-not $env:DATA_DIR) { + $env:DATA_DIR = Join-Path $ROOT "data" +} + +$VENV = Join-Path $ROOT ".venv" +$VENV_PYTHON = Join-Path $VENV "Scripts\python.exe" +$VENV_PIP = Join-Path $VENV "Scripts\pip.exe" +$VENV_PYTEST = Join-Path $VENV "Scripts\pytest.exe" +$VENV_ALEMBIC = Join-Path $VENV "Scripts\alembic.exe" +$WEB = Join-Path $ROOT "web" + +$env:WEBSITE_PROFILING_ROOT = $ROOT +if ($env:PYTHONPATH) { + $env:PYTHONPATH = "$($env:PYTHONPATH);$(Join-Path $ROOT 'src')" +} else { + $env:PYTHONPATH = Join-Path $ROOT "src" +} + +$PytestNoCov = $false + +function Write-Log([string]$Message) { + Write-Host "-> $Message" -ForegroundColor Cyan +} + +function Write-Ok([string]$Message) { + Write-Host "OK $Message" -ForegroundColor Green +} + +function Write-Warn([string]$Message) { + Write-Host "! $Message" -ForegroundColor Yellow +} + +function Write-Die([string]$Message) { + Write-Host "X $Message" -ForegroundColor Red + exit 1 +} + +function Assert-LastExitCode([string]$Message) { + $failed = $false + if ($PSVersionTable.PSVersion.Major -ge 7) { + $failed = ($LASTEXITCODE -ne 0) + } else { + $failed = (-not $?) + } + if ($failed) { + Write-Die $Message + } +} + +function Test-Command([string]$Name) { + if (-not (Get-Command $Name -ErrorAction SilentlyContinue)) { + Write-Die "Missing required command: $Name" + } +} + +function Get-PythonLauncher { + foreach ($cmd in @("python", "python3", "py")) { + if (Get-Command $cmd -ErrorAction SilentlyContinue) { + if ($cmd -eq "py") { + return ,@("py", "-3") + } + return ,@($cmd) + } + } + Write-Die "Missing required command: python (install Python 3.11+ and ensure it is on PATH)" +} + +function Invoke-PythonLauncher { + param( + [Parameter(Mandatory = $true)] + [string[]]$Launcher, + [Parameter(ValueFromRemainingArguments = $true)] + [string[]]$PythonArgs + ) + + if ($Launcher.Count -gt 1) { + & $Launcher[0] $Launcher[1] @PythonArgs + } else { + & $Launcher[0] @PythonArgs + } + Assert-LastExitCode "Python command failed: $($Launcher -join ' ') $($PythonArgs -join ' ')" +} + +function Get-DockerContainerNames { + param([switch]$All) + + $dockerArgs = if ($All) { @("ps", "-a") } else { @("ps") } + $output = & docker @dockerArgs --format "{{.Names}}" 2>$null + if (-not $output) { + return @() + } + return @($output | ForEach-Object { "$_".Trim() } | Where-Object { $_ }) +} + +function Test-DockerRunning { + Test-Command docker + $prevErrorAction = $ErrorActionPreference + $ErrorActionPreference = "Continue" + try { + cmd /c "docker info >nul 2>&1" + } finally { + $ErrorActionPreference = $prevErrorAction + } + Assert-LastExitCode "Docker is not running. Start Docker Desktop, then retry." +} + +function Test-ContainerExists([string]$Name) { + return (Get-DockerContainerNames -All) -contains $Name +} + +function Test-ContainerRunning([string]$Name) { + return (Get-DockerContainerNames) -contains $Name +} + +function Wait-ForPostgres { + for ($i = 1; $i -le 30; $i++) { + & docker exec $PG_CONTAINER pg_isready -U $PG_USER -d $PG_DB *> $null + if ($PSVersionTable.PSVersion.Major -ge 7) { + if ($LASTEXITCODE -eq 0) { return } + } elseif ($?) { + return + } + Start-Sleep -Seconds 1 + } + Write-Die "Postgres did not become ready in time (container: $PG_CONTAINER)" +} + +function Invoke-Db { + Test-DockerRunning + if (Test-ContainerExists $PG_CONTAINER) { + if (Test-ContainerRunning $PG_CONTAINER) { + Write-Log "Postgres already running ($PG_CONTAINER)" + } else { + Write-Log "Starting existing container $PG_CONTAINER" + & docker start $PG_CONTAINER *> $null + Assert-LastExitCode "Failed to start container $PG_CONTAINER" + } + } else { + Write-Log "Creating Postgres container $PG_CONTAINER on port $PG_PORT" + & docker run -d --name $PG_CONTAINER ` + -e "POSTGRES_PASSWORD=$PG_PASSWORD" ` + -e "POSTGRES_DB=$PG_DB" ` + -p "${PG_PORT}:5432" ` + $PG_IMAGE *> $null + Assert-LastExitCode "Failed to create Postgres container $PG_CONTAINER" + } + Wait-ForPostgres + Write-Log "DATABASE_URL=$($env:DATABASE_URL)" +} + +function Invoke-Venv { + $pyLauncher = Get-PythonLauncher + if (-not (Test-Path $VENV_PYTHON)) { + Write-Log "Creating Python venv at .venv" + Invoke-PythonLauncher -Launcher $pyLauncher -PythonArgs @("-m", "venv", $VENV) + } + if (-not (Test-Path $VENV_PYTEST)) { + Write-Log "Installing Python dependencies" + & $VENV_PIP install -q -r (Join-Path $ROOT "requirements.txt") + Assert-LastExitCode "Failed to install requirements.txt" + & $VENV_PIP install -q -r (Join-Path $ROOT "requirements-browser.txt") + Assert-LastExitCode "Failed to install requirements-browser.txt" + } +} + +function Invoke-Migrate { + Invoke-Db + if (-not (Test-Path $VENV_ALEMBIC)) { + Invoke-Venv + } + Write-Log "Applying database migrations (alembic upgrade head)" + & $VENV_ALEMBIC upgrade head + Assert-LastExitCode "Database migration failed (alembic upgrade head)" +} + +function Invoke-WebDeps { + Test-Command npm + $nodeModules = Join-Path $WEB "node_modules" + if (-not (Test-Path $nodeModules)) { + Write-Log "Installing web dependencies (npm ci)" + Push-Location $WEB + try { + & npm ci + Assert-LastExitCode "Failed to install web dependencies (npm ci)" + } finally { + Pop-Location + } + } +} + +function Invoke-PytestCore { + if ($PytestNoCov) { + Write-Log "Pytest (tests/ -q -m not browser --no-cov)" + & $VENV_PYTEST tests/ -q -m "not browser" --no-cov + } else { + Write-Log "Pytest (tests/ -q -m not browser, 100% coverage gate)" + & $VENV_PYTEST tests/ -q -m "not browser" + } + Assert-LastExitCode "Core pytest failed" +} + +function Invoke-PytestReporting { + Write-Log "Pytest (reporting coverage gate, 100%)" + & $VENV_PYTEST ` + tests/test_categories_roadmap.py ` + tests/test_report_categories_golden.py ` + tests/test_categories_coverage.py ` + tests/test_indexation_coverage.py ` + tests/test_crawl_segments.py ` + tests/test_terminology.py ` + tests/test_compare_payload.py ` + --cov=website_profiling.reporting ` + --cov-config=.coveragerc.reporting ` + --cov-report=term-missing ` + --cov-fail-under=100 ` + -q ` + -o addopts= + Assert-LastExitCode "Reporting coverage gate failed" +} + +function Invoke-PytestTools { + Write-Log "Pytest (tools coverage gate, 95%)" + & $VENV_PYTEST ` + tests/test_alert_checker.py ` + tests/test_schedule_runner.py ` + tests/test_export_audit.py ` + tests/test_export_audit_coverage.py ` + tests/test_audit_tools.py ` + tests/test_audit_tools_expanded.py ` + tests/test_audit_tools_coverage.py ` + tests/test_mcp_registry.py ` + tests/test_mcp_resources.py ` + --cov=website_profiling.tools ` + --cov-config=.coveragerc.tools ` + --cov-report=term-missing ` + --cov-fail-under=95 ` + -q ` + -o addopts= + Assert-LastExitCode "Tools coverage gate failed" +} + +function Invoke-PythonChecks { + Invoke-Db + Invoke-Venv + Invoke-Migrate + Invoke-PytestCore + Invoke-PytestReporting + Invoke-PytestTools + Write-Log "CLI smoke (python -m src --help)" + & $VENV_PYTHON -m src --help *> $null + Assert-LastExitCode "CLI smoke failed" + Write-Ok "Python checks passed" +} + +function Invoke-WebChecks { + Invoke-WebDeps + Write-Log "Web typecheck" + Push-Location $WEB + try { + & npm run typecheck + Assert-LastExitCode "Web typecheck failed" + Write-Log "Web lint" + & npm run lint + Assert-LastExitCode "Web lint failed" + Write-Log "Web tests (vitest)" + & npm test + Assert-LastExitCode "Web tests failed" + } finally { + Pop-Location + } + Write-Ok "Web checks passed" +} + +function Invoke-Quick { + if (-not $env:DATABASE_URL) { + Write-Die "DATABASE_URL is not set. Export it or run .\scripts\local-test.ps1 all" + } + Invoke-Venv + Invoke-WebDeps + Write-Warn "quick: assuming Postgres is up and migrated (.\local-run.ps1 db; .\local-run.ps1 migrate)" + $PytestNoCov = $true + Invoke-PytestCore + Write-Log "CLI smoke (python -m src --help)" + & $VENV_PYTHON -m src --help *> $null + Assert-LastExitCode "CLI smoke failed" + Invoke-WebChecks + Write-Ok "Quick test run passed" +} + +function Show-Help { + Write-Host @" +Local test runner — mirrors CI (python + web jobs) + + .\scripts\local-test.ps1 Same as: all + .\scripts\local-test.ps1 all Postgres + migrations + full pytest + web + .\scripts\local-test.ps1 python DB + pytest (core + reporting + tools) + CLI + .\scripts\local-test.ps1 reporting Reporting module 100% coverage gate only + .\scripts\local-test.ps1 tools Tools module coverage gate only + .\scripts\local-test.ps1 web typecheck, lint, vitest (no Docker) + .\scripts\local-test.ps1 quick pytest -NoCov + web (DB must be ready) + + .\scripts\local-test.ps1 all -NoCov skip pytest coverage gates (faster) + +Environment (same as .\local-run.ps1): + DATABASE_URL, DATA_DIR, WP_PG_CONTAINER, WP_PG_PORT, ... + +One-time dev setup: .\local-run.ps1 setup +"@ +} + +$cmd = "all" +$argList = @($args) +if ($argList.Count -gt 0) { + $cmd = $argList[0] + $argList = if ($argList.Count -gt 1) { $argList[1..($argList.Count - 1)] } else { @() } +} + +foreach ($arg in $argList) { + switch ($arg) { + "-NoCov" { $PytestNoCov = $true } + "-h" { Show-Help; exit 0 } + "--help" { Show-Help; exit 0 } + default { Write-Die "Unknown argument: $arg (try: .\scripts\local-test.ps1 help)" } + } +} + +switch ($cmd) { + "all" { + Invoke-PythonChecks + Invoke-WebChecks + Write-Ok "All local tests passed (CI python + web jobs)" + } + "python" { Invoke-PythonChecks } + "reporting" { + Invoke-Venv + Invoke-PytestReporting + Write-Ok "Reporting coverage gate passed" + } + "tools" { + Invoke-Venv + Invoke-PytestTools + Write-Ok "Tools coverage gate passed" + } + "web" { Invoke-WebChecks } + "quick" { + $PytestNoCov = $true + Invoke-Quick + } + "help" { Show-Help } + "-h" { Show-Help } + "--help" { Show-Help } + default { Write-Die "Unknown command: $cmd (try: .\scripts\local-test.ps1 help)" } +} diff --git a/src/website_profiling/commands/chat_cmd.py b/src/website_profiling/commands/chat_cmd.py index 5345226e..25c63b96 100644 --- a/src/website_profiling/commands/chat_cmd.py +++ b/src/website_profiling/commands/chat_cmd.py @@ -5,6 +5,7 @@ import json import sys +from ..text_sanitize import sanitize_unicode_deep from ..tools.audit_tools import AuditToolContext from ..llm.agent import run_agent_turn @@ -38,7 +39,7 @@ def run(_cfg: dict, args: argparse.Namespace) -> None: ctx = AuditToolContext(property_id=pid, report_id=rid) def on_event(event: dict) -> None: - print(json.dumps(event, default=str), flush=True) + print(json.dumps(sanitize_unicode_deep(event), default=str), flush=True) try: result = run_agent_turn(messages, ctx, on_event=on_event) diff --git a/src/website_profiling/db/_common.py b/src/website_profiling/db/_common.py index 860f7c24..635267e3 100644 --- a/src/website_profiling/db/_common.py +++ b/src/website_profiling/db/_common.py @@ -10,6 +10,8 @@ from psycopg import Connection from psycopg.types.json import Json +from ..text_sanitize import strip_surrogates + def _now_iso() -> str: return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") @@ -59,8 +61,10 @@ def _sanitize_for_json(obj: Any) -> Any: """Recursively replace NaN/Inf and numpy types so JSON is valid.""" if obj is None: return None - if isinstance(obj, (bool, str)): + if isinstance(obj, bool): return obj + if isinstance(obj, str): + return strip_surrogates(obj) if isinstance(obj, int): return int(obj) if isinstance(obj, float): diff --git a/src/website_profiling/llm/agent.py b/src/website_profiling/llm/agent.py index c3930374..236dcc18 100644 --- a/src/website_profiling/llm/agent.py +++ b/src/website_profiling/llm/agent.py @@ -5,6 +5,7 @@ from typing import Any, Callable from ..llm_config import llm_is_enabled, load_llm_config_from_db +from ..text_sanitize import sanitize_unicode_deep, strip_surrogates from ..tools.audit_tools import AuditToolContext from ..tools.audit_tools.registry import TOOL_DEFINITIONS, dispatch_tool, openai_tools_schema from .base import ChatResult, ToolCall, get_llm_client @@ -60,7 +61,7 @@ def _emit(on_event: Callable[[dict], None] | None, event: dict[str, Any]) -> None: if on_event: - on_event(event) + on_event(sanitize_unicode_deep(event)) def _supports_native_tools(client: Any) -> bool: @@ -112,7 +113,7 @@ def _build_openai_messages(history: list[dict[str, str]]) -> list[dict[str, Any] out: list[dict[str, Any]] = [{"role": "system", "content": SYSTEM_PROMPT}] for msg in history: role = msg.get("role") - content = str(msg.get("content") or "") + content = strip_surrogates(str(msg.get("content") or "")) if role in ("user", "assistant"): out.append({"role": role, "content": content}) return out @@ -147,7 +148,7 @@ def run_agent_turn( final_message = "" def on_token(text: str) -> None: - _emit(on_event, {"type": "token", "text": text}) + _emit(on_event, {"type": "token", "text": strip_surrogates(text)}) for _round in range(MAX_TOOL_ROUNDS): _emit(on_event, { @@ -156,10 +157,11 @@ def on_token(text: str) -> None: "detail": f"Thinking (step {_round + 1}/{MAX_TOOL_ROUNDS})…", }) try: + llm_messages = sanitize_unicode_deep(openai_messages) if _supports_native_tools(client): - result = client.chat_with_tools(openai_messages, tools, on_token=on_token) + result = client.chat_with_tools(llm_messages, tools, on_token=on_token) else: - result = _react_step(client, openai_messages, _tools_description(compact=True), on_token) + result = _react_step(client, llm_messages, _tools_description(compact=True), on_token) except Exception as e: msg = str(e).strip() or type(e).__name__ if "httpx" in msg.lower() or "requirements-llm" in msg.lower(): @@ -180,7 +182,7 @@ def on_token(text: str) -> None: "function": { "index": i, "name": tc.name, - "arguments": tc.arguments, + "arguments": sanitize_unicode_deep(tc.arguments), }, }) else: @@ -193,7 +195,7 @@ def on_token(text: str) -> None: if _supports_native_tools(client): openai_messages.append({ "role": "assistant", - "content": result.content or "", + "content": strip_surrogates(result.content or ""), "tool_calls": assistant_tool_calls, }) else: @@ -204,25 +206,28 @@ def on_token(text: str) -> None: for tc in result.tool_calls: _emit(on_event, {"type": "tool_start", "name": tc.name, "args": tc.arguments}) - tool_result = dispatch_tool(tc.name, tc.arguments, context=context) + tool_result = sanitize_unicode_deep( + dispatch_tool(tc.name, tc.arguments, context=context), + ) _emit(on_event, {"type": "tool_end", "name": tc.name, "result": tool_result}) tool_events.append({"name": tc.name, "args": tc.arguments, "result": tool_result}) + tool_content = json.dumps(tool_result, default=str) if ollama_format: openai_messages.append({ "role": "tool", "tool_name": tc.name, - "content": json.dumps(tool_result, default=str), + "content": tool_content, }) else: openai_messages.append({ "role": "tool", "tool_call_id": tc.id, - "content": json.dumps(tool_result, default=str), + "content": tool_content, }) continue - final_message = result.content.strip() + final_message = strip_surrogates(result.content).strip() if final_message: _emit(on_event, {"type": "done", "message": final_message}) return {"ok": True, "message": final_message, "tool_events": tool_events} diff --git a/src/website_profiling/text_sanitize.py b/src/website_profiling/text_sanitize.py new file mode 100644 index 00000000..3cb9bfe5 --- /dev/null +++ b/src/website_profiling/text_sanitize.py @@ -0,0 +1,24 @@ +"""Strip lone UTF-16 surrogates so strings are safe for UTF-8 JSON/HTTP.""" +from __future__ import annotations + +from typing import Any + + +def strip_surrogates(text: str) -> str: + """Replace lone surrogates (invalid in UTF-8) with U+FFFD.""" + if not text: + return text + return text.encode("utf-8", errors="replace").decode("utf-8") + + +def sanitize_unicode_deep(obj: Any) -> Any: + """Recursively sanitize strings in nested dicts/lists.""" + if isinstance(obj, str): + return strip_surrogates(obj) + if isinstance(obj, dict): + return {k: sanitize_unicode_deep(v) for k, v in obj.items()} + if isinstance(obj, list): + return [sanitize_unicode_deep(v) for v in obj] + if isinstance(obj, tuple): + return tuple(sanitize_unicode_deep(v) for v in obj) + return obj diff --git a/tests/test_audit_tools_expanded.py b/tests/test_audit_tools_expanded.py index 9db60c49..00fbd9e4 100644 --- a/tests/test_audit_tools_expanded.py +++ b/tests/test_audit_tools_expanded.py @@ -161,7 +161,7 @@ def conn() -> MagicMock: def test_handler_schema_parity() -> None: names = {t["name"] for t in TOOL_DEFINITIONS} assert names == tool_handler_names() - assert len(TOOL_DEFINITIONS) == 121 + assert len(TOOL_DEFINITIONS) == 123 def test_slice_helpers() -> None: diff --git a/tests/test_chat_cmd.py b/tests/test_chat_cmd.py new file mode 100644 index 00000000..50f5a5c7 --- /dev/null +++ b/tests/test_chat_cmd.py @@ -0,0 +1,99 @@ +"""CLI chat command tests.""" +from __future__ import annotations + +import argparse +import io +import json +from unittest.mock import patch + +import pytest + +from website_profiling.commands import chat_cmd + + +def test_chat_cmd_requires_stdin_json() -> None: + with pytest.raises(SystemExit) as exc: + chat_cmd.run({}, argparse.Namespace(stdin_json=False)) + assert exc.value.code == 1 + + +def test_chat_cmd_invalid_stdin_json(capsys) -> None: + with patch("sys.stdin", io.StringIO("not-json")): + with pytest.raises(SystemExit) as exc: + chat_cmd.run({}, argparse.Namespace(stdin_json=True)) + assert exc.value.code == 1 + assert "error" in capsys.readouterr().out + + +def test_chat_cmd_success(capsys) -> None: + payload = json.dumps({"messages": [{"role": "user", "content": "Hi"}], "property_id": 1}) + with patch("sys.stdin", io.StringIO(payload)): + with patch( + "website_profiling.commands.chat_cmd.run_agent_turn", + return_value={"ok": True, "message": "Done"}, + ) as mock_turn: + with pytest.raises(SystemExit) as exc: + chat_cmd.run({}, argparse.Namespace(stdin_json=True)) + assert exc.value.code == 0 + mock_turn.assert_called_once() + assert mock_turn.call_args[0][1].property_id == 1 + + +def test_chat_cmd_streams_sanitized_events(capsys) -> None: + payload = json.dumps({"messages": [{"role": "user", "content": "Hi"}]}) + + def fake_turn(_messages, _ctx, on_event=None): + if on_event: + on_event({"type": "token", "content": "bad\udc9d"}) + return {"ok": True} + + with patch("sys.stdin", io.StringIO(payload)): + with patch("website_profiling.commands.chat_cmd.run_agent_turn", side_effect=fake_turn): + with pytest.raises(SystemExit) as exc: + chat_cmd.run({}, argparse.Namespace(stdin_json=True)) + assert exc.value.code == 0 + out = capsys.readouterr().out + assert "\udc9d" not in out + assert "token" in out + + +def test_chat_cmd_coerces_invalid_ids_and_messages(capsys) -> None: + payload = json.dumps({"messages": "bad", "property_id": "x", "report_id": "y"}) + with patch("sys.stdin", io.StringIO(payload)): + with patch( + "website_profiling.commands.chat_cmd.run_agent_turn", + return_value={"ok": True}, + ) as mock_turn: + with pytest.raises(SystemExit) as exc: + chat_cmd.run({}, argparse.Namespace(stdin_json=True)) + assert exc.value.code == 0 + ctx = mock_turn.call_args[0][1] + assert ctx.property_id is None + assert ctx.report_id is None + assert mock_turn.call_args[0][0] == [] + + +def test_chat_cmd_agent_failure(capsys) -> None: + payload = json.dumps({"messages": []}) + with patch("sys.stdin", io.StringIO(payload)): + with patch( + "website_profiling.commands.chat_cmd.run_agent_turn", + return_value={"ok": False, "error": "LLM disabled"}, + ): + with pytest.raises(SystemExit) as exc: + chat_cmd.run({}, argparse.Namespace(stdin_json=True)) + assert exc.value.code == 1 + assert "LLM disabled" in capsys.readouterr().out + + +def test_chat_cmd_exception(capsys) -> None: + payload = json.dumps({"messages": []}) + with patch("sys.stdin", io.StringIO(payload)): + with patch( + "website_profiling.commands.chat_cmd.run_agent_turn", + side_effect=RuntimeError("boom"), + ): + with pytest.raises(SystemExit) as exc: + chat_cmd.run({}, argparse.Namespace(stdin_json=True)) + assert exc.value.code == 1 + assert "boom" in capsys.readouterr().out diff --git a/tests/test_chat_store.py b/tests/test_chat_store.py index f6f54625..cf37b7a7 100644 --- a/tests/test_chat_store.py +++ b/tests/test_chat_store.py @@ -40,12 +40,76 @@ def test_list_sessions() -> None: assert rows[0]["property_id"] == 7 +def test_get_session_found() -> None: + conn = FakeConn() + now = datetime.now(timezone.utc) + conn.set_next_cursor( + FakeCursor( + fetchone_value={ + "id": 5, + "property_id": 7, + "title": "Found", + "created_at": now, + "updated_at": now, + }, + ), + ) + row = get_session(conn, 5) + assert row is not None + assert row["title"] == "Found" + + def test_get_session_missing() -> None: conn = FakeConn() conn.set_next_cursor(FakeCursor(fetchone_value=None)) assert get_session(conn, 99) is None +def test_get_messages_parses_json_fields() -> None: + conn = FakeConn() + now = datetime.now(timezone.utc) + conn.set_next_cursor( + FakeCursor( + fetchall_value=[ + { + "id": 1, + "role": "tool", + "content": "", + "tool_name": "list_issues", + "tool_args": '{"limit": 5}', + "tool_result": "not-json", + "created_at": now, + }, + ], + ), + ) + msgs = get_messages(conn, 5) + assert msgs[0]["tool_args"] == {"limit": 5} + assert msgs[0]["tool_result"] == "not-json" + + +def test_get_messages_keeps_invalid_tool_args_json() -> None: + conn = FakeConn() + now = datetime.now(timezone.utc) + conn.set_next_cursor( + FakeCursor( + fetchall_value=[ + { + "id": 2, + "role": "tool", + "content": "", + "tool_name": "list_issues", + "tool_args": "not-json", + "tool_result": None, + "created_at": now, + }, + ], + ), + ) + msgs = get_messages(conn, 5) + assert msgs[0]["tool_args"] == "not-json" + + def test_get_messages() -> None: conn = FakeConn() now = datetime.now(timezone.utc) diff --git a/tests/test_compare_payload.py b/tests/test_compare_payload.py index e9bb1843..0d6723a9 100644 --- a/tests/test_compare_payload.py +++ b/tests/test_compare_payload.py @@ -1,13 +1,17 @@ """Tests for reporting/compare_payload.py — parity with web compare.""" from __future__ import annotations +from unittest.mock import patch + from website_profiling.reporting.compare_payload import ( + build_category_scores, build_content_metrics, build_duplicate_deltas, build_full_compare, build_google_metrics, build_issue_deltas, build_lighthouse_url_deltas, + build_link_metric_deltas, build_priority_counts, build_redirect_deltas, build_security_deltas, @@ -49,6 +53,11 @@ def _payload(**overrides) -> dict: def test_norm_report_url() -> None: assert norm_report_url("https://Ex.COM/page/") == "ex.com/page" + assert norm_report_url("") == "" + assert norm_report_url(" ") == "" + assert norm_report_url("relative/path/") == "relative/path" + with patch("website_profiling.reporting.compare_payload.urlparse", side_effect=ValueError("bad")): + assert norm_report_url("https://ex.com/x") == "https://ex.com/x" def test_issue_and_priority_deltas() -> None: @@ -98,3 +107,172 @@ def test_url_set_diff() -> None: assert diff["removed_count"] >= 1 assert diff["new_urls"][0].startswith("https://") assert diff["removed_urls"][0].startswith("https://") + assert build_url_set_diff({"links": ["not-a-dict"]}, {}) == { + "new_urls": [], + "removed_urls": [], + "new_count": 0, + "removed_count": 0, + } + + +def test_issue_deltas_edge_cases() -> None: + cur = { + "categories": [ + "skip", + {"name": "SEO", "issues": [ + "skip", + {"url": "", "message": ""}, + {"url": "https://ex.com/a", "message": "Fix", "priority": "Critical"}, + ]}, + ], + } + base = { + "categories": [ + {"name": "SEO", "issues": [ + {"url": "https://ex.com/b", "message": "Gone", "priority": "Low"}, + ]}, + ], + } + issues = build_issue_deltas(cur, base) + kinds = {i["kind"] for i in issues} + assert "new" in kinds + assert "resolved" in kinds + + +def test_priority_counts_skips_invalid_entries() -> None: + cur = {"categories": ["skip", {"issues": ["skip", {"priority": "High"}]}]} + base = {"categories": []} + counts = build_priority_counts(cur, base) + assert counts[1]["current"] == 1 + + +def test_lighthouse_from_links_and_skips() -> None: + cur = { + "lighthouse_by_url": { + "": {"performance": 50}, + "https://ex.com/a": "skip", + }, + "links": [ + {"url": "https://ex.com/b", "lighthouse": {"median_metrics": {"performance_score": 70, "seo_score": 90}}}, + "skip", + {"url": "https://ex.com/a", "lighthouse": {"median_metrics": {"performance_score": 80}}}, + ], + } + base = {"lighthouse_by_url": {"https://ex.com/c": {"median_metrics": {"performance_score": 50, "seo_score": 50}}}} + assert build_lighthouse_url_deltas(cur, base) == [] + + +def test_link_metric_deltas_edge_cases() -> None: + cur = {"links": [{"url": "https://ex.com/a", "inlinks": 10, "outlinks": 5}]} + base = { + "links": [ + "skip", + {"url": ""}, + {"url": "https://ex.com/missing"}, + {"url": "https://ex.com/a", "inlinks": 5, "outlinks": 5, "word_count": "x"}, + {"url": "https://ex.com/b", "inlinks": 1, "outlinks": 1}, + ], + } + deltas = build_link_metric_deltas(cur, base) + assert len(deltas) == 1 + assert deltas[0]["metric"] == "inlinks" + assert build_link_metric_deltas({"links": []}, {"links": []}) == [] + + +def test_redirect_deltas_removed_and_skips() -> None: + cur = {"redirects": ["skip", {"url": "", "from": ""}]} + base = {"redirects": [{"url": "https://ex.com/gone", "status": "301"}]} + deltas = build_redirect_deltas(cur, base) + assert any(d["kind"] == "removed" for d in deltas) + + +def test_security_deltas_resolved_and_skips() -> None: + cur = {"security_findings": []} + base = { + "security_findings": [ + "skip", + {"url": "https://ex.com", "finding_type": "csp", "message": "missing"}, + ], + } + deltas = build_security_deltas(cur, base) + assert len(deltas) == 1 + assert deltas[0]["kind"] == "resolved" + + +def test_duplicate_deltas_all_kinds() -> None: + cur = { + "content_duplicates": [ + "skip", + {"representative_url": ""}, + {"id": "new1", "representative_url": "https://ex.com/n", "member_count": 3}, + {"id": "chg", "representative_url": "https://ex.com/c", "member_urls": ["a", "b", "c"]}, + ], + } + base = { + "content_duplicates": [ + {"id": "chg", "representative_url": "https://ex.com/c", "member_count": 2}, + {"id": "rm", "representative_url": "https://ex.com/r", "member_count": 4}, + ], + } + deltas = build_duplicate_deltas(cur, base) + kinds = {d["kind"] for d in deltas} + assert kinds == {"new", "changed", "removed"} + + +def test_tech_deltas_removed_and_skips() -> None: + cur = {"tech_stack_summary": {"technologies": ["skip", {"name": "React", "count": 2}]}} + base = {"tech_stack_summary": {"technologies": [{"name": "jQuery", "count": 1}]}} + deltas = build_tech_deltas(cur, base) + kinds = {d["kind"] for d in deltas} + assert kinds == {"added", "removed"} + + +def test_google_metrics_unavailable() -> None: + assert build_google_metrics({}, {}) == {"available": False, "metrics": []} + + +def test_category_scores_skips_invalid() -> None: + cur = {"categories": ["skip", {"id": "", "score": 90}, {"id": "perf", "name": "Performance", "score": 75}]} + base = {"categories": [{"id": "perf", "score": 80}]} + scores = build_category_scores(cur, base) + assert len(scores) == 1 + assert scores[0]["id"] == "perf" + assert scores[0]["delta"] == -5 + + +def test_full_compare_truncation() -> None: + many_issues = [ + {"priority": "Low", "message": f"issue-{i}", "url": f"https://ex.com/p{i}"} + for i in range(105) + ] + cur = {"categories": [{"id": "x", "name": "X", "score": 50, "issues": many_issues}]} + base = {"categories": []} + full = build_full_compare(cur, base) + assert full["truncated_sections"].get("issue_deltas") is True + assert len(full["issue_deltas"]) == 100 + + links = [] + for i in range(210): + links.append({ + "url": f"https://ex.com/l{i}", + "inlinks": i + 10, + "outlinks": 1, + "word_count": 100, + "response_time_ms": 100, + }) + cur_links = {"links": links} + base_links = { + "links": [ + { + "url": f"https://ex.com/l{i}", + "inlinks": i, + "outlinks": 1, + "word_count": 100, + "response_time_ms": 100, + } + for i in range(210) + ], + } + full_links = build_full_compare(cur_links, base_links) + assert full_links["truncated_sections"].get("link_metric_deltas") is True + assert len(full_links["link_metric_deltas"]) == 200 diff --git a/tests/test_mcp_registry.py b/tests/test_mcp_registry.py index 89f4bfb6..95aada69 100644 --- a/tests/test_mcp_registry.py +++ b/tests/test_mcp_registry.py @@ -7,7 +7,7 @@ def test_tool_definitions_schema() -> None: - assert len(TOOL_DEFINITIONS) == 121 + assert len(TOOL_DEFINITIONS) == 123 for tool in TOOL_DEFINITIONS: assert tool.get("name") assert tool.get("description") diff --git a/tests/test_mcp_resources.py b/tests/test_mcp_resources.py index 9c181fbf..55abd6f4 100644 --- a/tests/test_mcp_resources.py +++ b/tests/test_mcp_resources.py @@ -12,6 +12,15 @@ def test_resolve_properties_resource() -> None: assert "properties" in text +def test_resolve_report_latest_missing_payload() -> None: + with patch("website_profiling.mcp.server.db_session") as mock_db, patch.object( + mcp_server.AuditToolContext, "load_payload", return_value=None, + ): + mock_db.return_value.__enter__.return_value = object() + text = mcp_server._resolve_resource("audit://property/1/report/latest") + assert "error" in text + + def test_resolve_glossary_and_tools() -> None: text = mcp_server._resolve_resource("audit://tools") assert "tool_count" in text diff --git a/tests/test_mcp_server_helpers.py b/tests/test_mcp_server_helpers.py new file mode 100644 index 00000000..9e19e99d --- /dev/null +++ b/tests/test_mcp_server_helpers.py @@ -0,0 +1,190 @@ +"""MCP server helper and main() coverage.""" +from __future__ import annotations + +import asyncio +import json +import os +import runpy +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from website_profiling.mcp import server as mcp_server + + +def test_default_property_id_env() -> None: + with patch.dict(os.environ, {"WP_PROPERTY_ID": "12"}): + assert mcp_server._default_property_id() == 12 + with patch.dict(os.environ, {"WP_PROPERTY_ID": "0"}): + assert mcp_server._default_property_id() is None + with patch.dict(os.environ, {"WP_PROPERTY_ID": "bad"}): + assert mcp_server._default_property_id() is None + with patch.dict(os.environ, {}, clear=True): + assert mcp_server._default_property_id() is None + + +def test_merge_context() -> None: + with patch.dict(os.environ, {"WP_PROPERTY_ID": "3"}): + ctx = mcp_server._merge_context({"property_id": 9, "report_id": 4}) + assert ctx.property_id == 9 + assert ctx.report_id == 4 + + with patch.dict(os.environ, {"WP_PROPERTY_ID": "3"}): + ctx = mcp_server._merge_context({"property_id": "bad", "report_id": "bad"}) + assert ctx.property_id == 3 + assert ctx.report_id is None + + +def test_payload_index_variants() -> None: + index = mcp_server._payload_index({ + "items": [1, 2], + "meta": {"a": 1}, + "score": 88, + }) + assert index["items"]["count"] == 2 + assert index["meta"]["type"] == "object" + assert index["score"]["type"] == "int" + + +def test_read_glossary_excerpt() -> None: + text = mcp_server._read_glossary_excerpt() + assert isinstance(text, str) + assert text + + +def test_read_glossary_excerpt_missing(monkeypatch) -> None: + monkeypatch.setattr(Path, "is_file", lambda _self: False) + assert mcp_server._read_glossary_excerpt() == "Glossary file not found." + + +def test_tools_catalog_json_includes_security_tools() -> None: + catalog = json.loads(mcp_server._tools_catalog_json()) + assert catalog["tool_count"] >= 123 + assert "get_security_findings" in catalog["domains"]["security"] + + +def test_tools_catalog_json_backlinks_domain() -> None: + fake_tools = [ + { + "name": "get_bing_overview", + "description": "Bing overview without link in name.", + "inputSchema": {"type": "object", "properties": {}}, + }, + ] + with patch("website_profiling.mcp.server.TOOL_DEFINITIONS", fake_tools): + catalog = json.loads(mcp_server._tools_catalog_json()) + assert catalog["domains"]["backlinks"] == ["get_bing_overview"] + + +def test_resolve_glossary_and_report_by_id() -> None: + glossary = mcp_server._resolve_resource("audit://glossary") + assert isinstance(glossary, str) + + with patch("website_profiling.mcp.server.db_session") as mock_db, patch.object( + mcp_server.AuditToolContext, "load_payload", return_value=None, + ): + mock_db.return_value.__enter__.return_value = object() + missing = mcp_server._resolve_resource("audit://property/1/report/99") + assert "error" in missing + + with patch("website_profiling.mcp.server.db_session") as mock_db, patch.object( + mcp_server.AuditToolContext, "load_payload", return_value={"summary": {"score": 1}, "pages": [1, 2]}, + ): + mock_db.return_value.__enter__.return_value = object() + found = mcp_server._resolve_resource("audit://property/1/report/99") + payload = json.loads(found) + assert payload["summary"]["type"] == "object" + + +def test_mcp_main_missing_sdk() -> None: + with patch.dict(sys.modules, {"mcp.server": None, "mcp.server.stdio": None, "mcp.types": None}): + with pytest.raises(SystemExit, match="MCP SDK"): + mcp_server.main() + + +def test_mcp_main_registers_handlers(monkeypatch) -> None: + captured: dict[str, object] = {} + + class FakeServer: + def __init__(self, name: str) -> None: + captured["name"] = name + + def list_tools(self): + def decorator(fn): + captured["list_tools"] = fn + return fn + return decorator + + def call_tool(self): + def decorator(fn): + captured["call_tool"] = fn + return fn + return decorator + + def list_resources(self): + def decorator(fn): + captured["list_resources"] = fn + return fn + return decorator + + def read_resource(self): + def decorator(fn): + captured["read_resource"] = fn + return fn + return decorator + + def create_initialization_options(self): + return {} + + async def run(self, *_args, **_kwargs) -> None: + captured["ran"] = True + + class FakeStdioCM: + async def __aenter__(self): + return (MagicMock(), MagicMock()) + + async def __aexit__(self, *_args): + return False + + fake_server_mod = MagicMock() + fake_server_mod.Server = FakeServer + fake_stdio_mod = MagicMock() + fake_stdio_mod.stdio_server = MagicMock(return_value=FakeStdioCM()) + fake_types_mod = MagicMock() + fake_types_mod.Tool = lambda **kwargs: kwargs + fake_types_mod.TextContent = lambda **kwargs: kwargs + fake_types_mod.Resource = lambda **kwargs: kwargs + + monkeypatch.setitem(sys.modules, "mcp", MagicMock()) + monkeypatch.setitem(sys.modules, "mcp.server", fake_server_mod) + monkeypatch.setitem(sys.modules, "mcp.server.stdio", fake_stdio_mod) + monkeypatch.setitem(sys.modules, "mcp.types", fake_types_mod) + + with patch.dict(os.environ, {"WP_PROPERTY_ID": "7"}, clear=False): + mcp_server.main() + + assert captured["name"] == "site-audit" + assert captured["ran"] is True + tools = asyncio.run(captured["list_tools"]()) # type: ignore[arg-type] + assert len(tools) >= 123 + resources = asyncio.run(captured["list_resources"]()) # type: ignore[arg-type] + assert any(r["uri"] == "audit://property/7" for r in resources) + + with patch("website_profiling.mcp.server.dispatch_tool", return_value={"ok": True}): + content = asyncio.run(captured["call_tool"]("list_properties", {"property_id": 1})) # type: ignore[arg-type] + assert content[0]["text"] == json.dumps({"ok": True}, indent=2, default=str) + read_text = asyncio.run(captured["read_resource"]("audit://tools")) # type: ignore[arg-type] + assert read_text.startswith("{") + + +def test_mcp_package_main(monkeypatch) -> None: + with patch("website_profiling.mcp.server.main") as mock_main: + runpy.run_module("website_profiling.mcp", run_name="__main__") + mock_main.assert_called_once() + + +def test_mcp_server_main_guard() -> None: + with pytest.raises(SystemExit, match="MCP SDK"): + runpy.run_module("website_profiling.mcp.server", run_name="__main__") diff --git a/tests/test_text_sanitize.py b/tests/test_text_sanitize.py new file mode 100644 index 00000000..a73d18ea --- /dev/null +++ b/tests/test_text_sanitize.py @@ -0,0 +1,89 @@ +"""Tests for surrogate stripping in chat/JSON paths.""" +from __future__ import annotations + +import json +from unittest.mock import patch + +from website_profiling.llm.agent import run_agent_turn +from website_profiling.llm.base import ChatResult, ToolCall +from website_profiling.text_sanitize import sanitize_unicode_deep, strip_surrogates +from website_profiling.tools.audit_tools import AuditToolContext + + +def test_strip_surrogates_replaces_lone_surrogate() -> None: + bad = "URL issue\udc9d here" + cleaned = strip_surrogates(bad) + assert "\udc9d" not in cleaned + cleaned.encode("utf-8") + + +def test_sanitize_unicode_deep_nested() -> None: + payload = { + "issues": [{"message": "broken\udc9d", "url": "https://example.com"}], + } + cleaned = sanitize_unicode_deep(payload) + serialized = json.dumps(cleaned, ensure_ascii=False) + serialized.encode("utf-8") + + +def test_sanitize_unicode_deep_tuple() -> None: + cleaned = sanitize_unicode_deep(("ok\udc9d", {"nested": "x\udc9d"})) + assert isinstance(cleaned, tuple) + assert "\udc9d" not in cleaned[0] + assert "\udc9d" not in cleaned[1]["nested"] + + +def test_agent_surrogate_tool_result_does_not_break_llm_request() -> None: + surrogate = "\udc9d" + tool_payload = { + "issues": [ + { + "category": "Technical SEO", + "priority": "High", + "message": f"URL in sitemap but not crawled: https://codefrydev.in/2048{surrogate}", + "url": "https://codefrydev.in/2048", + }, + ], + "total": 1, + "truncated": False, + } + + class RecordingClient: + def __init__(self) -> None: + self.last_messages: list[dict] | None = None + + def chat_with_tools(self, messages, tools, *, on_token=None): + self.last_messages = messages + if self.last_messages and any( + m.get("role") == "tool" for m in self.last_messages + ): + return ChatResult(content="Summary with no further tools.") + return ChatResult( + tool_calls=[ToolCall(id="tc1", name="list_issues", arguments={"priority": "High"})], + ) + + client = RecordingClient() + events: list[dict] = [] + + with patch("website_profiling.llm.agent.load_llm_config_from_db", return_value={ + "llm_enabled": True, "llm_provider": "openai", "llm_api_key": "sk-test", + }): + with patch("website_profiling.llm.agent.get_llm_client", return_value=client): + with patch( + "website_profiling.llm.agent.dispatch_tool", + return_value=tool_payload, + ): + result = run_agent_turn( + [{"role": "user", "content": "high risk audit issues"}], + AuditToolContext(property_id=1), + on_event=events.append, + ) + + assert result["ok"] is True + assert client.last_messages is not None + json.dumps( + {"messages": client.last_messages, "tools": [], "stream": True}, + ensure_ascii=False, + ).encode("utf-8") + tool_end = next(e for e in events if e["type"] == "tool_end") + assert "\udc9d" not in json.dumps(tool_end["result"]) diff --git a/web/app/api/jobs/[id]/cancel/route.ts b/web/app/api/jobs/[id]/cancel/route.ts new file mode 100644 index 00000000..44f99111 --- /dev/null +++ b/web/app/api/jobs/[id]/cancel/route.ts @@ -0,0 +1,32 @@ +import { NextResponse, type NextRequest } from 'next/server'; +import { forbiddenIfNotLocal } from '@/server/localOnly'; +import { requireApiAuth } from '@/server/auth'; +import { cancelPipelineJob } from '@/server/pipelineJobs'; +import type { ApiRouteHandlerWithParams } from '@/types/api'; + +export const runtime = 'nodejs'; + +/** + * POST /api/jobs/:id/cancel — stop a running pipeline job. + */ +export const POST: ApiRouteHandlerWithParams<{ id: string }> = async ( + request: NextRequest, + { params }: { params: Promise<{ id: string }> }, +): Promise => { + const denied = forbiddenIfNotLocal(request); + if (denied) return denied; + const authDenied = requireApiAuth(request); + if (authDenied) return authDenied; + + const { id } = await params; + const result = await cancelPipelineJob(id); + if (!result.ok) { + const status = result.error === 'Job not found' ? 404 : 409; + return NextResponse.json({ error: result.error || 'Unable to cancel job' }, { status }); + } + return NextResponse.json({ + ok: true, + status: result.status, + error: result.error ?? null, + }); +}; diff --git a/web/src/components/chat/ChatSidebar.tsx b/web/src/components/chat/ChatSidebar.tsx index 98dcbb86..c7c3f805 100644 --- a/web/src/components/chat/ChatSidebar.tsx +++ b/web/src/components/chat/ChatSidebar.tsx @@ -254,11 +254,14 @@ export default function ChatSidebar({ {!properties.length ? ( ) : ( - properties.map((p) => ( - - )) + <> + + {properties.map((p) => ( + + ))} + )} diff --git a/web/src/components/pipeline/PipelineRunPanel.tsx b/web/src/components/pipeline/PipelineRunPanel.tsx index f728409e..8e228b29 100644 --- a/web/src/components/pipeline/PipelineRunPanel.tsx +++ b/web/src/components/pipeline/PipelineRunPanel.tsx @@ -19,6 +19,7 @@ import { usePipeline } from '@/context/PipelineContext'; import { useReadOnlySession } from '@/hooks/useReadOnlySession'; import { PipelineStatusBadge, + PipelineStopButton, PRESET_COPY, PresetIcon, } from './pipelineUi'; @@ -27,6 +28,7 @@ import { CRAWL_PRESETS, type CrawlPresetId } from '@/lib/crawlPresets'; import PipelineWizardProgress, { type WizardStep } from './PipelineWizardProgress'; import PipelineLogViewer from './PipelineLogViewer'; import CrawlAuthorizeCheckbox from './CrawlAuthorizeCheckbox'; +import PipelineRunPreviewCard from './PipelineRunPreviewCard'; const s = strings.pipelineRunner; const crawlPresets = s.crawlPresets as Record; @@ -56,6 +58,8 @@ export default function PipelineRunPanel() { status, log, startUrl, + configState, + customCommand, presetId, handleStartUrlChange, handlePresetChange, @@ -63,6 +67,8 @@ export default function PipelineRunPanel() { handleCrawlPresetChange, setField, run, + cancelJob, + stopping, continueInBackground, } = usePipeline(); const { readOnly } = useReadOnlySession(); @@ -318,6 +324,13 @@ export default function PipelineRunPanel() { + +
{busy ? ( - + <> + + + ) : null}
- +
+ {busy ? ( + + ) : null} + +
{log ? ( diff --git a/web/src/components/pipeline/PipelineRunPreviewCard.tsx b/web/src/components/pipeline/PipelineRunPreviewCard.tsx new file mode 100644 index 00000000..75f19014 --- /dev/null +++ b/web/src/components/pipeline/PipelineRunPreviewCard.tsx @@ -0,0 +1,146 @@ +'use client'; + +import { useState } from 'react'; +import { ChevronDown, ChevronUp, Clock, Layers, Settings2 } from 'lucide-react'; +import { strings } from '@/lib/strings'; +import { + buildPipelineRunPreview, + formatPipelineRunDuration, +} from '@/lib/pipelineRunPreview'; +import type { PipelinePresetId } from '@/components/pipeline/pipelinePresets'; +import type { CrawlPresetId } from '@/lib/crawlPresets'; +import type { PipelineConfigState } from '@/types/api'; + +const s = strings.pipelineRunner.runPreview; + +export interface PipelineRunPreviewCardProps { + presetId: PipelinePresetId; + configState: PipelineConfigState; + customCommand?: string; + crawlPresetId?: CrawlPresetId | ''; +} + +export default function PipelineRunPreviewCard({ + presetId, + configState, + customCommand = '', + crawlPresetId = '', +}: PipelineRunPreviewCardProps) { + const [configOpen, setConfigOpen] = useState(false); + const preview = buildPipelineRunPreview({ + presetId, + configState, + customCommand, + crawlPresetId, + }); + + const duration = formatPipelineRunDuration(preview.timeMinSeconds, preview.timeMaxSeconds); + + return ( +
+
+
+

{s.title}

+

{s.hint}

+
+
+ + {s.estimatedTime}: {duration} +
+
+ +
+
+

+ {s.maxPagesLabel} +

+

+ {preview.maxCrawlPages != null ? preview.maxCrawlPages.toLocaleString() : '—'} +

+
+
+

+ {s.lighthousePagesLabel} +

+

+ {preview.lighthousePages != null ? preview.lighthousePages.toLocaleString() : '—'} +

+
+
+

+ {s.stepsLabel} +

+

+ {preview.phases.length} +

+
+
+ +
+

+ + {s.whatRunsLabel} +

+
    + {preview.phases.map((phase) => ( +
  • + +
    +

    {phase.label}

    + {phase.detail ? ( +

    {phase.detail}

    + ) : null} +
    +
    • + ))} +
+
+ +
    + {preview.summaryLines.map((line) => ( +
  • {line}
    • + ))} +
  • {s.estimateDisclaimer}
    • +
+ + {preview.configRows.length > 0 ? ( +
+ + {configOpen ? ( +
+ {preview.configRows.map((row) => ( +
+
+ {row.label} +
+
{row.value}
+
+ ))} +
+ ) : null} +
+ ) : null} +
+ ); +} diff --git a/web/src/components/pipeline/PipelineRunnerFab.tsx b/web/src/components/pipeline/PipelineRunnerFab.tsx index 3c216848..1f8e47ba 100644 --- a/web/src/components/pipeline/PipelineRunnerFab.tsx +++ b/web/src/components/pipeline/PipelineRunnerFab.tsx @@ -1,6 +1,6 @@ 'use client'; -import { Loader2, Maximize2, Terminal } from 'lucide-react'; +import { Loader2, Maximize2, Square, Terminal } from 'lucide-react'; import { usePathname, useRouter, useSearchParams } from 'next/navigation'; import { strings } from '@/lib/strings'; import { usePipeline } from '@/context/PipelineContext'; @@ -16,7 +16,7 @@ export default function PipelineRunnerFab() { const pathname = usePathname(); const router = useRouter(); const searchParams = useSearchParams(); - const { busy, status, log, backgroundMode, openPipelinePage } = usePipeline(); + const { busy, status, log, backgroundMode, stopping, cancelJob, openPipelinePage } = usePipeline(); const { loading: sessionLoading, canMutate } = useSession(); const onPipelinePage = pathname === '/pipeline' || pathname.startsWith('/pipeline/'); @@ -66,6 +66,20 @@ export default function PipelineRunnerFab() { : 'Idle'}

+ + + + ); +} + +function buildSectionTabs(group: PipelineSettingsGroup | undefined): { id: string; label: string }[] { + if (!group) return []; + + const tabs: { id: string; label: string }[] = []; + + if (group.id === 'google') { + tabs.push({ id: 'integrations', label: s.settingsTabIntegrations }); + } + + for (const sectionId of group.sectionIds) { + const section = PIPELINE_CONFIG_SECTIONS.find((sec) => sec.id === sectionId); + if (section) { + tabs.push({ id: section.id, label: section.label }); + } + } + + if (group.includesLlm) { + for (const section of LLM_CONFIG_SECTIONS) { + tabs.push({ id: section.id, label: section.label }); + } + } + + if (group.id === 'advanced') { + tabs.push({ id: 'runner', label: s.settingsTabRunner }); + } + + return tabs; +} + export function PipelineSettingsSaveBar({ onSaved }: { onSaved?: () => void }) { const { loading, saving, saveMsg, busy, saveSettings } = usePipeline(); const { readOnly } = useReadOnlySession(); @@ -74,24 +209,27 @@ export function PipelineSettingsSaveBar({ onSaved }: { onSaved?: () => void }) { }; const saveFailed = saveMsg.includes('Save failed') || saveMsg.includes('failed'); + const saveDisabled = saving || loading || readOnly; + + const statusHint = saveMsg + ? saveMsg + : busy + ? s.settingsSaveWhileRunningHint + : s.settingsSubtitle; return (
- {saveMsg ? ( - - {saveMsg} - - ) : ( - {s.settingsSubtitle} - )} + + {statusHint} +
-
- - ), - }); + ) : null + } + /> + ); } - return panels; - }, [ - group, - busy, - configState, - llmConfigState, - googleIntegrationsToast, - customCommand, - pythonExe, - repoRoot, - unknownKeys, - setField, - setLlmField, - setCustomCommand, - setPythonExe, - setRepoRoot, - resetConfig, - ]); - - const useSectionTabs = sectionPanels.length > 1; - const [activeSectionTab, setActiveSectionTab] = useState(sectionPanels[0]?.id ?? ''); - - useEffect(() => { - setActiveSectionTab(sectionPanels[0]?.id ?? ''); - }, [activeGroup]); - - useEffect(() => { - setActiveSectionTab((current) => - sectionPanels.some((p) => p.id === current) ? current : (sectionPanels[0]?.id ?? ''), - ); - }, [sectionPanels]); - - const activePanel = sectionPanels.find((p) => p.id === activeSectionTab) ?? sectionPanels[0]; + return null; + }; if (!group) { return null; @@ -332,6 +378,11 @@ export default function PipelineSettingsPanel({ return (
+ {readOnly ? ( +
+

{strings.app.readonlyBanner}

+
+ ) : null} {showBrowserCrawlBanner ? (

@@ -388,26 +439,24 @@ export default function PipelineSettingsPanel({ {useSectionTabs ? ( <> ({ id: p.id, label: p.label }))} + tabs={sectionTabs} activeTab={activeSectionTab} onChange={setActiveSectionTab} ariaLabel={s.settingsSectionTabsLabel} /> - {activePanel ? ( + {activeTabId ? (

- {activePanel.content} + {renderSectionContent(activeTabId)}
) : null} ) : ( -
- {activePanel?.content} -
+
{activeTabId ? renderSectionContent(activeTabId) : null}
)}
)} diff --git a/web/src/components/pipeline/pipelineUi.tsx b/web/src/components/pipeline/pipelineUi.tsx index 25153298..3b8fb4b0 100644 --- a/web/src/components/pipeline/pipelineUi.tsx +++ b/web/src/components/pipeline/pipelineUi.tsx @@ -9,12 +9,16 @@ import { Loader2, ScanSearch, Sparkles, + Square, Wrench, } from 'lucide-react'; import type { PipelineJobStatus } from '@/types/api'; import type { PipelinePresetId } from './pipelinePresets'; import type { PipelineSettingsGroupId } from './pipelineSettingsGroups'; import { strings } from '@/lib/strings'; +import Button from '@/components/Button'; + +const s = strings.pipelineRunner; const presetStrings = strings.pipelineRunner.presets; @@ -108,6 +112,35 @@ export function PipelineStatusBadge({ ); } +export function PipelineStopButton({ + onClick, + disabled, + stopping, + className = '', +}: { + onClick: () => void | Promise | Promise; + disabled?: boolean; + stopping?: boolean; + className?: string; +}) { + return ( + + ); +} + export function PresetIcon({ presetId, selected, diff --git a/web/src/context/PipelineContext.tsx b/web/src/context/PipelineContext.tsx index 4620134f..84004cbd 100644 --- a/web/src/context/PipelineContext.tsx +++ b/web/src/context/PipelineContext.tsx @@ -60,6 +60,8 @@ export interface PipelineContextValue { pythonExe: string; repoRoot: string; busy: boolean; + stopping: boolean; + activeJobId: string; log: string; status: PipelineJobStatus | ''; backgroundMode: boolean; @@ -83,6 +85,7 @@ export interface PipelineContextValue { saveSettings: () => Promise; saveLlmModel: (model: string) => Promise; run: () => Promise; + cancelJob: () => Promise; continueInBackground: () => void; openPipelinePage: (tab?: PipelineTab) => void; } @@ -120,6 +123,7 @@ export function PipelineProvider({ children }: { children: ReactNode }) { const [pythonExe, setPythonExe] = useState(() => loadPipelineRunnerPrefs().pythonExe); const [repoRoot, setRepoRoot] = useState(() => loadPipelineRunnerPrefs().repoRoot); const [busy, setBusy] = useState(false); + const [stopping, setStopping] = useState(false); const [log, setLog] = useState(''); const [status, setStatus] = useState(''); const [backgroundMode, setBackgroundMode] = useState(false); @@ -128,6 +132,7 @@ export function PipelineProvider({ children }: { children: ReactNode }) { const [browserCrawlChecking, setBrowserCrawlChecking] = useState(false); const [crawlPresetId, setCrawlPresetId] = useState(''); const pollStopRef = useRef<(() => void) | null>(null); + const activeJobIdRef = useRef(''); const refreshBrowserCrawlStatus = useCallback(async () => { setBrowserCrawlChecking(true); @@ -182,7 +187,9 @@ export function PipelineProvider({ children }: { children: ReactNode }) { ) => { if (!jobId) return; stopPoll(); + activeJobIdRef.current = jobId; setBusy(true); + setStopping(false); setLog(''); setStatus('running'); setBackgroundMode(false); @@ -206,7 +213,9 @@ export function PipelineProvider({ children }: { children: ReactNode }) { setStatus(job.status); if (job.status === 'success' || job.status === 'error') { stopPoll(); + activeJobIdRef.current = ''; setBusy(false); + setStopping(false); if (job.status === 'error') { logPipelineFailure('Job finished with error', { jobId, @@ -515,6 +524,7 @@ export function PipelineProvider({ children }: { children: ReactNode }) { }); setStatus('error'); setLog(message); + activeJobIdRef.current = ''; setBusy(false); } }, [ @@ -534,6 +544,40 @@ export function PipelineProvider({ children }: { children: ReactNode }) { router.push(readPipelineReturnPath()); }, [router]); + const cancelJob = useCallback(async (): Promise => { + const jobId = activeJobIdRef.current; + if (!jobId || stopping) return false; + setStopping(true); + try { + const res = await fetch(apiUrl(`/jobs/${encodeURIComponent(jobId)}/cancel`), { + method: 'POST', + }); + const data = await res.json().catch(() => ({})); + if (!res.ok) { + const message = typeof data.error === 'string' ? data.error : res.statusText; + logPipelineFailure('Cancel job failed', { jobId, message, status: res.status }); + setStatus('error'); + setLog(format(s.stopJobFailed, { message })); + stopPoll(); + activeJobIdRef.current = ''; + setBusy(false); + return false; + } + return true; + } catch (e) { + const message = e instanceof Error ? e.message : String(e); + logPipelineFailure('Cancel job failed', { jobId, message, error: e }); + setStatus('error'); + setLog(format(s.stopJobFailed, { message })); + stopPoll(); + activeJobIdRef.current = ''; + setBusy(false); + return false; + } finally { + setStopping(false); + } + }, [stopPoll, stopping]); + const value = useMemo( () => ({ presetId, @@ -550,6 +594,8 @@ export function PipelineProvider({ children }: { children: ReactNode }) { pythonExe, repoRoot, busy, + stopping, + activeJobId: activeJobIdRef.current, log, status, backgroundMode, @@ -573,6 +619,7 @@ export function PipelineProvider({ children }: { children: ReactNode }) { saveSettings, saveLlmModel, run, + cancelJob, continueInBackground, openPipelinePage, }), @@ -591,6 +638,7 @@ export function PipelineProvider({ children }: { children: ReactNode }) { pythonExe, repoRoot, busy, + stopping, log, status, backgroundMode, @@ -607,6 +655,7 @@ export function PipelineProvider({ children }: { children: ReactNode }) { saveSettings, saveLlmModel, run, + cancelJob, continueInBackground, openPipelinePage, ], diff --git a/web/src/lib/googlePropertySelection.test.ts b/web/src/lib/googlePropertySelection.test.ts new file mode 100644 index 00000000..2afc5c06 --- /dev/null +++ b/web/src/lib/googlePropertySelection.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest'; +import { pickInitialPropertyId, propertyIdsEqual } from './googlePropertySelection'; +import type { PropertyPickCandidate } from './googlePropertySelection'; + +function row( + id: number | string, + canonical_domain: string, + site_url: string | null = null, +): PropertyPickCandidate { + return { + id, + canonical_domain, + site_url, + }; +} + +describe('propertyIdsEqual', () => { + it('matches string and numeric ids', () => { + expect(propertyIdsEqual('1', 1)).toBe(true); + expect(propertyIdsEqual(1, '1')).toBe(true); + expect(propertyIdsEqual('1', 2)).toBe(false); + }); +}); + +describe('pickInitialPropertyId', () => { + const properties = [ + row(1, 'codefrydev.in', 'https://codefrydev.in'), + row(2, 'example.com'), + ]; + + it('falls back to first property when explicitId is stale', () => { + expect( + pickInitialPropertyId(properties, { + explicitId: 99999, + }), + ).toBe(1); + }); + + it('matches explicitId when property id is a string from JSON', () => { + const stringIds = [row('1', 'codefrydev.in'), row('2', 'example.com')]; + expect( + pickInitialPropertyId(stringIds, { + explicitId: 1, + }), + ).toBe(1); + }); + + it('matches by startUrl when explicitId is invalid', () => { + expect( + pickInitialPropertyId(properties, { + explicitId: 99999, + startUrl: 'https://example.com', + }), + ).toBe(2); + }); + + it('uses activePropertyId when explicitId is invalid', () => { + expect( + pickInitialPropertyId(properties, { + explicitId: 99999, + activePropertyId: '2', + }), + ).toBe(2); + }); + + it('returns null for empty list', () => { + expect(pickInitialPropertyId([], { explicitId: 1 })).toBeNull(); + }); +}); diff --git a/web/src/lib/googlePropertySelection.ts b/web/src/lib/googlePropertySelection.ts index 2c1c3b91..b6faa998 100644 --- a/web/src/lib/googlePropertySelection.ts +++ b/web/src/lib/googlePropertySelection.ts @@ -1,9 +1,32 @@ import { hostsMatch, normalizeDomainQueryParam } from '@/lib/domainSlug'; -import type { PropertyListItem } from '@/types/api'; + +export interface PropertyPickCandidate { + id: number | string; + canonical_domain: string; + site_url?: string | null; +} + +/** Coerce API / storage values to a finite positive integer property id. */ +export function normalizePropertyId(value: unknown): number | null { + const n = typeof value === 'number' ? value : Number(value); + if (!Number.isFinite(n) || n <= 0 || !Number.isInteger(n)) return null; + return n; +} + +/** Compare property ids across number/string sources (e.g. PostgreSQL BIGINT in JSON). */ +export function propertyIdsEqual(a: unknown, b: unknown): boolean { + const na = normalizePropertyId(a); + const nb = normalizePropertyId(b); + return na != null && nb != null && na === nb; +} + +function idFromRow(row: PropertyPickCandidate): number | null { + return normalizePropertyId(row.id); +} /** Pick initial property: explicit id → URL match → active_property_id → first list item. */ export function pickInitialPropertyId( - properties: PropertyListItem[], + properties: PropertyPickCandidate[], options: { explicitId?: number | null; startUrl?: string; @@ -13,8 +36,8 @@ export function pickInitialPropertyId( if (properties.length === 0) return null; if (options.explicitId != null && Number.isFinite(options.explicitId)) { - const found = properties.some((p) => p.id === options.explicitId); - if (found) return options.explicitId; + const found = properties.some((p) => propertyIdsEqual(p.id, options.explicitId)); + if (found) return normalizePropertyId(options.explicitId); } let startHost = ''; @@ -33,21 +56,21 @@ export function pickInitialPropertyId( hostsMatch(p.canonical_domain, startHost) || hostsMatch(p.site_url || '', startHost), ); - if (byUrl) return byUrl.id; + if (byUrl) return idFromRow(byUrl); } const activeRaw = (options.activePropertyId || '').trim(); if (activeRaw) { const pid = parseInt(activeRaw, 10); - if (Number.isFinite(pid) && properties.some((p) => p.id === pid)) { - return pid; + if (Number.isFinite(pid) && properties.some((p) => propertyIdsEqual(p.id, pid))) { + return normalizePropertyId(pid); } } - return properties[0]?.id ?? null; + return idFromRow(properties[0]!); } -export function siteUrlFromProperty(row: PropertyListItem): string { +export function siteUrlFromProperty(row: PropertyPickCandidate): string { const fromRow = (row.site_url || '').trim(); if (fromRow) return fromRow; const host = (row.canonical_domain || '').trim(); diff --git a/web/src/lib/pipelineRunPreview.test.ts b/web/src/lib/pipelineRunPreview.test.ts new file mode 100644 index 00000000..1158b57c --- /dev/null +++ b/web/src/lib/pipelineRunPreview.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from 'vitest'; +import { buildInitialPipelineConfigState } from '@/lib/pipelineConfigSchema'; +import { buildPipelineRunPreview } from '@/lib/pipelineRunPreview'; + +describe('buildPipelineRunPreview', () => { + it('includes crawl and report for full audit', () => { + const preview = buildPipelineRunPreview({ + presetId: 'full-audit', + configState: buildInitialPipelineConfigState(), + }); + expect(preview.maxCrawlPages).toBe(500); + expect(preview.phases.some((p) => p.id === 'crawl')).toBe(true); + expect(preview.phases.some((p) => p.id === 'report')).toBe(true); + expect(preview.timeMaxSeconds).toBeGreaterThan(preview.timeMinSeconds); + }); + + it('limits phases for crawl-only preset', () => { + const preview = buildPipelineRunPreview({ + presetId: 'crawl-only', + configState: buildInitialPipelineConfigState(), + }); + expect(preview.phases.map((p) => p.id)).toEqual(['crawl']); + expect(preview.phases.some((p) => p.id === 'report')).toBe(false); + }); + + it('reflects crawl preset max pages', () => { + const preview = buildPipelineRunPreview({ + presetId: 'full-audit', + configState: { ...buildInitialPipelineConfigState(), max_pages: '2000' }, + crawlPresetId: 'spa', + }); + expect(preview.maxCrawlPages).toBe(2000); + expect(preview.configRows.some((r) => r.label === 'Crawl preset' && r.value === 'SPA / JavaScript')).toBe( + true, + ); + }); +}); diff --git a/web/src/lib/pipelineRunPreview.ts b/web/src/lib/pipelineRunPreview.ts new file mode 100644 index 00000000..52c6bc68 --- /dev/null +++ b/web/src/lib/pipelineRunPreview.ts @@ -0,0 +1,287 @@ +import { applyPreset, getPresetById, type PipelinePresetId } from '@/components/pipeline/pipelinePresets'; +import { getCrawlPresetById, isCrawlPresetId, type CrawlPresetId } from '@/lib/crawlPresets'; +import type { PipelineConfigState } from '@/types/api'; + +export interface PipelineRunPhase { + id: string; + label: string; + detail?: string; +} + +export interface PipelineRunPreview { + phases: PipelineRunPhase[]; + maxCrawlPages: number | null; + lighthousePages: number | null; + timeMinSeconds: number; + timeMaxSeconds: number; + configRows: { label: string; value: string }[]; + summaryLines: string[]; +} + +function isTruthy(value: string | boolean | undefined, defaultWhenUnset = false): boolean { + if (value === true || value === 'true') return true; + if (value === false || value === 'false') return false; + return defaultWhenUnset; +} + +function num(value: string | boolean | undefined, fallback: number): number { + if (value == null || value === '') return fallback; + const n = Number(String(value).trim()); + return Number.isFinite(n) ? n : fallback; +} + +function renderModeLabel(mode: string): string { + if (mode === 'javascript') return 'JavaScript (browser)'; + if (mode === 'auto') return 'Auto (static + JS when needed)'; + return 'Static HTML'; +} + +function formatDurationRange(minSeconds: number, maxSeconds: number): string { + const fmt = (sec: number) => { + if (sec < 90) return `${Math.max(1, Math.round(sec))} sec`; + if (sec < 3600) return `${Math.max(1, Math.round(sec / 60))} min`; + const h = Math.floor(sec / 3600); + const m = Math.round((sec % 3600) / 60); + return m > 0 ? `${h} hr ${m} min` : `${h} hr`; + }; + if (maxSeconds <= minSeconds * 1.15) { + return `~${fmt(minSeconds)}`; + } + return `${fmt(minSeconds)} – ${fmt(maxSeconds)}`; +} + +export function formatPipelineRunDuration(minSeconds: number, maxSeconds: number): string { + return formatDurationRange(minSeconds, maxSeconds); +} + +interface RunPlan { + command: string; + state: PipelineConfigState; + includesCrawl: boolean; + includesReport: boolean; + includesPlot: boolean; + includesStandaloneLighthouse: boolean; + includesLighthouseOnPages: boolean; + includesGoogle: boolean; + includesKeywords: boolean; +} + +function resolveRunPlan( + presetId: PipelinePresetId, + configState: PipelineConfigState, + customCommand: string, +): RunPlan { + const preset = getPresetById(presetId); + const command = customCommand.trim() || preset.command; + const { configState: state } = applyPreset(presetId, configState); + + const includesCrawl = + command === 'crawl' || + (!command && isTruthy(state.run_crawl, true)); + const includesReport = + command === 'report' || + (!command && isTruthy(state.run_report, true)); + const includesPlot = !command && isTruthy(state.run_plot, true); + const includesStandaloneLighthouse = + command === 'lighthouse' || + (!command && + isTruthy(state.run_lighthouse, true) && + !isTruthy(state.run_lighthouse_on_pages, true)); + const includesLighthouseOnPages = + !command && isTruthy(state.run_lighthouse_on_pages, true) && includesCrawl; + const includesGoogle = command === 'google'; + const includesKeywords = command.startsWith('keywords'); + + return { + command, + state, + includesCrawl, + includesReport, + includesPlot, + includesStandaloneLighthouse, + includesLighthouseOnPages, + includesGoogle, + includesKeywords, + }; +} + +function estimateCrawlSeconds(state: PipelineConfigState, maxPages: number): { min: number; max: number } { + const concurrency = Math.max(1, num(state.concurrency, 8)); + const politeDelay = Math.max(0, num(state.polite_delay, 0.2)); + const renderMode = String(state.crawl_render_mode ?? 'static').toLowerCase(); + const perPage = + renderMode === 'javascript' ? 5.5 : renderMode === 'auto' ? 2.8 : 1.1; + const batchSeconds = (maxPages / concurrency) * (perPage + politeDelay); + return { min: batchSeconds * 0.75, max: batchSeconds * 1.35 }; +} + +function estimateLighthouseSeconds(pageCount: number, concurrency: number): { min: number; max: number } { + const c = Math.max(1, concurrency); + const batch = (pageCount / c) * 55; + return { min: batch * 0.85, max: batch * 1.25 }; +} + +export function buildPipelineRunPreview({ + presetId, + configState, + customCommand = '', + crawlPresetId = '', +}: { + presetId: PipelinePresetId; + configState: PipelineConfigState; + customCommand?: string; + crawlPresetId?: CrawlPresetId | ''; +}): PipelineRunPreview { + const plan = resolveRunPlan(presetId, configState, customCommand); + const { state } = plan; + + const maxCrawlPages = plan.includesCrawl ? num(state.max_pages, 500) : null; + const lhOnPages = plan.includesLighthouseOnPages + ? num(state.lighthouse_max_pages, 2) + : null; + const lhStandalone = plan.includesStandaloneLighthouse ? 1 : null; + const lighthousePages = lhOnPages ?? lhStandalone; + + const phases: PipelineRunPhase[] = []; + if (plan.includesCrawl && maxCrawlPages != null) { + phases.push({ + id: 'crawl', + label: 'Site crawl', + detail: `Up to ${maxCrawlPages.toLocaleString()} URLs`, + }); + } + if (plan.includesReport) { + phases.push({ id: 'report', label: 'Audit report', detail: 'Issues, links, and on-page analysis' }); + } + if (plan.includesPlot) { + phases.push({ id: 'plot', label: 'Charts & exports', detail: 'Visual summaries and data files' }); + } + if (plan.includesLighthouseOnPages && lighthousePages != null) { + phases.push({ + id: 'lighthouse-pages', + label: 'Lighthouse (sampled pages)', + detail: `${lighthousePages.toLocaleString()} URL${lighthousePages === 1 ? '' : 's'}`, + }); + } + if (plan.includesStandaloneLighthouse) { + phases.push({ + id: 'lighthouse-single', + label: 'Lighthouse (single URL)', + detail: String(state.lighthouse_url || state.start_url || 'Start URL').trim() || 'Start URL', + }); + } + if (plan.includesGoogle) { + phases.push({ id: 'google', label: 'Google Search Console sync', detail: 'GSC + GA4 data pull' }); + } + if (plan.includesKeywords) { + phases.push({ id: 'keywords', label: 'Keywords explorer', detail: 'GSC keywords with optional enrichment' }); + } + + let timeMin = 0; + let timeMax = 0; + + if (plan.includesCrawl && maxCrawlPages != null) { + const crawl = estimateCrawlSeconds(state, maxCrawlPages); + timeMin += crawl.min; + timeMax += crawl.max; + } + if (plan.includesReport) { + const pages = maxCrawlPages ?? num(state.analysis_dup_max_pages, 2000); + timeMin += 30 + pages * 0.02; + timeMax += 90 + pages * 0.06; + } + if (plan.includesPlot) { + timeMin += 20; + timeMax += 60; + } + if (lighthousePages != null && lighthousePages > 0) { + const lh = estimateLighthouseSeconds( + lighthousePages, + num(state.lighthouse_concurrency, 2), + ); + timeMin += lh.min; + timeMax += lh.max; + } + if (plan.includesGoogle) { + timeMin += 120; + timeMax += 360; + } + if (plan.includesKeywords) { + timeMin += 180; + timeMax += 720; + } + + timeMin = Math.max(15, Math.round(timeMin)); + timeMax = Math.max(timeMin + 10, Math.round(timeMax)); + + const renderMode = String(state.crawl_render_mode ?? 'static'); + const crawlPresetLabel = + crawlPresetId && isCrawlPresetId(crawlPresetId) + ? getCrawlPresetById(crawlPresetId).label + : null; + + const configRows: { label: string; value: string }[] = []; + + if (plan.includesCrawl) { + configRows.push( + { label: 'Crawl limit', value: `${maxCrawlPages?.toLocaleString() ?? '—'} URLs` }, + { label: 'Render mode', value: renderModeLabel(renderMode) }, + { label: 'Concurrent requests', value: String(num(state.concurrency, 8)) }, + { label: 'Crawl delay', value: `${num(state.polite_delay, 0.2)}s` }, + { label: 'Max depth', value: String(num(state.max_depth, 6)) }, + ); + if (crawlPresetLabel) { + configRows.push({ label: 'Crawl preset', value: crawlPresetLabel }); + } + } + if (plan.includesLighthouseOnPages || plan.includesStandaloneLighthouse) { + configRows.push( + { + label: 'Lighthouse strategy', + value: String(state.lighthouse_strategy || 'mobile'), + }, + { + label: 'Lighthouse URLs', + value: lighthousePages != null ? String(lighthousePages) : '—', + }, + ); + } + if (plan.includesReport) { + configRows.push({ + label: 'Property name', + value: String(state.site_name || '').trim() || '(from site URL)', + }); + } + + const summaryLines: string[] = []; + if (plan.includesCrawl && maxCrawlPages != null) { + summaryLines.push( + `Crawls up to ${maxCrawlPages.toLocaleString()} pages using ${renderModeLabel(renderMode).toLowerCase()}.`, + ); + } + if (plan.includesReport) { + summaryLines.push('Builds a full SEO audit report from crawl data.'); + } + if (lighthousePages != null && lighthousePages > 0) { + summaryLines.push(`Runs Lighthouse on ${lighthousePages.toLocaleString()} page(s).`); + } + if (plan.includesGoogle) { + summaryLines.push('Pulls Search Console and Analytics metrics.'); + } + if (plan.includesKeywords) { + summaryLines.push('Generates keyword clusters from Search Console.'); + } + if (summaryLines.length === 0) { + summaryLines.push('Review settings below before starting.'); + } + + return { + phases, + maxCrawlPages, + lighthousePages, + timeMinSeconds: timeMin, + timeMaxSeconds: timeMax, + configRows, + summaryLines, + }; +} diff --git a/web/src/server/jobsCancelRoute.test.ts b/web/src/server/jobsCancelRoute.test.ts new file mode 100644 index 00000000..9033482c --- /dev/null +++ b/web/src/server/jobsCancelRoute.test.ts @@ -0,0 +1,48 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { localRequest, remoteRequest } from '@/server/testHelpers/routeTestUtils'; + +const cancelMock = vi.fn(); + +vi.mock('@/server/pipelineJobs', () => ({ + cancelPipelineJob: (...args: unknown[]) => cancelMock(...args), +})); + +vi.mock('@/server/auth', () => ({ + requireApiAuth: () => null, +})); + +describe('jobs cancel route', () => { + beforeEach(() => { + cancelMock.mockReset(); + vi.resetModules(); + cancelMock.mockResolvedValue({ ok: true, status: 'error', error: 'Cancelled by user' }); + }); + + it('returns 403 for non-local host', async () => { + const { POST } = await import('../../app/api/jobs/[id]/cancel/route'); + const res = await POST(remoteRequest('/api/jobs/abc/cancel'), { + params: Promise.resolve({ id: 'abc' }), + }); + expect(res.status).toBe(403); + }); + + it('cancels a running job for local request', async () => { + const { POST } = await import('../../app/api/jobs/[id]/cancel/route'); + const res = await POST(localRequest('/api/jobs/job-1/cancel'), { + params: Promise.resolve({ id: 'job-1' }), + }); + expect(res.status).toBe(200); + expect(cancelMock).toHaveBeenCalledWith('job-1'); + const body = await res.json(); + expect(body.ok).toBe(true); + }); + + it('returns 409 when job is not running', async () => { + cancelMock.mockResolvedValue({ ok: false, status: 'success', error: 'Job is not running' }); + const { POST } = await import('../../app/api/jobs/[id]/cancel/route'); + const res = await POST(localRequest('/api/jobs/job-1/cancel'), { + params: Promise.resolve({ id: 'job-1' }), + }); + expect(res.status).toBe(409); + }); +}); diff --git a/web/src/server/pipelineJobs.ts b/web/src/server/pipelineJobs.ts index cf0174cc..3ecb1799 100644 --- a/web/src/server/pipelineJobs.ts +++ b/web/src/server/pipelineJobs.ts @@ -1,4 +1,4 @@ -import { spawn } from 'child_process'; +import { spawn, type ChildProcess } from 'child_process'; import path from 'path'; import fs from 'fs'; import { randomUUID } from 'crypto'; @@ -6,13 +6,14 @@ import { getPipelineSpawnEnv } from '@/server/pipelineSpawnEnv'; import { formatPythonSpawnError, resolvePythonExecutable } from '@/server/resolvePython'; import { appendPipelineJobLog, + cancelPipelineJobInDb, finishPipelineJob, getPipelineJobFromDb, insertPipelineJob, isAnyPipelineJobRunning, reconcileStaleRunningJobs, } from '@/server/pipelineJobsDb'; -import type { PipelineJob, PipelineJobStore } from '@/types/api'; +import type { PipelineJob, PipelineJobEntry, PipelineJobStore } from '@/types/api'; function isDbJobsEnabled(): boolean { return Boolean((process.env.DATABASE_URL || '').trim()); @@ -44,13 +45,41 @@ const ALLOWED_COMMANDS = new Set([ function getStore(): PipelineJobStore { if (!globalThis.__websiteProfilingPipelineJobs) { globalThis.__websiteProfilingPipelineJobs = { - jobs: new Map(), + jobs: new Map(), running: false, }; } return globalThis.__websiteProfilingPipelineJobs; } +function getProcessMap(): Map { + if (!globalThis.__websiteProfilingPipelineProcesses) { + globalThis.__websiteProfilingPipelineProcesses = new Map(); + } + return globalThis.__websiteProfilingPipelineProcesses; +} + +const CANCELLED_MESSAGE = 'Cancelled by user'; + +function markJobFinished( + id: string, + entry: PipelineJobEntry, + status: 'success' | 'error', + exitCode: number | null, + error?: string, +): void { + if (entry.finished) return; + entry.finished = true; + entry.status = status; + entry.exitCode = exitCode; + if (error) entry.error = error; + getStore().running = false; + getProcessMap().delete(id); + if (isDbJobsEnabled()) { + void finishPipelineJob(id, status, exitCode, error).catch(() => {}); + } +} + function sanitizePython(py: string | undefined | null, repoRoot: string): string { const resolved = resolvePythonExecutable(py, repoRoot); if (resolved.length > 256) throw new Error('Python path too long'); @@ -140,7 +169,7 @@ export function startPipelineJob( } const id = randomUUID(); - const entry: PipelineJob = { + const entry: PipelineJobEntry = { status: 'running', exitCode: null, log: '', @@ -164,6 +193,7 @@ export function startPipelineJob( env: getPipelineSpawnEnv(repoRoot, options.propertyId ?? null), shell: false, }); + getProcessMap().set(id, proc); const append = (chunk: Buffer | string): void => { const text = chunk.toString(); @@ -180,28 +210,26 @@ export function startPipelineJob( proc.stderr?.on('data', append); proc.on('error', (err: Error) => { - entry.status = 'error'; - entry.error = formatPythonSpawnError(err, pythonExe, repoRoot); - entry.exitCode = -1; - store.running = false; - if (isDbJobsEnabled()) { - void finishPipelineJob(id, 'error', -1, entry.error).catch(() => {}); - } + if (entry.finished) return; + const message = formatPythonSpawnError(err, pythonExe, repoRoot); + markJobFinished(id, entry, 'error', -1, message); }); proc.on('close', (code: number | null) => { - entry.exitCode = code; - entry.status = code === 0 ? 'success' : 'error'; - if (code !== 0 && !entry.error) { + if (entry.finished) return; + if (entry.cancelled) { + markJobFinished(id, entry, 'error', code ?? -1, CANCELLED_MESSAGE); + return; + } + const status = code === 0 ? 'success' : 'error'; + let error: string | undefined; + if (code !== 0) { const tail = entry.log.trim().slice(-500); - entry.error = tail + error = tail ? `Process exited with code ${code ?? 'unknown'}` : `Process exited with code ${code ?? 'unknown'} (no output captured)`; } - store.running = false; - if (isDbJobsEnabled()) { - void finishPipelineJob(id, entry.status, code, entry.error).catch(() => {}); - } + markJobFinished(id, entry, status, code, error); }); return id; @@ -221,3 +249,67 @@ export async function getJob(id: string): Promise { export function getJobSync(id: string): PipelineJob | null { return getStore().jobs.get(id) ?? null; } + +export interface CancelPipelineJobResult { + ok: boolean; + status: PipelineJob['status']; + error?: string; +} + +/** + * Stop a running pipeline job. Kills the child process when this server instance + * spawned it; otherwise marks the DB row cancelled (best effort after restart). + */ +export async function cancelPipelineJob(id: string): Promise { + const trimmed = id.trim(); + if (!trimmed) { + return { ok: false, status: 'error', error: 'Job id is required' }; + } + + const store = getStore(); + const entry = store.jobs.get(trimmed); + const proc = getProcessMap().get(trimmed); + + if (entry?.status === 'running' && proc && !proc.killed) { + entry.cancelled = true; + entry.error = CANCELLED_MESSAGE; + const cancelLine = `\n[Cancelled] ${CANCELLED_MESSAGE}\n`; + entry.log += cancelLine; + if (isDbJobsEnabled()) { + void appendPipelineJobLog(trimmed, cancelLine).catch(() => {}); + } + try { + proc.kill(); + } catch { + /* process may already be gone */ + } + return { ok: true, status: 'running' }; + } + + if (entry?.status === 'running') { + entry.cancelled = true; + markJobFinished(trimmed, entry, 'error', -1, CANCELLED_MESSAGE); + return { ok: true, status: 'error', error: CANCELLED_MESSAGE }; + } + + if (isDbJobsEnabled()) { + const fromDb = await getPipelineJobFromDb(trimmed); + if (!fromDb) { + return { ok: false, status: 'error', error: 'Job not found' }; + } + if (fromDb.status !== 'running') { + return { ok: false, status: fromDb.status, error: 'Job is not running' }; + } + const updated = await cancelPipelineJobInDb(trimmed, CANCELLED_MESSAGE); + if (!updated) { + return { ok: false, status: fromDb.status, error: 'Job is not running' }; + } + store.running = false; + return { ok: true, status: 'error', error: CANCELLED_MESSAGE }; + } + + if (!entry) { + return { ok: false, status: 'error', error: 'Job not found' }; + } + return { ok: false, status: entry.status, error: 'Job is not running' }; +} diff --git a/web/src/server/pipelineJobsDb.ts b/web/src/server/pipelineJobsDb.ts index 421f36d5..6a5491d6 100644 --- a/web/src/server/pipelineJobsDb.ts +++ b/web/src/server/pipelineJobsDb.ts @@ -40,6 +40,25 @@ export async function appendPipelineJobLog(id: string, chunk: string): Promise { + return withDb(async (client) => { + const cur = await client.query<{ id: string }>( + `UPDATE pipeline_jobs + SET status = 'error', + error_text = $2, + exit_code = -1, + finished_at = now() + WHERE id = $1::uuid AND status = 'running' + RETURNING id::text`, + [id, message], + ); + return (cur.rowCount ?? 0) > 0; + }); +} + export async function finishPipelineJob( id: string, status: 'success' | 'error', diff --git a/web/src/strings.json b/web/src/strings.json index 379823ae..3de47c8d 100644 --- a/web/src/strings.json +++ b/web/src/strings.json @@ -159,6 +159,17 @@ "wizardEdit": "Edit", "wizardReviewTitle": "Ready to run", "wizardReviewHint": "Confirm your choices, then start the audit.", + "runPreview": { + "title": "Run preview", + "hint": "Estimated scope from your audit type and crawl settings.", + "estimatedTime": "Est. duration", + "maxPagesLabel": "Max pages", + "lighthousePagesLabel": "Lighthouse URLs", + "stepsLabel": "Pipeline steps", + "whatRunsLabel": "What will run", + "configPreviewLabel": "Configuration preview", + "estimateDisclaimer": "Duration is an estimate — actual time depends on site speed, server response, and network conditions." + }, "wizardAdvancedSettings": "Advanced settings", "wizardUrlHint": "Enter the site you want to analyze.", "wizardWorkflowHint": "Choose what to include: full site audit, crawl only, Lighthouse, Google data, or keywords.", @@ -170,6 +181,10 @@ "sidebarSettingsSection": "Advanced settings", "backToReports": "Back to audits", "continueInBackground": "Continue in background", + "stopJob": "Stop", + "stopJobAria": "Stop running audit", + "stoppingJob": "Stopping…", + "stopJobFailed": "Could not stop the audit: {message}", "outputLabel": "Audit run log", "outputTitle": "Audit run log", "setupStepsAria": "Run audit setup steps", @@ -181,6 +196,7 @@ "loadingSettings": "Loading settings…", "settingsTitle": "Audit settings", "settingsSubtitle": "Fine-tune crawl, audit report, Lighthouse, keywords, and AI options.", + "settingsSaveWhileRunningHint": "A job is running; saved settings apply to the next run.", "browserCrawlBannerTitle": "Headless browser not available", "browserCrawlBannerHint": "JavaScript and Auto crawl modes need Playwright Python packages and Chromium. Run: pip install -r requirements-browser.txt. Ensure Chrome or Chromium is on PATH or set CHROME_PATH.", "browserCrawlChecking": "Checking browser availability…", @@ -2156,6 +2172,7 @@ "noSessions": "No chats yet", "deleteSession": "Delete chat", "propertyLabel": "Property", + "selectProperty": "Select property…", "noProperties": "No properties", "aiDisabledTitle": "AI is not enabled", "aiDisabledHint": "Enable AI insights and configure a provider in Run audit → AI settings.", diff --git a/web/src/types/api.ts b/web/src/types/api.ts index f760554e..600cff3f 100644 --- a/web/src/types/api.ts +++ b/web/src/types/api.ts @@ -25,8 +25,14 @@ export interface PipelineJob { error?: string; } +/** In-memory job entry (server only). */ +export interface PipelineJobEntry extends PipelineJob { + cancelled?: boolean; + finished?: boolean; +} + export interface PipelineJobStore { - jobs: Map; + jobs: Map; running: boolean; } @@ -200,4 +206,5 @@ export interface AuditSqlExample { declare global { var __websiteProfilingPipelineJobs: PipelineJobStore | undefined; + var __websiteProfilingPipelineProcesses: Map | undefined; } diff --git a/web/src/views/Chat.tsx b/web/src/views/Chat.tsx index 6e9dbf0f..81a52450 100644 --- a/web/src/views/Chat.tsx +++ b/web/src/views/Chat.tsx @@ -25,6 +25,11 @@ import { readStoredChatContext, writeStoredChatContext, } from '@/lib/chatUrlState'; +import { + normalizePropertyId, + pickInitialPropertyId, + propertyIdsEqual, +} from '@/lib/googlePropertySelection'; const c = strings.components.chat; @@ -91,7 +96,7 @@ export default function ChatPage() { const showConversation = Boolean(sessionId) || messages.length > 0 || busy || loadingMessages; const isHero = !showConversation; - const activeProperty = properties.find((p) => p.id === propertyId) ?? null; + const activeProperty = properties.find((p) => propertyIdsEqual(p.id, propertyId)) ?? null; const activeSession = sessions.find((s) => s.id === sessionId) ?? null; const loadProperties = useCallback(async () => { @@ -100,7 +105,10 @@ export default function ChatPage() { const res = await fetch(apiUrl('/properties')); if (!res.ok) return; const data = (await res.json()) as { properties?: PropertyOption[] }; - const rows = data.properties || []; + const rows = (data.properties || []).map((p) => ({ + ...p, + id: normalizePropertyId(p.id) ?? p.id, + })); setProperties(rows); const urlCtx = parseChatUrlContext( new URLSearchParams( @@ -108,23 +116,22 @@ export default function ChatPage() { ), ); const stored = readStoredChatContext(); - const activeRaw = configState.active_property_id; - const activeId = activeRaw ? Number(activeRaw) : null; - const preferred = - urlCtx.propertyId ?? - stored.propertyId ?? - (activeId && rows.some((p) => p.id === activeId) ? activeId : null) ?? - rows[0]?.id ?? - null; - if (preferred && rows.some((p) => p.id === preferred)) { - setPropertyId(preferred); - } + const explicitId = urlCtx.propertyId ?? stored.propertyId ?? null; + const nextId = pickInitialPropertyId(rows, { + explicitId, + startUrl: String(configState.start_url || ''), + activePropertyId: String(configState.active_property_id || ''), + }); + setPropertyId((current) => { + if (nextId != null) return nextId; + return current != null ? null : current; + }); } catch { /* ignore */ } finally { setLoadingProperties(false); } - }, [configState.active_property_id]); + }, [configState.active_property_id, configState.start_url]); const resolveSessionFromUrl = useCallback(async (sid: number, pid: number | null) => { try { diff --git a/web/src/views/Pipeline.tsx b/web/src/views/Pipeline.tsx index 578949e3..a86331b0 100644 --- a/web/src/views/Pipeline.tsx +++ b/web/src/views/Pipeline.tsx @@ -11,9 +11,10 @@ import PipelineShell, { pipelineNavFromSearchParams, type PipelineNavId, } from '@/components/pipeline/PipelineShell'; -import { PipelineStatusBadge } from '@/components/pipeline/pipelineUi'; +import { PipelineStatusBadge, PipelineStopButton } from '@/components/pipeline/pipelineUi'; import { isPipelinePresetId } from '@/components/pipeline/pipelinePresets'; import { usePipeline } from '@/context/PipelineContext'; +import { useReadOnlySession } from '@/hooks/useReadOnlySession'; import { OPEN_INTEGRATIONS } from '@/lib/pipelineJobEvents'; export default function PipelinePage() { @@ -21,7 +22,8 @@ export default function PipelinePage() { const pathname = usePathname(); const searchParams = useSearchParams(); const activeNav = pipelineNavFromSearchParams(searchParams); - const { busy, status, handlePresetChange } = usePipeline(); + const { busy, status, stopping, handlePresetChange, cancelJob } = usePipeline(); + const { readOnly } = useReadOnlySession(); const [googleIntegrationsToast, setGoogleIntegrationsToast] = useState( null, ); @@ -80,8 +82,18 @@ export default function PipelinePage() { }; const headerExtra = - activeNav === 'run' && (busy || status) ? ( - + busy || status ? ( +
+ + {busy ? ( + + ) : null} +
) : null; return (