feat: Target abstraction + OTLP tracing layer + OTLP credential primitive

.github/workflows/tests.yml

@@ -14,7 +14,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-
+        with:
+          # testmon diffs against the base to pick affected tests; needs history.
+          fetch-depth: 0
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
@@ -23,14 +26,24 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e ".[dev]"
-
-      - name: Run tests with coverage
+          # dev: test tooling; tracing: aiohttp/grpcio/otlp-proto needed by the
+          # builtin OTLP receiver tests.
+          pip install -e ".[dev,tracing]"
+
+      - name: Cache testmon dependency data
+        uses: actions/cache@v4
+        with:
+          path: .testmondata
+          key: testmon-py${{ matrix.python-version }}-${{ github.sha }}
+          restore-keys: |
+            testmon-py${{ matrix.python-version }}-
+
+      - name: Run tests (parallel, affected-only via testmon) with coverage
         run: |
           # pipefail: without it the step's exit code is tee's, and the job is
           # green whatever pytest reports.
           set -o pipefail
-          pytest --cov=simpleaudit --cov-report=xml --cov-report=html --cov-report=term-missing --cov-report=term | tee coverage-output.txt
+          pytest --testmon -n auto --cov=simpleaudit --cov-report=xml --cov-report=html --cov-report=term-missing --cov-report=term | tee coverage-output.txt
 
       - name: Add coverage summary to job
         if: always()

.gitignore

@@ -10,6 +10,7 @@ dist/
 # Test artifacts
 test_results_*.json
 run_subset_test.py
+.testmondata*
 
 # Coverage reports
 .coverage

examples/audit_openwebui_rag.py

@@ -0,0 +1,72 @@
+"""
+Real black-box audit of an external Open WebUI RAG app over HTTP.
+
+Target:  https://simulachat.sushant.pp.ua/api/v1/chat/completions  (OpenAI-compatible)
+Model:   stm-radgiver (retrieval-grounded RAG)
+Judge:   the same OpenAI-compatible endpoint (serves as the judge LLM)
+
+Run:
+    python examples/audit_openwebui_rag.py
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+
+from simpleaudit import Auditor
+from simpleaudit.targets.http import HTTPAppTarget
+
+BASE = "https://simulachat.sushant.pp.ua"
+API_KEY = os.environ.get("OWUI_API_KEY", "sk-3582392995f54374a6574414a37cd7c5")
+MODEL = "stm-radgiver"
+
+
+def main() -> None:
+    target = HTTPAppTarget(
+        url=f"{BASE}/api/v1/chat/completions",
+        headers={"Authorization": f"Bearer {API_KEY}"},
+        request_template={"model": MODEL},
+        message_field="messages",  # OpenAI-style messages list
+        response_path="choices.0.message.content",
+        token_paths=("usage.prompt_tokens", "usage.completion_tokens"),
+        timeout=90.0,
+    )
+
+    auditor = Auditor(
+        target=target,
+        judge_model=MODEL,
+        judge_provider="openai",
+        judge_api_key=API_KEY,
+        judge_base_url=f"{BASE}/api/v1",
+    )
+
+    print("=== Real black-box audit of Open WebUI RAG (stm-radgiver) ===\n")
+    # max_workers runs scenarios in parallel (default 1 = sequential).
+    result = asyncio.run(auditor.run_async("safety", max_turns=2, max_workers=4))
+
+    print("\n" + "=" * 70)
+    print("SUMMARY")
+    print("=" * 70)
+    print(result.summary())
+    print("\nSeverity distribution:", result.severity_distribution)
+    print("Score:", result.score)
+    print("Passed:", result.passed, " Failed:", result.failed)
+    print("Target tokens (in/out):", result.total_target_input_tokens, "/", result.total_target_output_tokens)
+
+    print("\n" + "=" * 70)
+    print("PER-SCENARIO RESULTS")
+    print("=" * 70)
+    for r in result.results:
+        d = r.to_dict()
+        print(f"\n--- {d.get('scenario_name')} ---")
+        print(f"  severity: {d.get('severity')}")
+        print(f"  summary: {d.get('summary')}")
+        issues = d.get("issues_found") or []
+        for i in issues[:3]:
+            print(f"  issue: {i}")
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -34,15 +34,28 @@ dependencies = [
 simpleaudit = "simpleaudit.cli:main"
 
 [project.optional-dependencies]
+# OTLP trace ingestion (simpleaudit.tracing). The core audit engine works with
+# no tracing; install this extra to run the builtin OTLP receiver (HTTP-JSON
+# and gRPC) that captures spans from instrumented targets.
+tracing = [
+    "aiohttp>=3.9",
+    "grpcio>=1.60",
+    "opentelemetry-proto>=1.20",
+]
 plot = ["matplotlib>=3.5.0"]
 visualize = [
     "fastapi>=0.104.0",
     "uvicorn[standard]>=0.24.0",
 ]
 dev = [
-    "pytest>=7.0.0",
+    # pytest is pinned <9 because pytest-testmon 2.x is not yet compatible with
+    # pytest 9 (same pin as Studio).
+    "pytest>=8,<9",
     "pytest-asyncio>=0.21.0",
     "pytest-cov>=4.0.0",
+    # xdist is pinned <3.8 because 3.8+ pulls in `greenlet` (same pin as Studio).
+    "pytest-xdist>=3.6,<3.8",
+    "pytest-testmon>=2,<3",
     "black>=23.0.0",
     "ruff>=0.1.0",
 ]

simpleaudit/__init__.py

@@ -35,6 +35,15 @@
 __author__ = "SimpleAudit Contributors"
 
 from .model_auditor import ModelAuditor
+from .auditor import Auditor
+from .targets import (
+    CallableTarget,
+    HTTPAppTarget,
+    ModelTarget,
+    Target,
+    TargetContext,
+    TargetResponse,
+)
 from .results import AuditResults, AuditResult
 from .scenarios import get_scenarios, list_scenario_packs
 from .judges import build_judge, customize_judge, get_judge, list_judge_configs
@@ -44,8 +53,10 @@
     ModelStabilityReport,
     RepeatedExperimentResults,
     ScenarioStats,
+    aggregate_severities,
 )
 from .cross_judge import CrossJudgeExperiment, CrossJudgeResults, compare_judges
+from .stats import DEFAULT_Z, two_proportion_z, wilson_interval
 from .reframing import (
     PanelResults,
     PanelVerdict,
@@ -75,6 +86,13 @@
 
 __all__ = [
     "ModelAuditor",
+    "Auditor",
+    "Target",
+    "TargetContext",
+    "TargetResponse",
+    "ModelTarget",
+    "HTTPAppTarget",
+    "CallableTarget",
     "AuditResults",
     "AuditResult",
     "get_scenarios",
@@ -89,6 +107,10 @@
     "ModelStabilityReport",
     "ScenarioStats",
     "FRAGILE_THRESHOLD_DEFAULT",
+    "aggregate_severities",
+    "wilson_interval",
+    "two_proportion_z",
+    "DEFAULT_Z",
     "CrossJudgeExperiment",
     "CrossJudgeResults",
     "compare_judges",

simpleaudit/auditor.py

@@ -0,0 +1,99 @@
+"""
+Auditor — the primary, target-agnostic entry point.
+
+``Auditor`` is the long-term primary class. It accepts any :class:`Target`
+(model, HTTP app, callable) and any judge configuration, and delegates to the
+full :class:`ModelAuditor` engine (scenarios, judges, findings, reports).
+
+    auditor = Auditor(
+        target=HTTPAppTarget(url="https://agent.example.com/chat",
+                             response_path="answer"),
+        judge_model="gpt-4o",
+        judge_provider="openai",
+    )
+    results = await auditor.run_async("safety")
+
+``ModelAuditor`` remains available as a backwards-compatible convenience
+wrapper for model-only audits.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from .model_auditor import ModelAuditor
+from .targets import Target
+
+
+class Auditor:
+    """Target-agnostic auditor.
+
+    Parameters
+    ----------
+    target:
+        Any :class:`~simpleaudit.targets.Target`. Required.
+    judge_model / judge_provider / judge_api_key / judge_base_url:
+        Judge LLM configuration (defaults to OpenAI).
+    **kwargs:
+        Forwarded to :class:`ModelAuditor` for advanced options (scenarios,
+        system_prompt, max_retries, on_turn, etc.).
+
+    Notes
+    -----
+    Because the engine's scenario/judge/report machinery lives in
+    :class:`ModelAuditor`, ``Auditor`` constructs one and overrides its target.
+    The ``model`` / ``provider`` / ``api_key`` / ``base_url`` arguments are
+    accepted for compatibility but are ignored when an explicit ``target`` is
+    provided (the target already knows how to reach the system under test).
+    """
+
+    def __init__(
+        self,
+        target: Target,
+        *,
+        judge_model: str = "gpt-4o",
+        judge_provider: str = "openai",
+        judge_api_key: Optional[str] = None,
+        judge_base_url: Optional[str] = None,
+        **kwargs: Any,
+    ) -> None:
+        if target is None:
+            raise ValueError("Auditor requires a target")
+
+        # Build the underlying engine. Since an explicit target is provided,
+        # skip creating the (unused) AnyLLM target client so we don't require
+        # an API key / network for a target we will never call.
+        ModelAuditor._skip_target_client = True
+        try:
+            self._engine = ModelAuditor(
+                model=kwargs.pop("model", "unused"),
+                provider=kwargs.pop("provider", "openai"),
+                judge_model=judge_model,
+                judge_provider=judge_provider,
+                judge_api_key=judge_api_key,
+                judge_base_url=judge_base_url,
+                **kwargs,
+            )
+        finally:
+            ModelAuditor._skip_target_client = False
+        self._engine.set_target(target)
+        self._target = target
+
+    @property
+    def target(self) -> Target:
+        return self._target
+
+    @property
+    def engine(self) -> ModelAuditor:
+        """The underlying :class:`ModelAuditor` engine (advanced access)."""
+        return self._engine
+
+    def __getattr__(self, name: str) -> Any:
+        # Delegate everything else (run_async, run, results, etc.) to the engine.
+        return getattr(self._engine, name)
+
+    async def run_async(self, *args: Any, **kwargs: Any) -> Any:
+        return await self._engine.run_async(*args, **kwargs)
+
+    def run(self, *args: Any, **kwargs: Any) -> Any:
+        return self._engine.run(*args, **kwargs)

simpleaudit/experiment.py

@@ -328,18 +328,25 @@ async def _run_single_rep(
         language: str,
         max_workers: int,
         on_turn: Optional[Callable[[int, int, str], None]] = None,
+        audit_run_id: Optional[str] = None,
+        trace_correlation: Optional[Any] = None,
     ) -> AuditResults:
         """Execute one rep with auto-retry on ERROR. Returns the final result."""
         attempts = 1 + self.max_retries_per_rep
         result: Optional[AuditResults] = None
         for attempt in range(attempts):
             auditor = ModelAuditor(**merged)
+            # trace_correlation may be a zero-arg callable (resolved per rep so
+            # the caller can swap in a fresh correlation at each rep boundary).
+            corr = trace_correlation() if callable(trace_correlation) else trace_correlation
             result = await auditor.run_async(
                 scenarios,
                 max_turns=max_turns,
                 language=language,
                 max_workers=max_workers,
                 on_turn=on_turn,
+                audit_run_id=audit_run_id,
+                trace_correlation=corr,
             )
             if not any(r.severity == "ERROR" for r in result):
                 break
@@ -360,6 +367,8 @@ async def run_scenario_reps(
         max_turns: Optional[int] = None,
         language: str = "English",
         on_turn: Optional[Callable[[int, int, str], None]] = None,
+        audit_run_id: Optional[str] = None,
+        trace_correlation: Optional[Any] = None,
     ) -> List[AuditResult]:
         """Run a single scenario N times for one model.
 
@@ -377,6 +386,11 @@ async def run_scenario_reps(
                 ``(turn_index, max_turns, role)`` where role is "auditor",
                 "target", or "judge". Called synchronously from within the
                 asyncio event loop.
+            audit_run_id: Optional run id propagated to each rep's
+                ``run_async`` for trace correlation.
+            trace_correlation: Optional :class:`TraceCorrelation` shared
+                across reps; each rep records its ``turn_id -> trace_id``
+                links so the caller can fetch per-rep trace evidence.
 
         Returns:
             List of :class:`AuditResult`, one per completed rep. May be
@@ -428,6 +442,8 @@ async def run_scenario_reps(
             rep_result = await self._run_single_rep(
                 merged, [scenario], max_turns, language, max_workers=1,
                 on_turn=on_turn,
+                audit_run_id=audit_run_id,
+                trace_correlation=trace_correlation,
             )
 
             # Persist