diff --git a/backend/app/main.py b/backend/app/main.py index e92fad8..3c1dd98 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,50 +1,90 @@ from __future__ import annotations -from datetime import datetime, timezone -from uuid import uuid4 import asyncio -import traceback +from datetime import datetime, timezone from pathlib import Path +from uuid import uuid4 -from fastapi import FastAPI, HTTPException, Request, Depends +from fastapi import Depends, FastAPI, HTTPException, Request from fastapi.responses import FileResponse, JSONResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel, Field -from app.validators import normalize_domain -from app.services.dns_audit import audit_dns, audit_mail -from app.services.tls_audit import audit_tls -from app.services.web_audit import audit_web_targets -from app.services.subdomain_audit import discover_subdomains_ct -from app.services.domain_profile import classify_domain, adjust_findings_for_profile -from app.services.scoring import collect_findings, compute_score -from app.services.rate_limit import check_rate_limit -from app.services.ip_inventory import build_ip_inventory -from app.services.tls_scoring import score_tls -from app.services.cve_passive import detect_passive_cves -from app.services.cti_audit import audit_cti -from app.services.patching_sla import build_patching_sla -from app.services.finding_location import enrich_findings_locations +from app.database import get_db, init_db_with_retry from app.reports.excel_report import generate_excel_report +from app.reports.html_report import generate_html_report from app.reports.json_report import generate_json_report from app.reports.pdf_report import generate_pdf_report -from app.reports.html_report import generate_html_report -from app.database import init_db_with_retry, get_db -from app.services.storage import save_audit, list_audits, get_audit_record, dashboard_stats, compare_latest, delete_audit, delete_all_audits, delete_domain_audits -from app.services.domain_verification import start_verification, check_verification, get_domain_status, list_verified_domains, delete_verification, serialize_verification +from app.services.attack_graph import build_attack_graph +from app.services.cti_audit import audit_cti +from app.services.cve_passive import detect_passive_cves from app.services.diagnostics import build_system_diagnostics, test_export_dependencies +from app.services.dns_audit import audit_dns, audit_mail +from app.services.domain_profile import adjust_findings_for_profile, classify_domain +from app.services.domain_verification import ( + check_verification, + delete_verification, + get_domain_status, + list_verified_domains, + serialize_verification, + start_verification, +) from app.services.executive_risk import build_executive_risk +from app.services.finding_location import enrich_findings_locations +from app.services.host_enrichment import attach_services_to_hosts, enrich_public_hosts +from app.services.ip_inventory import build_ip_inventory +from app.services.legal_terms import create_acceptance, legal_payload, validate_acceptance from app.services.nmap_audit import audit_service_versions -from app.services.legal_terms import legal_payload, create_acceptance, validate_acceptance -from app.services.attack_graph import build_attack_graph +from app.services.patching_sla import build_patching_sla +from app.services.rate_limit import check_rate_limit +from app.services.scoring import collect_findings, compute_score +from app.services.storage import ( + compare_latest, + dashboard_stats, + delete_all_audits, + delete_audit, + delete_domain_audits, + get_audit_record, + list_audits, + save_audit, +) +from app.services.subdomain_audit import discover_subdomains_ct +from app.services.tls_audit import audit_tls +from app.services.tls_scoring import score_tls +from app.services.web_audit import audit_web_targets +from app.validators import normalize_domain + +APP_VERSION = "OpenEASM Beta 26.6" +REPORT_DIR = Path("/app/reports") +AUDITS: dict[str, dict] = {} app = FastAPI( - title="OpenEASM Beta", - description="OpenEASM Beta : EASM défensif avec avertissement juridique bloquant, Graph Explorer, Nmap service/version/CVE non exploitant et rapports professionnels.", - version="beta-1.0", + title=APP_VERSION, + description=( + "OpenEASM Beta 26.6 : EASM défensif avec enrichissement DNSDumpster-like, " + "ASN/hébergeur/localisation, scan service/version sur sous-domaines publics, " + "Graph Explorer et rapports professionnels." + ), + version="26.6", ) +class AuditRequest(BaseModel): + domain: str = Field(..., description="Nom de domaine à auditer, exemple : example.com") + accepted_terms: bool = Field(False, description="Confirme le cadre autorisé et défensif.") + terms_token: str | None = Field(None, description="Jeton d'acceptation juridique délivré par /api/legal/accept-terms.") + + +class LegalAcceptRequest(BaseModel): + accepted: bool = Field(False, description="Confirme que l'utilisateur a lu et accepté les conditions.") + terms_hash: str | None = Field(None, description="Hash SHA-256 du texte juridique affiché.") + terms_version: str | None = Field(None, description="Version du règlement affiché.") + + +class DomainVerificationRequest(BaseModel): + domain: str = Field(..., description="Nom de domaine à vérifier, exemple : example.com") + + @app.on_event("startup") def startup_event(): init_db_with_retry() @@ -61,30 +101,17 @@ async def openeasm_unhandled_exception_handler(request: Request, exc: Exception) }, ) -AUDITS: dict[str, dict] = {} -REPORT_DIR = Path("/app/reports") - -class AuditRequest(BaseModel): - domain: str = Field(..., description="Nom de domaine à auditer, exemple : example.com") - accepted_terms: bool = Field(False, description="L'utilisateur confirme être autorisé ou rester dans le cadre défensif autorisé.") - terms_token: str | None = Field(None, description="Jeton d'acceptation juridique délivré par /api/legal/accept-terms.") - -class LegalAcceptRequest(BaseModel): - accepted: bool = Field(False, description="Confirme que l'utilisateur a lu et accepté les conditions.") - terms_hash: str | None = Field(None, description="Hash SHA-256 du texte juridique affiché.") - terms_version: str | None = Field(None, description="Version du règlement affiché.") - -class DomainVerificationRequest(BaseModel): - domain: str = Field(..., description="Nom de domaine à vérifier, exemple : example.com") @app.get("/api/health") async def health(): - return {"status": "ok", "service": "openeasm-beta", "version": "beta-1.0"} + return {"status": "ok", "service": "openeasm-beta", "version": "26.6", "edition": APP_VERSION} + @app.get("/api/legal/terms") async def api_legal_terms(): return legal_payload() + @app.post("/api/legal/accept-terms") async def api_accept_terms(payload: LegalAcceptRequest, request: Request, db=Depends(get_db)): client_host = request.client.host if request.client else "unknown" @@ -100,32 +127,25 @@ async def api_accept_terms(payload: LegalAcceptRequest, request: Request, db=Dep except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) + @app.get("/api/legal/status") async def api_legal_status(token: str | None = None, db=Depends(get_db)): return validate_acceptance(db, token) + @app.post("/api/audit") async def create_audit(payload: AuditRequest, request: Request, db=Depends(get_db)): if not payload.accepted_terms: - raise HTTPException( - status_code=400, - detail="Vous devez accepter l'usage responsable avant de lancer l'audit.", - ) + raise HTTPException(status_code=400, detail="Vous devez accepter l'usage responsable avant de lancer l'audit.") legal_status = validate_acceptance(db, payload.terms_token) if not legal_status.get("accepted"): - raise HTTPException( - status_code=403, - detail="Acceptation juridique obligatoire avant d'utiliser OpenEASM.", - ) + raise HTTPException(status_code=403, detail="Acceptation juridique obligatoire avant d'utiliser OpenEASM.") client_host = request.client.host if request.client else "unknown" rate = check_rate_limit(client_host) if not rate["allowed"]: - raise HTTPException( - status_code=429, - detail=f"Trop d'audits lancés. Réessayez dans {rate['retry_after']} secondes.", - ) + raise HTTPException(status_code=429, detail=f"Trop d'audits lancés. Réessayez dans {rate['retry_after']} secondes.") try: domain = normalize_domain(payload.domain) @@ -138,23 +158,22 @@ async def create_audit(payload: AuditRequest, request: Request, db=Depends(get_d dns_result = audit_dns(domain) mail_result = audit_mail(domain) web_result = await audit_web_targets(domain) + subdomains_result = await discover_subdomains_ct(domain) + ip_inventory = build_ip_inventory(domain, dns_result, mail_result, web_result, subdomains_result) - tls_targets = [] - for target in [domain, f"www.{domain}"]: - tls_targets.append(audit_tls(target)) - + tls_targets = [audit_tls(target) for target in [domain, f"www.{domain}"]] tls_result = { "domain": domain, "targets": tls_targets, "findings": [f for t in tls_targets for f in t.get("findings", [])], } - - subdomains_result = await discover_subdomains_ct(domain) - ip_inventory = build_ip_inventory(domain, dns_result, mail_result, web_result, subdomains_result) tls_score = score_tls(tls_result, web_result) passive_cves = detect_passive_cves(web_result) cti_result = audit_cti(ip_inventory, domain) + service_scan = await asyncio.to_thread(audit_service_versions, domain, ip_inventory) + host_enrichment = await enrich_public_hosts(domain, subdomains_result, ip_inventory) + host_enrichment = attach_services_to_hosts(host_enrichment, service_scan) raw_findings = collect_findings( dns_result, @@ -174,14 +193,15 @@ async def create_audit(payload: AuditRequest, request: Request, db=Depends(get_d legacy_score = compute_score(findings, domain_profile) patching_sla = build_patching_sla(findings, created_at) executive_risk = build_executive_risk(findings, domain_profile, tls_score, ip_inventory, subdomains_result, passive_cves, cti_result) - score = executive_risk.get('global_score', legacy_score) + score = executive_risk.get("global_score", legacy_score) audit_id = str(uuid4()) audit = { "id": audit_id, + "product": APP_VERSION, "domain": domain, "created_at": created_at, - "mode": "public_defensive_beta_service_version_cve", + "mode": "public_defensive_beta_26_6_dnsdumpster_like", "verification": verification_status, "domain_profile": domain_profile, "dns": dns_result, @@ -191,6 +211,7 @@ async def create_audit(payload: AuditRequest, request: Request, db=Depends(get_d "web": web_result, "subdomains": subdomains_result, "ip_inventory": ip_inventory, + "host_enrichment": host_enrichment, "passive_cves": passive_cves, "service_scan": service_scan, "cti": cti_result, @@ -204,43 +225,30 @@ async def create_audit(payload: AuditRequest, request: Request, db=Depends(get_d "legal_acceptance": legal_status, "client": client_host, "rate_limit": rate, - "anti_ssrf": "HTTP/TLS only if target resolves to public IPs and no blocked IP.", - "active_scan": "enabled_light_service_version_only", + "anti_ssrf": "HTTP/TLS/Nmap only if target resolves to public IPs and no blocked IP.", + "active_scan": "enabled_light_service_version_all_public_subdomains", "nmap": service_scan.get("mode"), "cve_scan": "passive_headers_and_nmap_service_version_correlation", "leak_search": "disabled_public_mode", }, "report_filename": None, "json_filename": None, + "pdf_filename": None, "html_filename": None, } audit["attack_graph"] = build_attack_graph(audit) - audit["report_errors"] = [] - try: - report_filename = generate_excel_report(audit) - audit["report_filename"] = report_filename - except Exception as exc: - audit["report_errors"].append(f"Excel: {exc}") - - try: - json_filename = generate_json_report(audit) - audit["json_filename"] = json_filename - except Exception as exc: - audit["report_errors"].append(f"JSON: {exc}") - - try: - pdf_filename = generate_pdf_report(audit) - audit["pdf_filename"] = pdf_filename - except Exception as exc: - audit["report_errors"].append(f"PDF: {exc}") - - try: - html_filename = generate_html_report(audit) - audit["html_filename"] = html_filename - except Exception as exc: - audit["report_errors"].append(f"HTML: {exc}") + for label, key, generator in [ + ("Excel", "report_filename", generate_excel_report), + ("JSON", "json_filename", generate_json_report), + ("PDF", "pdf_filename", generate_pdf_report), + ("HTML", "html_filename", generate_html_report), + ]: + try: + audit[key] = generator(audit) + except Exception as exc: + audit["report_errors"].append(f"{label}: {exc}") AUDITS[audit_id] = audit save_audit(db, audit) @@ -248,32 +256,33 @@ async def create_audit(payload: AuditRequest, request: Request, db=Depends(get_d return { "id": audit_id, "domain": domain, + "version": "26.6", "mode": audit["mode"], "verification": verification_status, "created_at": audit["created_at"], "domain_profile": domain_profile, "score": audit["score"], "executive_risk": executive_risk, - "tls_score": tls_score, "findings": audit["findings"], "subdomains": { "source": subdomains_result.get("source"), "count": subdomains_result.get("count"), - "subdomains": subdomains_result.get("subdomains", [])[:80], + "subdomains": subdomains_result.get("subdomains", [])[:120], "error": subdomains_result.get("error"), }, "ip_inventory": { "public_ip_count": ip_inventory.get("public_ip_count"), "total_ip_count": ip_inventory.get("total_ip_count"), - "unique_ips": ip_inventory.get("display_ips", ip_inventory.get("unique_ips", []))[:100], + "unique_ips": ip_inventory.get("display_ips", ip_inventory.get("unique_ips", []))[:120], + "location_counts": ip_inventory.get("location_counts", {}), + "hosting_networks": ip_inventory.get("hosting_networks", {}), "core_public_ip_count": ip_inventory.get("core_public_ip_count", 0), "third_party_provider_ip_count": ip_inventory.get("third_party_provider_ip_count", 0), - "total_ip_count": ip_inventory.get("total_ip_count", 0), }, - "passive_cves": { - "count": passive_cves.get("count"), - "items": passive_cves.get("items", [])[:30], - "note": passive_cves.get("note"), + "host_enrichment": { + "summary": host_enrichment.get("summary", {}), + "hosts": host_enrichment.get("hosts", [])[:120], + "note": host_enrichment.get("summary", {}).get("note"), }, "service_scan": { "enabled": service_scan.get("enabled"), @@ -281,65 +290,33 @@ async def create_audit(payload: AuditRequest, request: Request, db=Depends(get_d "count_open_ports": service_scan.get("count_open_ports", 0), "count_cves": service_scan.get("count_cves", 0), "elapsed_seconds": service_scan.get("elapsed_seconds", 0), - "targets": service_scan.get("targets", [])[:10], - "open_ports": service_scan.get("open_ports", [])[:80], - "cves": service_scan.get("cves", [])[:50], + "targets": service_scan.get("targets", [])[:120], + "open_ports": service_scan.get("open_ports", [])[:200], + "cves": service_scan.get("cves", [])[:100], "note": service_scan.get("note"), }, - "cti": { - "summary": cti_result.get("summary", {}), - "ip_reputation": cti_result.get("ip_reputation", [])[:30], - "leak_monitoring": cti_result.get("leak_monitoring"), - "note": cti_result.get("note"), - }, - "patching_sla": { - "sla_policy": patching_sla.get("sla_policy"), - "items": patching_sla.get("items", [])[:30], - "note": patching_sla.get("note"), - }, - "web_targets": [ - { - "hostname": t.get("hostname"), - "reachable": t.get("reachable"), - "best_scheme": t.get("best_scheme"), - "public_ips": (t.get("guard") or {}).get("public_ips", []), - "blocked_ips": (t.get("guard") or {}).get("blocked_ips", []), - "http_status": (t.get("http") or {}).get("status_code"), - "https_status": (t.get("https") or {}).get("status_code"), - "https_final_url": (t.get("https") or {}).get("final_url"), - } - for t in web_result.get("targets", []) - ], "report_url": f"/api/reports/{audit_id}/excel", "json_url": f"/api/reports/{audit_id}/json", "pdf_url": f"/api/reports/{audit_id}/pdf", - "html_filename": audit.get("html_filename"), "html_url": f"/api/reports/{audit_id}/html", "graph_url": f"/api/audits/{audit_id}/graph", "attack_graph": audit.get("attack_graph", {}), "report_errors": audit.get("report_errors", []), "summary": { - "dns_public_ips": audit["dns"].get("public_ips", []), - "spf_records": audit["dns"].get("spf_records", []), - "mx_records": audit["mail"]["mx"].get("values", []), - "dmarc_records": audit["mail"].get("dmarc_records", []), - "has_web": web_result.get("has_web"), - "reachable_web_targets": domain_profile.get("reachable_web_targets", []), "subdomain_count": subdomains_result.get("count", 0), "public_ip_count": ip_inventory.get("public_ip_count", 0), - "core_public_ip_count": ip_inventory.get("core_public_ip_count", 0), - "third_party_provider_ip_count": ip_inventory.get("third_party_provider_ip_count", 0), - "passive_cve_count": passive_cves.get("count", 0), + "host_enrichment_active": host_enrichment.get("summary", {}).get("active_host_count", 0), "service_open_port_count": service_scan.get("count_open_ports", 0), "service_cve_count": service_scan.get("count_cves", 0), "service_scan_elapsed_seconds": service_scan.get("elapsed_seconds", 0), - "tls_score": tls_score.get("global_score"), - "tls_level": tls_score.get("global_level"), - "executive_score": executive_risk.get("overall_score"), - "executive_risk": executive_risk.get("risk_level"), + "locations": host_enrichment.get("summary", {}).get("location_counts", {}), + "hosting_networks": host_enrichment.get("summary", {}).get("hosting_networks", {}), + "service_banners": host_enrichment.get("summary", {}).get("service_banners", {}), + "technologies": host_enrichment.get("summary", {}).get("technology_counts", {}), }, } + @app.get("/api/audits") async def api_list_audits(limit: int = 50, domain: str | None = None, db=Depends(get_db)): records = list_audits(db, limit=limit, domain=domain) @@ -359,6 +336,7 @@ async def api_list_audits(limit: int = 50, domain: str | None = None, db=Depends for r in records ] + @app.get("/api/audits/{audit_id}") async def get_audit(audit_id: str, db=Depends(get_db)): audit = AUDITS.get(audit_id) @@ -369,6 +347,7 @@ async def get_audit(audit_id: str, db=Depends(get_db)): raise HTTPException(status_code=404, detail="Audit introuvable.") return record.audit_json + @app.get("/api/dashboard") async def api_dashboard(db=Depends(get_db)): return dashboard_stats(db) @@ -379,12 +358,10 @@ async def api_audit_graph(audit_id: str, db=Depends(get_db)): audit = AUDITS.get(audit_id) if not audit: record = get_audit_record(db, audit_id) - if record: - audit = record.audit_json - else: + if not record: raise HTTPException(status_code=404, detail="Audit introuvable.") - graph = audit.get("attack_graph") or build_attack_graph(audit) - return graph + audit = record.audit_json + return audit.get("attack_graph") or build_attack_graph(audit) @app.get("/api/graph/latest") @@ -400,15 +377,18 @@ async def api_latest_graph(db=Depends(get_db)): async def api_system_diagnostics(db=Depends(get_db)): return build_system_diagnostics(db) + @app.post("/api/system/export-test") async def api_export_test(): return test_export_dependencies(cleanup=True) + @app.get("/api/reports") async def api_reports_center(limit: int = 100, domain: str | None = None, db=Depends(get_db)): records = list_audits(db, limit=limit, domain=domain) items = [] for r in records: + html_filename = r.html_filename or (r.audit_json or {}).get("html_filename") items.append({ "id": r.id, "domain": r.domain, @@ -419,20 +399,17 @@ async def api_reports_center(limit: int = 100, domain: str | None = None, db=Dep "excel_filename": r.excel_filename, "json_filename": r.json_filename, "pdf_filename": r.pdf_filename, - "html_filename": r.html_filename or (r.audit_json or {}).get("html_filename"), + "html_filename": html_filename, "excel_exists": bool(r.excel_filename and (REPORT_DIR / r.excel_filename).exists()), "json_exists": bool(r.json_filename and (REPORT_DIR / r.json_filename).exists()), "pdf_exists": bool(r.pdf_filename and (REPORT_DIR / r.pdf_filename).exists()), - "html_exists": bool((r.html_filename or (r.audit_json or {}).get("html_filename")) and (REPORT_DIR / (r.html_filename or (r.audit_json or {}).get("html_filename"))).exists()), + "html_exists": bool(html_filename and (REPORT_DIR / html_filename).exists()), "excel_url": f"/api/reports/{r.id}/excel", "json_url": f"/api/reports/{r.id}/json", "pdf_url": f"/api/reports/{r.id}/pdf", "html_url": f"/api/reports/{r.id}/html", }) - return { - "count": len(items), - "items": items, - } + return {"count": len(items), "items": items} @app.post("/api/domains/verification/start") @@ -441,8 +418,8 @@ async def api_start_domain_verification(payload: DomainVerificationRequest, db=D domain = normalize_domain(payload.domain) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) - record = start_verification(db, domain) - return serialize_verification(record) + return serialize_verification(start_verification(db, domain)) + @app.post("/api/domains/{domain}/verification/check") async def api_check_domain_verification(domain: str, db=Depends(get_db)): @@ -455,10 +432,12 @@ async def api_check_domain_verification(domain: str, db=Depends(get_db)): raise HTTPException(status_code=404, detail="Aucune vérification démarrée pour ce domaine.") return serialize_verification(record) + @app.get("/api/domains/verified") async def api_list_verified_domains(db=Depends(get_db)): return [serialize_verification(r) for r in list_verified_domains(db)] + @app.get("/api/domains/{domain}/verification") async def api_get_domain_verification(domain: str, db=Depends(get_db)): try: @@ -467,6 +446,7 @@ async def api_get_domain_verification(domain: str, db=Depends(get_db)): raise HTTPException(status_code=400, detail=str(exc)) return get_domain_status(db, normalized) + @app.delete("/api/domains/{domain}/verification") async def api_delete_domain_verification(domain: str, db=Depends(get_db)): try: @@ -494,12 +474,14 @@ async def api_delete_audit(audit_id: str, db=Depends(get_db)): AUDITS.pop(audit_id, None) return {"deleted": True, "audit_id": audit_id} + @app.delete("/api/audits") async def api_delete_all_audits(db=Depends(get_db)): count = delete_all_audits(db) AUDITS.clear() return {"deleted": count} + @app.delete("/api/domains/{domain}/audits") async def api_delete_domain_audits(domain: str, db=Depends(get_db)): try: @@ -513,66 +495,44 @@ async def api_delete_domain_audits(domain: str, db=Depends(get_db)): return {"domain": normalized, "deleted": count} -@app.get("/api/reports/{audit_id}/excel") -async def download_excel_report(audit_id: str, db=Depends(get_db)): +def _load_audit(audit_id: str, db): audit = AUDITS.get(audit_id) + record = None if not audit: record = get_audit_record(db, audit_id) - if record: - audit = record.audit_json - else: + if not record: raise HTTPException(status_code=404, detail="Audit introuvable.") + audit = record.audit_json + return audit, record + +@app.get("/api/reports/{audit_id}/excel") +async def download_excel_report(audit_id: str, db=Depends(get_db)): + audit, _ = _load_audit(audit_id, db) filename = audit.get("report_filename") if not filename: raise HTTPException(status_code=404, detail="Rapport introuvable.") - path = REPORT_DIR / filename if not path.exists(): raise HTTPException(status_code=404, detail="Fichier rapport introuvable.") + return FileResponse(path, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", filename=filename) - return FileResponse( - path, - media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - filename=filename, - ) @app.get("/api/reports/{audit_id}/json") async def download_json_report(audit_id: str, db=Depends(get_db)): - audit = AUDITS.get(audit_id) - if not audit: - record = get_audit_record(db, audit_id) - if record: - audit = record.audit_json - else: - raise HTTPException(status_code=404, detail="Audit introuvable.") - + audit, _ = _load_audit(audit_id, db) filename = audit.get("json_filename") if not filename: raise HTTPException(status_code=404, detail="Rapport JSON introuvable.") - path = REPORT_DIR / filename if not path.exists(): raise HTTPException(status_code=404, detail="Fichier JSON introuvable.") - - return FileResponse( - path, - media_type="application/json", - filename=filename, - ) + return FileResponse(path, media_type="application/json", filename=filename) @app.get("/api/reports/{audit_id}/html") async def download_html_report(audit_id: str, db=Depends(get_db)): - audit = AUDITS.get(audit_id) - record = None - if not audit: - record = get_audit_record(db, audit_id) - if record: - audit = record.audit_json - else: - raise HTTPException(status_code=404, detail="Audit introuvable.") - + audit, record = _load_audit(audit_id, db) filename = audit.get("html_filename") or (record.html_filename if record else None) path = REPORT_DIR / filename if filename else None if not filename or not path or not path.exists(): @@ -585,38 +545,21 @@ async def download_html_report(audit_id: str, db=Depends(get_db)): except Exception as exc: raise HTTPException(status_code=500, detail=f"Impossible de générer le rapport HTML: {exc}") path = REPORT_DIR / filename - if not path.exists(): raise HTTPException(status_code=404, detail="Fichier HTML introuvable.") + return FileResponse(path, media_type="text/html", filename=filename) - return FileResponse( - path, - media_type="text/html", - filename=filename, - ) @app.get("/api/reports/{audit_id}/pdf") async def download_pdf_report(audit_id: str, db=Depends(get_db)): - audit = AUDITS.get(audit_id) - if not audit: - record = get_audit_record(db, audit_id) - if record: - audit = record.audit_json - else: - raise HTTPException(status_code=404, detail="Audit introuvable.") - + audit, _ = _load_audit(audit_id, db) filename = audit.get("pdf_filename") if not filename: raise HTTPException(status_code=404, detail="Rapport PDF introuvable.") - path = REPORT_DIR / filename if not path.exists(): raise HTTPException(status_code=404, detail="Fichier PDF introuvable.") + return FileResponse(path, media_type="application/pdf", filename=filename) - return FileResponse( - path, - media_type="application/pdf", - filename=filename, - ) app.mount("/", StaticFiles(directory="/app/app/static", html=True), name="static") diff --git a/backend/app/services/attack_graph.py b/backend/app/services/attack_graph.py index 1292c99..ae765a4 100644 --- a/backend/app/services/attack_graph.py +++ b/backend/app/services/attack_graph.py @@ -3,18 +3,18 @@ from collections import Counter from typing import Any -MAX_SUBDOMAINS = 120 -MAX_IPS = 120 -MAX_FINDINGS = 60 -MAX_SERVICES = 100 +MAX_SUBDOMAINS = 180 +MAX_IPS = 180 +MAX_FINDINGS = 80 +MAX_SERVICES = 180 +MAX_HOSTS = 180 def build_attack_graph(audit: dict[str, Any]) -> dict[str, Any]: - """Build a defensive relationship graph for the Graph Explorer. + """Build a defensive relationship graph for Graph Explorer. - The graph is intentionally explanatory, not offensive: it links domains, - subdomains, public IPs, web targets, exposed services, CVE correlations and - prioritized findings already produced by OpenEASM. + Beta 26.6 adds DNSDumpster-like host enrichment nodes: ASN/provider, + locations, technologies, banners and TLS certificate summaries. """ nodes: dict[str, dict[str, Any]] = {} edges: dict[str, dict[str, Any]] = {} @@ -23,13 +23,7 @@ def add_node(node_id: str, label: str, node_type: str, **props: Any) -> str: if not node_id: return "" if node_id not in nodes: - nodes[node_id] = { - "id": node_id, - "label": label or node_id, - "type": node_type, - "weight": 1, - "properties": {}, - } + nodes[node_id] = {"id": node_id, "label": label or node_id, "type": node_type, "weight": 1, "properties": {}} else: nodes[node_id]["weight"] = int(nodes[node_id].get("weight", 1)) + 1 nodes[node_id]["properties"].update({k: v for k, v in props.items() if v not in (None, "", [], {})}) @@ -40,21 +34,13 @@ def add_edge(source: str, target: str, label: str, edge_type: str = "related", * return edge_id = f"{source}->{target}:{edge_type}:{label}" if edge_id not in edges: - edges[edge_id] = { - "id": edge_id, - "source": source, - "target": target, - "label": label, - "type": edge_type, - "weight": 1, - "properties": {}, - } + edges[edge_id] = {"id": edge_id, "source": source, "target": target, "label": label, "type": edge_type, "weight": 1, "properties": {}} else: edges[edge_id]["weight"] = int(edges[edge_id].get("weight", 1)) + 1 edges[edge_id]["properties"].update({k: v for k, v in props.items() if v not in (None, "", [], {})}) domain = audit.get("domain") or "domain" - root_id = add_node(f"domain:{domain}", domain, "domain", score=(audit.get("score") or {}).get("score")) + root_id = add_node(f"domain:{domain}", domain, "domain", score=(audit.get("score") or {}).get("score"), product=audit.get("product")) profile = audit.get("domain_profile") or {} if profile: @@ -82,14 +68,51 @@ def add_edge(source: str, target: str, label: str, edge_type: str = "related", * ip, "ip", scope=item.get("scope") if isinstance(item, dict) else None, + asn=item.get("asn") if isinstance(item, dict) else None, + asn_name=item.get("asn_name") if isinstance(item, dict) else None, + network=item.get("network") if isinstance(item, dict) else None, + country=item.get("country") if isinstance(item, dict) else None, + provider=item.get("provider") if isinstance(item, dict) else None, sources=item.get("sources") if isinstance(item, dict) else None, ) add_edge(root_id, ip_id, "IP inventoriée", "inventory") - for host in (item.get("hostnames", []) if isinstance(item, dict) else [])[:12]: + if isinstance(item, dict) and item.get("asn"): + asn_id = add_node(f"asn:{item.get('asn')}", item.get("asn_name") or item.get("asn"), "asn", network=item.get("network"), country=item.get("country")) + add_edge(ip_id, asn_id, "hébergé par", "hosted_by") + for host in (item.get("hostnames", []) if isinstance(item, dict) else [])[:16]: host_type = "domain" if host == domain else "subdomain" host_id = add_node(f"domain:{host}", host, host_type) add_edge(host_id, ip_id, "résout vers", "dns_resolution") + for host in (audit.get("host_enrichment") or {}).get("hosts", [])[:MAX_HOSTS]: + if not isinstance(host, dict): + continue + hostname = host.get("hostname") + if not hostname: + continue + host_id = add_node( + f"domain:{hostname}", + hostname, + "domain" if hostname == domain else "subdomain", + title=host.get("title"), + best_url=host.get("best_url"), + status=host.get("status"), + ) + add_edge(root_id, host_id, "host enrichi", "host_enrichment") + for ip in host.get("public_ips", [])[:12]: + ip_id = add_node(f"ip:{ip}", ip, "ip") + add_edge(host_id, ip_id, "résout vers", "host_resolution") + for tech in host.get("technologies", [])[:12]: + tech_id = add_node(f"tech:{tech}", tech, "technology") + add_edge(host_id, tech_id, "technologie", "fingerprint") + for banner in host.get("banners", [])[:8]: + banner_id = add_node(f"banner:{banner}", banner[:80], "banner") + add_edge(host_id, banner_id, "banner", "fingerprint") + cert = host.get("tls_certificate") or {} + if cert.get("subject_cn"): + cert_id = add_node(f"cert:{hostname}", cert.get("subject_cn"), "certificate", issuer=cert.get("issuer_cn"), not_after=cert.get("not_after")) + add_edge(host_id, cert_id, "certificat TLS", "tls_certificate") + for target in (audit.get("web") or {}).get("targets", [])[:MAX_SUBDOMAINS]: host = target.get("hostname") if not host: @@ -99,10 +122,6 @@ def add_edge(source: str, target: str, label: str, edge_type: str = "related", * if target.get("reachable"): web_id = add_node(f"web:{host}", target.get("best_scheme") or host, "web", reachable=True) add_edge(host_id, web_id, "HTTP(S)", "web_service") - guard = target.get("guard") or {} - for ip in guard.get("public_ips", [])[:12]: - ip_id = add_node(f"ip:{ip}", ip, "ip", source="web_guard") - add_edge(host_id, ip_id, "résout vers", "web_resolution") scan = audit.get("service_scan") or {} for port in scan.get("open_ports", [])[:MAX_SERVICES]: @@ -119,40 +138,23 @@ def add_edge(source: str, target: str, label: str, edge_type: str = "related", * ) add_edge(host_id, service_id, "expose", "exposes_service", port=port.get("port")) for cve in port.get("cves", []) or []: - cve_id = add_node( - f"cve:{cve.get('cve')}", - cve.get("cve", "CVE"), - "cve", - severity=cve.get("severity"), - cvss=cve.get("cvss"), - ) + cve_id = add_node(f"cve:{cve.get('cve')}", cve.get("cve", "CVE"), "cve", severity=cve.get("severity"), cvss=cve.get("cvss")) add_edge(service_id, cve_id, "corrélation CVE", "cve_correlation", confidence=cve.get("confidence")) for idx, finding in enumerate(sorted(audit.get("findings", []), key=lambda f: _severity_rank(f.get("severity")))[:MAX_FINDINGS]): title = finding.get("title") or finding.get("category") or "Constat" sev = finding.get("severity") or "info" - finding_id = add_node( - f"finding:{idx}:{abs(hash(title))}", - title[:80], - "finding", - severity=sev, - category=finding.get("category"), - recommendation=finding.get("recommendation"), - ) + finding_id = add_node(f"finding:{idx}:{abs(hash(title))}", title[:80], "finding", severity=sev, category=finding.get("category"), recommendation=finding.get("recommendation")) loc = finding.get("location") or {} host = loc.get("hostname") or loc.get("host") or loc.get("domain") or domain host_id = add_node(f"domain:{host}", host, "domain" if host == domain else "subdomain") add_edge(host_id, finding_id, sev, "has_finding", severity=sev) type_counts = Counter(node.get("type") for node in nodes.values()) - severity_counts = Counter( - str((node.get("properties") or {}).get("severity", "info")) - for node in nodes.values() - if node.get("type") in {"finding", "cve"} - ) + severity_counts = Counter(str((node.get("properties") or {}).get("severity", "info")) for node in nodes.values() if node.get("type") in {"finding", "cve"}) return { - "version": "v7.5", + "version": "OpenEASM Beta 26.6", "domain": domain, "generated_from_audit_id": audit.get("id"), "nodes": list(nodes.values()), @@ -166,11 +168,16 @@ def add_edge(source: str, target: str, label: str, edge_type: str = "related", * "service_cves": scan.get("count_cves", 0), "public_ips": inventory.get("public_ip_count", 0), "subdomains": (audit.get("subdomains") or {}).get("count", 0), + "enriched_hosts": (audit.get("host_enrichment") or {}).get("summary", {}).get("active_host_count", 0), }, "legend": { "domain": "Domaine racine", "subdomain": "Sous-domaine public", "ip": "Adresse IP publique", + "asn": "ASN / hébergeur", + "technology": "Technologie détectée", + "banner": "Bannière HTTP/service", + "certificate": "Certificat TLS", "web": "Service web observé", "service": "Port/service détecté par Nmap", "cve": "CVE corrélée sans exploitation", diff --git a/backend/app/services/host_enrichment.py b/backend/app/services/host_enrichment.py new file mode 100644 index 0000000..a8518b1 --- /dev/null +++ b/backend/app/services/host_enrichment.py @@ -0,0 +1,476 @@ +from __future__ import annotations + +import asyncio +import ipaddress +import os +import re +import socket +import ssl +from collections import Counter, defaultdict +from datetime import datetime, timezone +from html import unescape +from typing import Any + +import dns.resolver +import httpx +from cryptography import x509 +from cryptography.x509.oid import ExtensionOID, NameOID + +from app.services.network_guard import resolve_ips + +MAX_HOSTS = int(os.getenv("OPENEASM_ENRICH_MAX_HOSTS", "150")) +CONCURRENCY = int(os.getenv("OPENEASM_ENRICH_CONCURRENCY", "10")) +HTTP_TIMEOUT = float(os.getenv("OPENEASM_HTTP_TIMEOUT", "8.0")) +RDAP_TIMEOUT = float(os.getenv("OPENEASM_RDAP_TIMEOUT", "5.0")) +ENABLE_RDAP = os.getenv("OPENEASM_ENABLE_RDAP", "true").lower() in {"1", "true", "yes", "on"} + +TECH_RULES = [ + ("Apache", re.compile(r"apache", re.I)), + ("nginx", re.compile(r"nginx", re.I)), + ("Microsoft-IIS", re.compile(r"microsoft-iis|\biis\b", re.I)), + ("OpenSSL", re.compile(r"openssl", re.I)), + ("PHP", re.compile(r"php", re.I)), + ("ASP.NET", re.compile(r"asp\.net|x-aspnet", re.I)), + ("jQuery", re.compile(r"jquery(?:[-.]|/)?([0-9][0-9A-Za-z_.-]*)?", re.I)), + ("Bootstrap", re.compile(r"bootstrap(?:[-.]|/)?([0-9][0-9A-Za-z_.-]*)?", re.I)), + ("Modernizr", re.compile(r"modernizr", re.I)), + ("OneTrust", re.compile(r"onetrust", re.I)), + ("WordPress", re.compile(r"wp-content|wordpress", re.I)), + ("GLPI", re.compile(r"\bglpi\b", re.I)), + ("Zabbix", re.compile(r"zabbix", re.I)), + ("Fortinet", re.compile(r"fortinet|fortigate|fortipam|fortiguard", re.I)), + ("Apache Tomcat", re.compile(r"tomcat", re.I)), +] + +SECURITY_HEADERS = [ + "strict-transport-security", + "content-security-policy", + "x-frame-options", + "x-content-type-options", + "referrer-policy", + "permissions-policy", +] + + +def _domain_match(hostname: str, domain: str) -> bool: + hostname = hostname.strip(".").lower() + domain = domain.strip(".").lower() + return hostname == domain or hostname.endswith("." + domain) + + +def _uniq(items: list[str]) -> list[str]: + return sorted({str(item).strip().strip(".").lower() for item in items if item}) + + +def _host_candidates(domain: str, subdomains_result: dict, ip_inventory: dict) -> list[dict[str, Any]]: + by_host: dict[str, dict[str, Any]] = {} + + def add(hostname: str, source: str = "candidate", entry: dict | None = None) -> None: + if not hostname or not _domain_match(hostname, domain): + return + hostname = hostname.strip(".").lower() + item = by_host.setdefault( + hostname, + { + "hostname": hostname, + "sources": set(), + "ips": set(), + "public_ips": set(), + "blocked_ips": set(), + "cname_chain": [], + "resolved_name": None, + }, + ) + item["sources"].add(source) + if entry: + item["resolved_name"] = entry.get("resolved_name") or item.get("resolved_name") + item["cname_chain"] = entry.get("cname_chain") or item.get("cname_chain") or [] + for ip in entry.get("ips", []) or []: + item["ips"].add(ip) + for ip in entry.get("public_ips", []) or []: + item["public_ips"].add(ip) + for ip in entry.get("blocked_ips", []) or []: + item["blocked_ips"].add(ip) + + add(domain, "root") + add(f"www.{domain}", "www") + + for sub in subdomains_result.get("subdomains", []) or []: + add(str(sub), "subdomain") + + for entry in ip_inventory.get("entries", []) or []: + add(entry.get("hostname"), entry.get("source") or "inventory", entry) + + for ip_item in ip_inventory.get("unique_ips", []) or []: + if not isinstance(ip_item, dict): + continue + for host in ip_item.get("hostnames", []) or []: + add(host, "ip_inventory") + item = by_host.get(host.strip(".").lower()) + if item: + if ip_item.get("is_public"): + item["public_ips"].add(ip_item.get("ip")) + else: + item["blocked_ips"].add(ip_item.get("ip")) + + # Resolve missing candidates. This keeps the enrichment useful even when a + # passive source discovered a hostname after the inventory cap was reached. + for hostname, item in by_host.items(): + if item["ips"] or item["public_ips"] or item["blocked_ips"]: + continue + guard = resolve_ips(hostname) + for ip in guard.get("ips", []) or []: + item["ips"].add(ip) + for ip in guard.get("public_ips", []) or []: + item["public_ips"].add(ip) + for ip in guard.get("blocked_ips", []) or []: + item["blocked_ips"].add(ip) + + ordered = [] + for hostname in sorted(by_host): + item = by_host[hostname] + ordered.append( + { + "hostname": hostname, + "sources": sorted(item["sources"]), + "ips": sorted(item["ips"] or item["public_ips"] or item["blocked_ips"]), + "public_ips": sorted(ip for ip in item["public_ips"] if ip), + "blocked_ips": sorted(ip for ip in item["blocked_ips"] if ip), + "cname_chain": item.get("cname_chain") or [], + "resolved_name": item.get("resolved_name") or hostname, + } + ) + return ordered[:MAX_HOSTS] + + +def _reverse_for_cymru(ip: str) -> tuple[str, str]: + address = ipaddress.ip_address(ip) + if address.version == 4: + return ".".join(reversed(ip.split("."))) + ".origin.asn.cymru.com", "TXT" + nibbles = address.exploded.replace(":", "") + return ".".join(reversed(nibbles)) + ".origin6.asn.cymru.com", "TXT" + + +def _cymru_lookup(ip: str) -> dict[str, Any]: + try: + qname, rtype = _reverse_for_cymru(ip) + resolver = dns.resolver.Resolver() + resolver.lifetime = 3.0 + resolver.timeout = 3.0 + answers = resolver.resolve(qname, rtype) + raw = " ".join(str(a).strip('"') for a in answers) + # Format: AS | BGP Prefix | CC | Registry | Allocated | AS Name + parts = [p.strip() for p in raw.split("|")] + if len(parts) >= 5 and parts[0].lower() != "as": + return { + "asn": f"AS{parts[0]}", + "network": parts[1] if len(parts) > 1 else None, + "country": parts[2] if len(parts) > 2 else None, + "registry": parts[3] if len(parts) > 3 else None, + "allocated": parts[4] if len(parts) > 4 else None, + "asn_name": parts[5] if len(parts) > 5 else None, + "source": "Team Cymru", + } + except Exception as exc: + return {"error": str(exc), "source": "Team Cymru"} + return {"source": "Team Cymru"} + + +async def _rdap_lookup(client: httpx.AsyncClient, ip: str) -> dict[str, Any]: + if not ENABLE_RDAP: + return {} + try: + response = await client.get(f"https://rdap.org/ip/{ip}") + response.raise_for_status() + data = response.json() + return { + "rdap_handle": data.get("handle"), + "rdap_name": data.get("name"), + "country": data.get("country"), + "start_address": data.get("startAddress"), + "end_address": data.get("endAddress"), + "provider": data.get("name") or data.get("handle"), + "rdap_source": "rdap.org", + } + except Exception as exc: + return {"rdap_error": str(exc), "rdap_source": "rdap.org"} + + +async def _enrich_ips(ip_inventory: dict) -> dict[str, dict[str, Any]]: + ip_meta: dict[str, dict[str, Any]] = {} + ips = [] + for item in ip_inventory.get("unique_ips", []) or []: + if isinstance(item, dict) and item.get("ip") and item.get("is_public"): + ips.append(item["ip"]) + ips = sorted(set(ips)) + + async with httpx.AsyncClient(timeout=RDAP_TIMEOUT, follow_redirects=True) as client: + rdap_tasks = {ip: asyncio.create_task(_rdap_lookup(client, ip)) for ip in ips} + for ip in ips: + cymru = await asyncio.to_thread(_cymru_lookup, ip) + rdap = await rdap_tasks[ip] + merged = {**cymru, **{k: v for k, v in rdap.items() if v}} + if rdap.get("country") and not merged.get("country"): + merged["country"] = rdap["country"] + if not merged.get("provider"): + merged["provider"] = merged.get("asn_name") or merged.get("rdap_name") + ip_meta[ip] = merged + + for item in ip_inventory.get("unique_ips", []) or []: + if isinstance(item, dict) and item.get("ip") in ip_meta: + item.update({k: v for k, v in ip_meta[item["ip"]].items() if v not in (None, "", [], {})}) + return ip_meta + + +def _title_from_html(text: str) -> str | None: + match = re.search(r"