Skip to content

Canonical host deduplication (www.*), consolidate Nmap services and tidy reports/Graph Explorer #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Swapshadow merged 1 commit into from
Jun 11, 2026
Merged

Canonical host deduplication (www.*), consolidate Nmap services and tidy reports/Graph Explorer #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions backend/app/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
from app.services.dnsdumpster_like_inventory import build_dnsdumpster_like_inventory
from app.services.legal_terms import legal_payload, create_acceptance, validate_acceptance
from app.services.attack_graph import build_attack_graph
from app.services.host_canonicalization import consolidate_nmap_services

app = FastAPI(
title="OpenEASM Beta",
Expand Down Expand Up @@ -156,6 +157,7 @@ async def create_audit(payload: AuditRequest, request: Request, db=Depends(get_d
passive_cves = detect_passive_cves(web_result)
cti_result = audit_cti(ip_inventory, domain)
service_scan = await asyncio.to_thread(audit_service_versions, domain, ip_inventory)
service_scan["raw_nmap_services"] = list(service_scan.get("open_ports", []) or [])
dnsdumpster_like = await build_dnsdumpster_like_inventory(
domain,
dns_result,
Expand All @@ -164,6 +166,8 @@ async def create_audit(payload: AuditRequest, request: Request, db=Depends(get_d
ip_inventory,
service_scan,
)
alias_map = dnsdumpster_like.get("alias_map", {}) or {}
service_scan["canonical_open_ports"] = consolidate_nmap_services(service_scan.get("open_ports", []), alias_map)

raw_findings = collect_findings(
dns_result,
Expand Down Expand Up @@ -203,6 +207,9 @@ async def create_audit(payload: AuditRequest, request: Request, db=Depends(get_d
"passive_cves": passive_cves,
"service_scan": service_scan,
"dnsdumpster_like": dnsdumpster_like,
"canonical_hosts": dnsdumpster_like.get("canonical_hosts", []),
"raw_subdomains": subdomains_result.get("subdomains", []) or [],
"raw_nmap_services": service_scan.get("raw_nmap_services", []),
"cti": cti_result,
"patching_sla": patching_sla,
"executive_risk": executive_risk,
Expand Down Expand Up @@ -294,6 +301,9 @@ async def create_audit(payload: AuditRequest, request: Request, db=Depends(get_d
"ns_records": dnsdumpster_like.get("ns_records", [])[:50],
"txt_records": dnsdumpster_like.get("txt_records", [])[:80],
"hosts": dnsdumpster_like.get("hosts", [])[:80],
"canonical_hosts": dnsdumpster_like.get("canonical_hosts", [])[:80],
"aliases_count": dnsdumpster_like.get("aliases_count", 0),
"canonicalization": dnsdumpster_like.get("canonicalization", {}),
"policy": dnsdumpster_like.get("policy", {}),
},
"service_scan": {
Expand All @@ -303,7 +313,8 @@ async def create_audit(payload: AuditRequest, request: Request, db=Depends(get_d
"count_cves": service_scan.get("count_cves", 0),
"elapsed_seconds": service_scan.get("elapsed_seconds", 0),
"targets": service_scan.get("targets", [])[:10],
"open_ports": service_scan.get("open_ports", [])[:80],
"open_ports": service_scan.get("canonical_open_ports", service_scan.get("open_ports", []))[:80],
"raw_open_ports": service_scan.get("raw_nmap_services", [])[:80],
"cves": service_scan.get("cves", [])[:50],
"note": service_scan.get("note"),
},
Expand Down Expand Up @@ -338,6 +349,7 @@ async def create_audit(payload: AuditRequest, request: Request, db=Depends(get_d
"html_url": f"/api/reports/{audit_id}/html",
"graph_url": f"/api/audits/{audit_id}/graph",
"attack_graph": audit.get("attack_graph", {}),
"canonical_hosts": audit.get("canonical_hosts", [])[:120],
"report_errors": audit.get("report_errors", []),
"summary": {
"dns_public_ips": audit["dns"].get("public_ips", []),
Expand All @@ -351,8 +363,15 @@ async def create_audit(payload: AuditRequest, request: Request, db=Depends(get_d
"core_public_ip_count": ip_inventory.get("core_public_ip_count", 0),
"third_party_provider_ip_count": ip_inventory.get("third_party_provider_ip_count", 0),
"passive_cve_count": passive_cves.get("count", 0),
"service_open_port_count": service_scan.get("count_open_ports", 0),
"canonical_host_count": (dnsdumpster_like.get("canonicalization", {}) or {}).get("canonical_hosts", len(dnsdumpster_like.get("canonical_hosts", []) or [])),
"aliases_grouped": dnsdumpster_like.get("aliases_count", 0),
"http_https_service_count": (dnsdumpster_like.get("canonicalization", {}) or {}).get("http_https_services", 0),
"service_open_port_count": len(service_scan.get("canonical_open_ports", service_scan.get("open_ports", [])) or []),
"service_cve_count": service_scan.get("count_cves", 0),
"technology_count": (dnsdumpster_like.get("canonicalization", {}) or {}).get("technologies", 0),
"asn_network_count": (dnsdumpster_like.get("canonicalization", {}) or {}).get("asn_networks", 0),
"location_count": (dnsdumpster_like.get("canonicalization", {}) or {}).get("locations", 0),
"http_without_nmap_count": (dnsdumpster_like.get("canonicalization", {}) or {}).get("http_hosts_without_nmap", 0),
"service_scan_elapsed_seconds": service_scan.get("elapsed_seconds", 0),
"tls_score": tls_score.get("global_score"),
"tls_level": tls_score.get("global_level"),
Expand Down
14 changes: 9 additions & 5 deletions backend/app/reports/excel_report.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,9 @@ def _sheet_summary(ws, audit: dict):
("Posture", risk.get("posture")),
("Profil", profile.get("label")),
("IP publiques", ip.get("public_ip_count", 0)),
("Sous-domaines", sub.get("count", 0)),
("Ports Nmap", scan.get("count_open_ports", 0)),
("Hosts canoniques", len((audit.get("canonical_hosts") or (audit.get("dnsdumpster_like", {}) or {}).get("canonical_hosts") or []))),
("Aliases regroupés", (audit.get("dnsdumpster_like", {}) or {}).get("aliases_count", 0)),
("Ports Nmap", len(scan.get("canonical_open_ports", scan.get("open_ports", [])) or [])),
("CVE service/version", scan.get("count_cves", 0)),
("TLS", f"{audit.get('tls_score', {}).get('global_score', 'N/A')} / 100"),
("Date audit", audit.get("created_at")),
Expand Down Expand Up @@ -184,7 +185,7 @@ def _sheet_dnsdumpster_like(wb, audit):


def _host_headers():
return ["Host", "IP", "ASN", "Network", "ASN Name", "Country", "Provider", "Open Services", "HTTP Status", "HTTPS Status", "Final URL", "Title", "Server Banner", "TLS CN", "TLS SAN", "TLS Issuer", "TLS Expiration", "Technologies", "Nmap Services", "RevIP", "Sources"]
return ["Host", "Aliases", "IP", "ASN", "Network", "ASN Name", "Country", "Provider", "Open Services", "HTTP Status", "HTTPS Status", "Final URL", "Title", "Server Banner", "TLS CN", "TLS SAN", "TLS Issuer", "TLS Expiration", "Technologies", "Nmap Services", "RevIP", "Sources"]


def _host_row(h):
Expand All @@ -195,7 +196,7 @@ def _host_row(h):
server_banner = "; ".join(dict.fromkeys([s.get("banner") for s in h.get("open_services", []) if s.get("banner")]))
title = https.get("title") or http.get("title") or next((s.get("title") for s in h.get("open_services", []) if s.get("title") and s.get("title") != "Non détecté"), "Non détecté")
return [
h.get("host"), h.get("ip"), h.get("asn"), h.get("network"), h.get("asn_name"), h.get("country"), h.get("provider"),
h.get("host"), "\n".join(h.get("aliases", []) or []), h.get("ip"), h.get("asn"), h.get("network"), h.get("asn_name"), h.get("country"), h.get("provider"),
services, http.get("status_code") or "Non détecté", https.get("status_code") or "Non détecté", https.get("final_url") or http.get("final_url") or "Non détecté", title,
server_banner or "Non détecté", tls.get("cn") or "Non détecté", "\n".join(tls.get("san", []) or []), tls.get("issuer") or "Non détecté", tls.get("expires_at") or "Non détecté",
_tech_text(h.get("technologies", [])) or "Non détecté", _nmap_text(h.get("nmap_services", [])) or "Non détecté par Nmap", h.get("revip_count", 0), ", ".join(h.get("sources", [])),
Expand Down Expand Up @@ -277,7 +278,7 @@ def _sheet_nmap(wb, audit):
ws = wb.create_sheet("Nmap Services")
scan = audit.get("service_scan", {}) or {}
rows = []
for port in scan.get("open_ports", []):
for port in scan.get("canonical_open_ports", scan.get("open_ports", [])):
cves = port.get("cves", []) or []
if cves:
for cve in cves:
Expand Down Expand Up @@ -363,6 +364,9 @@ def _sheet_raw(wb, audit):
["score", json.dumps(audit.get("score", {}), ensure_ascii=False, default=str)],
["executive_risk", json.dumps(audit.get("executive_risk", {}), ensure_ascii=False, default=str)[:32000]],
["dnsdumpster_like", json.dumps(audit.get("dnsdumpster_like", {}), ensure_ascii=False, default=str)[:32000]],
["canonical_hosts", json.dumps(audit.get("canonical_hosts", []), ensure_ascii=False, default=str)[:32000]],
["raw_subdomains", json.dumps(audit.get("raw_subdomains", []), ensure_ascii=False, default=str)[:32000]],
["raw_nmap_services", json.dumps(audit.get("raw_nmap_services", []), ensure_ascii=False, default=str)[:32000]],
]
_table_sheet(ws, "RÉSUMÉ JSON", ["Clé", "Valeur"], rows)

Expand Down
20 changes: 14 additions & 6 deletions backend/app/reports/html_report.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,7 @@ def _render_ips(audit: dict) -> str:

def _render_nmap(audit: dict) -> str:
rows = []
for port in safe_get(audit, "service_scan.open_ports", []):
for port in (safe_get(audit, "service_scan.canonical_open_ports", []) or safe_get(audit, "service_scan.open_ports", [])):
if isinstance(port, dict):
rows.append([port.get("host") or port.get("ip"), port.get("port"), port.get("protocol"), port.get("service"), port.get("product"), port.get("version"), _join(port.get("cpe") or port.get("cpes")), _join(port.get("cves"))])
return '<p class="notice">Nmap service/version/port est utilisé en mode non exploitant.</p>' + render_table(["Host", "Port", "Proto", "Service", "Produit", "Version", "CPE", "CVE corrélées"], rows)
Expand Down Expand Up @@ -404,17 +404,20 @@ def _render_a_records_filterable(audit: dict) -> str:
<label><input id="webNoNmap" type="checkbox" onchange="filterDnsdumpsterTable()"> HTTP/HTTPS détecté mais Nmap vide</label>
<label><input id="guardOnly" type="checkbox" onchange="filterDnsdumpsterTable()"> Bloqué par garde-fou</label>
</div>"""
headers = ["Host", "IP", "ASN", "Network", "ASN Name", "Country", "Open Services observed", "RevIP", "Sources"]
headers = ["Host", "Aliases", "IP", "ASN", "Network", "ASN Name", "Country", "Open Services", "Title", "TLS CN", "Technologies", "RevIP", "Sources"]
rows = []
for h in records:
tech = _tech_text(h)
services = _open_services_html(h)
badges = _host_badges(h)
detail = _host_detail_html(h)
attrs = {"host": h.get("host", ""), "asn": h.get("asn", ""), "country": h.get("country", ""), "tech": tech, "service": _plain_services(h), "webnonmap": str(bool(h.get("open_services") and not h.get("nmap_services"))).lower(), "guard": str(bool(h.get("guardrail"))).lower()}
aliases = ", ".join(h.get("aliases", []) or [])
title = (h.get("https") or {}).get("title") or (h.get("http") or {}).get("title") or next((svc.get("title") for svc in h.get("open_services", []) if svc.get("title")), "Non détecté")
tls_cn = (h.get("tls") or {}).get("cn") or "Non détecté"
attrs = {"host": f"{h.get('host', '')} {aliases}", "asn": h.get("asn", ""), "country": h.get("country", ""), "tech": tech, "service": _plain_services(h), "webnonmap": str(bool(h.get("open_services") and not h.get("nmap_services"))).lower(), "guard": str(bool(h.get("guardrail"))).lower()}
attr_str = " ".join(f'data-{k}="{esc(v).lower()}"' for k, v in attrs.items())
host_cell = SafeHtml(f'<details><summary>{esc(h.get("host"))} {badges}</summary>{detail}</details>')
rows.append(SafeHtml(f'<tr {attr_str} class="dnsdumpster-row"><td>{host_cell}</td><td>{esc(h.get("ip"))}</td><td>{esc(h.get("asn"))}</td><td>{esc(h.get("network"))}</td><td>{esc(h.get("asn_name"))}</td><td>{esc(h.get("country"))}</td><td>{services}</td><td>{esc(h.get("revip_count", 0))}</td><td>{esc(", ".join(h.get("sources", []) or []))}</td></tr>'))
rows.append(SafeHtml(f'<tr {attr_str} class="dnsdumpster-row"><td>{host_cell}</td><td>{esc(aliases or "—")}</td><td>{esc(h.get("ip"))}</td><td>{esc(h.get("asn"))}</td><td>{esc(h.get("network"))}</td><td>{esc(h.get("asn_name"))}</td><td>{esc(h.get("country"))}</td><td>{services}</td><td>{esc(title)}</td><td>{esc(tls_cn)}</td><td>{esc(tech or "Non détecté")}</td><td>{esc(h.get("revip_count", 0))}</td><td>{esc(", ".join(h.get("sources", []) or []))}</td></tr>'))
if not rows:
return controls + '<p class="notice">Aucun A record enrichi disponible.</p>'
head = "".join(f"<th>{esc(h)}</th>" for h in headers)
Expand Down Expand Up @@ -523,15 +526,19 @@ def generate_html_report(audit: dict) -> str:
("services_banners", "Services / Banners"), ("a_records", "A Records / Subdomains"),
("mx_records", "MX Records"), ("ns_records", "NS Records"), ("txt_records", "TXT Records"),
("host_inventory", "Host Inventory détaillé"), ("nmap", "Nmap en annexe"), ("cve", "CVE"),
("limites", "Limites et sources"),
("graph", "Graph summary"), ("limites", "Limites et sources"),
]
hero_metrics = (
'<div class="metric-grid">'
+ _metric("Score global", f"{score.get('score', 'N/A')} / {score.get('max_score', 1000)}", score.get("level") or risk.get("risk_level") or "")
+ _metric("Niveau de risque", risk.get("risk_level") or score.get("level") or "N/A")
+ _metric("IP publiques", safe_get(audit, "ip_inventory.public_ip_count", 0))
+ _metric("Sous-domaines", safe_get(audit, "subdomains.count", len(_as_list(subdomains))))
+ _metric("Ports Nmap", safe_get(audit, "service_scan.count_open_ports", 0), "non exploitant")
+ _metric("Hosts canoniques", safe_get(audit, "dnsdumpster_like.canonicalization.canonical_hosts", len(safe_get(audit, "canonical_hosts", []))))
+ _metric("Aliases regroupés", safe_get(audit, "dnsdumpster_like.aliases_count", 0))
+ _metric("HTTP/HTTPS observés", safe_get(audit, "dnsdumpster_like.canonicalization.http_https_services", 0))
+ _metric("Ports Nmap", len(safe_get(audit, "service_scan.canonical_open_ports", []) or safe_get(audit, "service_scan.open_ports", [])), "consolidés")
+ _metric("Technologies", safe_get(audit, "dnsdumpster_like.canonicalization.technologies", 0))
+ _metric("CVE corrélées", cve_count, "corrélation de version")
+ '</div>'
)
Expand Down Expand Up @@ -560,6 +567,7 @@ def generate_html_report(audit: dict) -> str:
{section('host_inventory','Host Inventory détaillé',_render_host_inventory_detail(audit))}
{section('nmap','Nmap en annexe',_render_nmap(audit),'Rappel : collecte non exploitante, sans scripts intrusifs.')}
{section('cve','CVE',_render_cves(audit))}
{section('graph','Graph summary',_render_graph(audit))}
{section('limites','Limites et sources',limits)}
</div></main><script>
function filterDnsdumpsterTable(){{
Expand Down
14 changes: 7 additions & 7 deletions backend/app/reports/pdf_report.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ def _kpi_cards(audit: dict, styles):
("Score global", f"{score.get('score', 'N/A')} / {score.get('max_score', 1000)}", score.get("level", "N/A")),
("Risque exécutif", f"{risk.get('overall_score', 'N/A')} / {risk.get('max_score', 100)}", risk.get("risk_level", "N/A")),
("Surface publique", f"{audit.get('ip_inventory', {}).get('public_ip_count', 0)} IP", f"{audit.get('subdomains', {}).get('count', 0)} sous-domaines"),
("Nmap", f"{scan.get('count_open_ports', 0)} ports", f"{scan.get('count_cves', 0)} CVE corrélées"),
("Nmap", f"{len(scan.get('canonical_open_ports', scan.get('open_ports', [])) or [])} ports consolidés", f"{scan.get('count_cves', 0)} CVE corrélées"),
]
row = [Paragraph(f"<b>{escape(str(label))}</b><br/><font size='15' color='#65000B'><b>{escape(str(value))}</b></font><br/><font color='#7B5500'>{escape(str(note))}</font>", styles["Body"]) for label, value, note in items]
t = Table([row], colWidths=[6.25 * cm] * 4)
Expand Down Expand Up @@ -323,7 +323,7 @@ def _ip_table(audit: dict, styles, limit: int = 80):
def _nmap_table(audit: dict, styles, limit: int = 80):
scan = audit.get("service_scan", {}) or {}
rows = [["Hôte", "Port", "Service", "Produit", "Version", "CVE", "Sévérité"]]
ports = scan.get("open_ports", []) or []
ports = scan.get("canonical_open_ports", scan.get("open_ports", [])) or []
for port in ports[:limit]:
cves = port.get("cves", []) or []
if cves:
Expand Down Expand Up @@ -381,16 +381,16 @@ def _services_banners_table(dd: dict, styles):


def _dnsdumpster_hosts_table(records: list, styles, limit: int = 70, mx: bool = False):
headers = (["Priority"] if mx else []) + ["Host", "IP", "ASN", "Network", "ASN Name", "Country", "Open Services observed", "RevIP", "Sources"]
headers = (["Priority"] if mx else []) + ["Host", "Aliases", "IP", "ASN", "Network", "ASN Name", "Country", "Open Services observed", "RevIP", "Sources"]
rows = [headers]
for h in (records or [])[:limit]:
base = [h.get("host"), h.get("ip"), h.get("asn"), h.get("network"), h.get("asn_name"), h.get("country"), _open_services_line(h), h.get("revip_count", 0), ", ".join(h.get("sources", []) or [])]
base = [h.get("host"), ", ".join(h.get("aliases", []) or []), h.get("ip"), h.get("asn"), h.get("network"), h.get("asn_name"), h.get("country"), _open_services_line(h), h.get("revip_count", 0), ", ".join(h.get("sources", []) or [])]
rows.append(([h.get("priority")] if mx else []) + base)
if len(records or []) > limit:
rows.append((["..."] if mx else []) + [f"{len(records)-limit} lignes supplémentaires", "Voir JSON/Excel", "", "", "", "", "", "", ""])
rows.append((["..."] if mx else []) + [f"{len(records)-limit} lignes supplémentaires", "Voir JSON/Excel", "", "", "", "", "", "", "", ""])
if len(rows) == 1:
rows.append(([""] if mx else []) + ["Aucun", "Not found", "Non détecté", "Non détecté", "Non détecté", "Unknown", "Non détecté", "0", ""])
widths = ([1.8 * cm] if mx else []) + [4.2 * cm, 2.8 * cm, 2.4 * cm, 3.4 * cm, 5.2 * cm, 2.5 * cm, 7.4 * cm, 1.7 * cm, 2.7 * cm]
rows.append(([""] if mx else []) + ["Aucun", "", "Not found", "Non détecté", "Non détecté", "Non détecté", "Unknown", "Non détecté", "0", ""])
widths = ([1.5 * cm] if mx else []) + [3.5 * cm, 3.0 * cm, 2.4 * cm, 2.1 * cm, 3.0 * cm, 4.6 * cm, 2.2 * cm, 5.6 * cm, 1.4 * cm, 2.2 * cm]
scale = 25.5 * cm / sum(widths)
widths = [w * scale for w in widths]
return _table(rows, widths, small=True, styles=styles)
Expand Down
Loading
Loading