fix(census): settle the class-count shortfalls against AutoCAD's DXF export (#230)

ARCHITECTURE.md

@@ -5260,6 +5260,56 @@ cargo run --release --example probe_vp_ent_hdr -- ../dwg-corpus
 |---|---|---|---|
 | R14 | 3 | 3 | no twin |
 | R2000 | 3 | 3 | the DXF holds no such record |
+
+## 7bv. The class table is not a census; AutoCAD's export agrees with the walk (2026-10-26, #230)
+
+`AcDb:Classes` gives every class an instance count (DXF group 91), and
+`examples/probe_class_census` compares it with the walk. #56 and #76
+showed the count over-declares on the sample corpus. The public corpus
+(#225) added three more shortfalls: ACDB_TEXTOBJECTCONTEXTDATA_CLASS on
+ACadSharp's R2004 and R2007 saves, and GEODATA and AcGeoBackupObject on
+`geoloc.dwg`. On each, the walk is closed, with zero inter-record bytes
+and no unanswered reference.
+
+The probe's new `--twin` mode settles them against an independent census:
+AutoCAD's DXF export of the same drawing. It counts the export's records
+of each class, how many of those carry a handle that is no object of the
+drawing ("export-only"), and the count the export's own CLASSES section
+gives:
+
+| Drawing | Class | DWG declares | Walked | Export records | Export-only | Export's CLASSES count |
+|---|---|---:|---:|---:|---:|---:|
+| `sample_AC1018.dwg` (R2004) | ACDB_TEXTOBJECTCONTEXTDATA_CLASS | 2 | 0 | 0 | 0 | 2 |
+| `sample_AC1021.dwg` (R2007) | ACDB_TEXTOBJECTCONTEXTDATA_CLASS | 2 | 0 | 0 | 0 | 2 |
+| `sample_AC1024/1027/1032.dwg` | ACDB_TEXTOBJECTCONTEXTDATA_CLASS | 2 | 2 | 2 | 0 | 2 |
+| `geoloc.dwg` (R2018) | GEODATA | 2 | 1 | 1 | 0 | 1 |
+| `geoloc.dwg` | AcGeoBackupObject | 1 | 0 | 0 | 0 | 0 |
+| `sample_AC1018` … `sample_AC1032.dwg` | TABLECONTENT, TABLEGEOMETRY | 9 / 8 / 7 / 6 / 5 | 2 | 2 | 1 | 16 / 15 / 14 / 13 / 12 |
+
+AutoCAD's export of the R2004 and R2007 saves holds no
+ACDB_TEXTOBJECTCONTEXTDATA_CLASS record, though the class tables of both
+the DWG and the export count 2. `geoloc.dwg`'s export holds one GEODATA
+and no AcGeoBackupObject, and its own class table counts 1 and 0 where the
+DWG's counts 2 and 1. The two class tables disagree with each other, while
+the export's records agree with the walk. So the count is a stale
+registration figure, not a census, and the three classes join the
+allowlist.
+
+With `--twin`, an allowlisted shortfall is excused only when the export
+holds no more records of the class than the walk reached, besides its
+export-only ones, and a missing twin fails the run. All 42 drawings of
+the corpus that have a twin pass. The one export-only surplus is the
+nextgis R2010 saves' CELLSTYLEMAP: the export gives TABLESTYLE `0x65`
+an xdictionary `0xD2` holding a round-trip CELLSTYLEMAP `0xD3`. Neither
+handle is an object of the DWG, and nothing in the drawing refers to a
+handle it does not hold (`probe_reference_closure`). It is the same kind
+of round-trip data the export adds to LTYPE `GAS_LINE` in ACadSharp's
+R2004 to R2010 exports (#40).
+
+```sh
+cargo run --release --example probe_class_census -- ../dwg-corpus/acadsharp/sample_AC1018.dwg --twin ../dwg-corpus/acadsharp/sample_AC1018_ascii.dxf
+cargo run --release --example probe_class_census -- ../dwg-corpus/acadsharp/geoloc.dwg --twin ../dwg-corpus/acadsharp/geoloc.dxf
+```
 ## 8. Write pipeline (current scope)
 
 The inverse pipeline is partially shipped. Stage-1 (per-section

examples/probe_class_census.rs

@@ -7,7 +7,7 @@
 //! missing records.
 //!
 //! ```bash
-//! cargo run --release --example probe_class_census -- file.dwg [--strict]
+//! cargo run --release --example probe_class_census -- file.dwg [--strict] [--twin file.dxf]
 //! ```
 //!
 //! Exit code is non-zero when any class is under-walked, so the probe
@@ -23,10 +23,45 @@
 //! the probe reports them as `ALLOWED` instead of failing; `--strict`
 //! disables it so the numbers stay visible, and is what CI runs over
 //! the canonical fixtures.
+//!
+//! # The DXF twin (#230)
+//!
+//! `--twin` takes AutoCAD's DXF export of the same drawing and adds three
+//! columns: how many records of each class the export holds, how many of
+//! those have a handle the drawing's handle map does not list (`export-only`),
+//! and the instance count the export's own CLASSES section gives the
+//! class (group 91).
+//!
+//! The export is an independent census of what the drawing contains, so
+//! with a twin an allowlisted shortfall is excused only when the export
+//! holds no more records of the class than the walk reached, not counting
+//! export-only ones, and a missing, unreadable or malformed twin fails the run.
+//! The handle map, not the walk, says what the drawing holds, so a record the
+//! walker skipped cannot pass for export-only data and hide its own shortfall.
+//! A malformed twin is one whose lines are not whole group pairs, whose group
+//! codes are not numbers, whose sections do not open and close in order, that
+//! lacks an ENTITIES or an OBJECTS section or its closing `0 EOF`, or that has
+//! data after the EOF; an empty or truncated export would otherwise census
+//! every class at zero. The crate's `dxf_reader` parses and validates the
+//! grammar but sits behind a feature the CI builds of this probe do not enable.
+//! Export-only records are data the export adds: on the nextgis R2010
+//! saves it gives the TABLESTYLE (`0x65`) an xdictionary `0xD2` holding a
+//! round-trip CELLSTYLEMAP `0xD3`. Neither handle is an object of the
+//! DWG, and `probe_reference_closure` finds no record of the drawing
+//! referring to a handle it does not hold. The export also gives some
+//! records handles the DWG does not use (one of the two TABLECONTENT
+//! records of each ACadSharp save), so the rule bounds the export from
+//! above rather than asking for equality.
+//!
+//! Wherever the corpus has a twin, no shortfall has the export holding
+//! more records of the class than the walk reached, besides export-only
+//! ones, while the two class tables disagree with each other:
+//! `sample_AC1018.dwg` declares 9 TABLECONTENT, its export's table says
+//! 16, and the drawing and the export both hold 2 records.
 
 use dwg::error::Result;
 use dwg::{DwgFile, ObjectType};
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 use std::env;
 use std::process::ExitCode;
 
@@ -69,16 +104,42 @@ use std::process::ExitCode;
 /// `ACAD_ROUNDTRIP_2008_TABLESTYLE_CELLSTYLEMAP` dictionary key for one
 /// to hang from. Run `examples/probe_reference_closure` to reproduce.
 /// The counts stay pinned per file by `tests/class_census.rs`.
+///
+/// The public corpus (#225) added three more, each settled by AutoCAD's
+/// DXF export of the same drawing (`--twin`, #230). ACadSharp's R2004 and
+/// R2007 saves declare 2 ACDB_TEXTOBJECTCONTEXTDATA_CLASS and contain
+/// none, and their exports hold none either, while the R2010-R2018 saves
+/// and their exports hold the 2. `geoloc.dwg` declares 2 GEODATA and 1
+/// AcGeoBackupObject and holds 1 and 0; its export holds 1 and 0, and its
+/// own class table counts them 1 and 0. The same twins hold exactly 2
+/// TABLECONTENT and 2 TABLEGEOMETRY records on every release from R2004.
 pub const ALLOWLIST: &[(&str, &str)] = &[
     (
         "TABLECONTENT",
-        "declares 5 against 2 ACAD_TABLE entities; the object stream is \
-         99.92 % covered and no handle reference is unresolved (#56)",
+        "declares 5 to 9 against 2 ACAD_TABLE entities; the object stream is \
+         99.92 % covered, no handle reference is unresolved (#56), and \
+         AutoCAD's DXF export of each save holds 2 (#230)",
     ),
     (
         "TABLEGEOMETRY",
-        "declares 5 against 2 ACAD_TABLE entities; the object stream is \
-         99.92 % covered and no handle reference is unresolved (#56)",
+        "declares 5 to 9 against 2 ACAD_TABLE entities; the object stream is \
+         99.92 % covered, no handle reference is unresolved (#56), and \
+         AutoCAD's DXF export of each save holds 2 (#230)",
+    ),
+    (
+        "ACDB_TEXTOBJECTCONTEXTDATA_CLASS",
+        "the R2004 and R2007 saves declare 2 and hold none, and so do \
+         AutoCAD's DXF exports of them; the R2010+ saves and exports hold 2 (#230)",
+    ),
+    (
+        "GEODATA",
+        "geoloc.dwg declares 2 and holds 1; AutoCAD's DXF export holds 1 \
+         and its class table counts 1 (#230)",
+    ),
+    (
+        "AcGeoBackupObject",
+        "geoloc.dwg declares 1 and holds none; AutoCAD's DXF export holds \
+         none and its class table counts 0 (#230)",
     ),
     (
         "DICTIONARYVAR",
@@ -94,13 +155,130 @@ pub const ALLOWLIST: &[(&str, &str)] = &[
     ),
 ];
 
+/// The handle of every record per DXF name in a DXF export, and the
+/// instance count its CLASSES section gives each class (group 91), keyed
+/// by normalized name.
+#[derive(Default)]
+struct Twin {
+    records: BTreeMap<String, Vec<Option<u64>>>,
+    class_counts: BTreeMap<String, usize>,
+}
+
+fn read_twin(path: &str) -> Result<Twin, String> {
+    let bytes = std::fs::read(path).map_err(|e| e.to_string())?;
+    let text = String::from_utf8_lossy(&bytes);
+    let lines: Vec<&str> = text.lines().collect();
+    if lines.len() % 2 == 1 {
+        return Err(format!(
+            "{} lines are not whole group pairs; the file is truncated",
+            lines.len()
+        ));
+    }
+    let mut twin = Twin::default();
+    let mut class_name: Option<String> = None;
+    let mut in_class = false;
+    let mut current: Option<String> = None;
+    let mut open_section: Option<Option<String>> = None;
+    let mut sections = BTreeSet::new();
+    let mut saw_eof = false;
+    for (n, pair) in lines.chunks_exact(2).enumerate() {
+        let line = 2 * n + 1;
+        let (code, value) = (pair[0].trim(), pair[1].trim());
+        let numeric: i64 = code
+            .parse()
+            .map_err(|_| format!("line {line}: group code {code:?} is not a number"))?;
+        if saw_eof {
+            return Err(format!("line {line}: data after EOF"));
+        }
+        if numeric == 999 {
+            continue;
+        }
+        if numeric != 0 {
+            match &mut open_section {
+                Some(name @ None) if numeric == 2 => {
+                    sections.insert(normalize(value));
+                    *name = Some(normalize(value));
+                }
+                Some(None) => return Err(format!("line {line}: SECTION is not followed by 2")),
+                None => return Err(format!("line {line}: group {numeric} outside any section")),
+                Some(Some(_)) => {}
+            }
+        } else {
+            match (value, &open_section) {
+                ("SECTION", None) => open_section = Some(None),
+                ("SECTION", Some(_)) => {
+                    return Err(format!("line {line}: SECTION inside a section"));
+                }
+                (_, Some(None)) => {
+                    return Err(format!("line {line}: SECTION is not followed by 2"));
+                }
+                ("ENDSEC", Some(Some(_))) => open_section = None,
+                ("ENDSEC", None) => {
+                    return Err(format!("line {line}: ENDSEC with no open section"));
+                }
+                ("EOF", None) => saw_eof = true,
+                ("EOF", Some(_)) => return Err(format!("line {line}: EOF inside a section")),
+                (_, Some(Some(_))) => {}
+                (_, None) => return Err(format!("line {line}: {value} outside any section")),
+            }
+            in_class = value == "CLASS";
+            class_name = None;
+            let name = normalize(value);
+            twin.records.entry(name.clone()).or_default().push(None);
+            current = Some(name);
+            continue;
+        }
+        if code == "5" || code == "105" {
+            // The record's own handle is its first `5` (`105` on DIMSTYLE).
+            if let Some(slot) = current
+                .as_ref()
+                .and_then(|n| twin.records.get_mut(n))
+                .and_then(|v| v.last_mut())
+                .filter(|h| h.is_none())
+            {
+                *slot = u64::from_str_radix(value, 16).ok();
+            }
+        } else if in_class && code == "1" {
+            class_name = Some(normalize(value));
+        } else if in_class && code == "91" {
+            if let (Some(name), Ok(n)) = (class_name.as_ref(), value.parse()) {
+                twin.class_counts.insert(name.clone(), n);
+            }
+        }
+    }
+    if !saw_eof {
+        return Err("no closing EOF; the file is empty or truncated".to_string());
+    }
+    for needed in ["ENTITIES", "OBJECTS"] {
+        if !sections.contains(needed) {
+            return Err(format!("no {needed} section"));
+        }
+    }
+    Ok(twin)
+}
+
 fn main() -> ExitCode {
-    let Some(path) = env::args().nth(1) else {
-        eprintln!("usage: probe_class_census <file.dwg> [--strict]");
+    let args: Vec<String> = env::args().skip(1).collect();
+    let Some(path) = args.first().cloned() else {
+        eprintln!("usage: probe_class_census <file.dwg> [--strict] [--twin <file.dxf>]");
         return ExitCode::FAILURE;
     };
-    let strict = env::args().any(|a| a == "--strict");
-    match run(&path, strict) {
+    let strict = args.iter().any(|a| a == "--strict");
+    let twin = match args.iter().position(|a| a == "--twin") {
+        Some(i) => match args.get(i + 1).map(|p| read_twin(p)) {
+            Some(Ok(twin)) => Some(twin),
+            Some(Err(e)) => {
+                eprintln!("cannot read the DXF twin: {e}");
+                return ExitCode::FAILURE;
+            }
+            None => {
+                eprintln!("--twin needs a DXF path");
+                return ExitCode::FAILURE;
+            }
+        },
+        None => None,
+    };
+    match run(&path, strict, twin.as_ref()) {
         Ok(true) => ExitCode::SUCCESS,
         Ok(false) => ExitCode::FAILURE,
         Err(e) => {
@@ -110,7 +288,7 @@ fn main() -> ExitCode {
     }
 }
 
-fn run(path: &str, strict: bool) -> Result<bool> {
+fn run(path: &str, strict: bool, twin: Option<&Twin>) -> Result<bool> {
     let file = DwgFile::open(path)?;
     println!("file    : {path}");
     println!("version : {}", file.version());
@@ -225,11 +403,21 @@ fn run(path: &str, strict: bool) -> Result<bool> {
         );
     }
     println!();
+    let twin_header = if twin.is_some() {
+        format!(" {:>6} {:>11} {:>8}", "twin", "export-only", "twin 91")
+    } else {
+        String::new()
+    };
+    let Some(handle_map) = file.handle_map().transpose()? else {
+        println!("no handle map to tell the export's records from the drawing's");
+        return Ok(false);
+    };
+    let in_drawing: std::collections::HashSet<u64> = handle_map.iter().map(|e| e.handle).collect();
     println!(
-        "{:<38} {:>6} {:>9} {:>7} {:>7} {:>7}",
+        "{:<38} {:>6} {:>9} {:>7} {:>7} {:>7}{twin_header}",
         "class", "code", "declared", "walked", "fixed", "delta"
     );
-    println!("{}", "-".repeat(80));
+    println!("{}", "-".repeat(if twin.is_some() { 108 } else { 80 }));
 
     let mut complete = true;
     let mut allowed = Vec::new();
@@ -244,19 +432,45 @@ fn run(path: &str, strict: bool) -> Result<bool> {
             continue;
         }
         let delta = (seen + fixed) as i64 - declared as i64;
+        let key = normalize(&def.dxf_class_name);
+        // The export's records of this class, and how many of them have a
+        // handle the drawing's handle map does not list: data the export writes
+        // and the DWG does not hold, such as the round-trip objects AutoCAD
+        // adds when it writes an older release.
+        let twin_records = twin.map(|t| {
+            let handles = t.records.get(&key).map(Vec::as_slice).unwrap_or(&[]);
+            let export_only = handles
+                .iter()
+                .filter(|h| h.is_some_and(|h| !in_drawing.contains(&h)))
+                .count();
+            (handles.len(), export_only)
+        });
+        // With a twin, a shortfall is excused only when the export holds no
+        // more records of the class than the walk reached, besides its
+        // export-only ones: the allowlist names the classes, and the twin
+        // has to agree for this file.
         let excused = !strict
             && delta < 0
-            && ALLOWLIST
-                .iter()
-                .any(|(name, _)| normalize(name) == normalize(&def.dxf_class_name));
+            && twin_records.is_none_or(|(n, export_only)| n - export_only <= seen + fixed)
+            && ALLOWLIST.iter().any(|(name, _)| normalize(name) == key);
         if delta < 0 && !excused {
             complete = false;
         }
         if excused {
             allowed.push(def.dxf_class_name.clone());
         }
+        let twin_cells = match (twin, twin_records) {
+            (Some(t), Some((records, export_only))) => {
+                let counted = t
+                    .class_counts
+                    .get(&key)
+                    .map_or_else(|| "-".to_string(), |n| n.to_string());
+                format!(" {records:>6} {export_only:>11} {counted:>8}")
+            }
+            _ => String::new(),
+        };
         println!(
-            "{:<38} {:>6} {:>9} {:>7} {:>7} {:>+7} {}",
+            "{:<38} {:>6} {:>9} {:>7} {:>7} {:>+7}{twin_cells} {}",
             def.dxf_class_name,
             def.class_number,
             declared,