From c8cab4f746167d6dfdd27c12bdc46c7c8ed79443 Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Mon, 26 Oct 2026 08:51:09 -0400 Subject: [PATCH 1/4] fix(census): settle the class-count shortfalls against AutoCAD's DXF export (#230) probe_class_census gains --twin: the export's records of each class, how many carry a handle that is no object of the drawing, and the export's own CLASSES count. ACDB_TEXTOBJECTCONTEXTDATA_CLASS on the R2004/R2007 saves and GEODATA and AcGeoBackupObject on geoloc.dwg are short of the DWG's class count, and the export holds exactly what the walk reaches, while the two class tables disagree with each other. They join the allowlist. With --twin, an allowlisted shortfall is excused only when the export agrees, and a missing twin fails the run. --- ARCHITECTURE.md | 50 +++++++++ examples/probe_class_census.rs | 179 ++++++++++++++++++++++++++++++--- 2 files changed, 213 insertions(+), 16 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 4fd7f5b9..2f786c45 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1660,6 +1660,56 @@ strongest of the four checks: an encoder built from the measured layout reproduces every parity byte AutoCAD wrote, so the layout is not merely consistent with the bytes but the one that produced them. +## 7y. The class table is not a census; AutoCAD's export agrees with the walk (2026-10-26, #230) + +`AcDb:Classes` gives every class an instance count (DXF group 91), and +`examples/probe_class_census` compares it with the walk. #56 and #76 +showed the count over-declares on the sample corpus. The public corpus +(#225) added three more shortfalls: ACDB_TEXTOBJECTCONTEXTDATA_CLASS on +ACadSharp's R2004 and R2007 saves, and GEODATA and AcGeoBackupObject on +`geoloc.dwg`. On each, the walk is closed, with zero inter-record bytes +and no unanswered reference. + +The probe's new `--twin` mode settles them against an independent census: +AutoCAD's DXF export of the same drawing. It counts the export's records +of each class, how many of those carry a handle that is no object of the +drawing ("export-only"), and the count the export's own CLASSES section +gives: + +| Drawing | Class | DWG declares | Walked | Export records | Export-only | Export's CLASSES count | +|---|---|---:|---:|---:|---:|---:| +| `sample_AC1018.dwg` (R2004) | ACDB_TEXTOBJECTCONTEXTDATA_CLASS | 2 | 0 | 0 | 0 | 2 | +| `sample_AC1021.dwg` (R2007) | ACDB_TEXTOBJECTCONTEXTDATA_CLASS | 2 | 0 | 0 | 0 | 2 | +| `sample_AC1024/1027/1032.dwg` | ACDB_TEXTOBJECTCONTEXTDATA_CLASS | 2 | 2 | 2 | 0 | 2 | +| `geoloc.dwg` (R2018) | GEODATA | 2 | 1 | 1 | 0 | 1 | +| `geoloc.dwg` | AcGeoBackupObject | 1 | 0 | 0 | 0 | 0 | +| `sample_AC1018` … `sample_AC1032.dwg` | TABLECONTENT, TABLEGEOMETRY | 9 / 8 / 7 / 6 / 5 | 2 | 2 | 1 | 16 / 15 / 14 / 13 / 12 | + +AutoCAD's export of the R2004 and R2007 saves holds no +ACDB_TEXTOBJECTCONTEXTDATA_CLASS record, though the class tables of both +the DWG and the export count 2. `geoloc.dwg`'s export holds one GEODATA +and no AcGeoBackupObject, and its own class table counts 1 and 0 where the +DWG's counts 2 and 1. The two class tables disagree with each other, while +the export's records agree with the walk. So the count is a stale +registration figure, not a census, and the three classes join the +allowlist. + +With `--twin`, an allowlisted shortfall is excused only when the export +holds no more records of the class than the walk reached, besides its +export-only ones, and a missing twin fails the run. All 42 drawings of +the corpus that have a twin pass. The one export-only surplus is the +nextgis R2010 saves' CELLSTYLEMAP: the export gives TABLESTYLE `0x65` +an xdictionary `0xD2` holding a round-trip CELLSTYLEMAP `0xD3`. Neither +handle is an object of the DWG, and nothing in the drawing refers to a +handle it does not hold (`probe_reference_closure`). It is the same kind +of round-trip data the export adds to LTYPE `GAS_LINE` in ACadSharp's +R2004 to R2010 exports (#40). + +```sh +cargo run --release --example probe_class_census -- ../dwg-corpus/acadsharp/sample_AC1018.dwg --twin ../dwg-corpus/acadsharp/sample_AC1018_ascii.dxf +cargo run --release --example probe_class_census -- ../dwg-corpus/acadsharp/geoloc.dwg --twin ../dwg-corpus/acadsharp/geoloc.dxf +``` + ## 8. Write pipeline (current scope) The inverse pipeline is partially shipped. Stage-1 (per-section diff --git a/examples/probe_class_census.rs b/examples/probe_class_census.rs index 41f72a22..8e0baf4c 100644 --- a/examples/probe_class_census.rs +++ b/examples/probe_class_census.rs @@ -7,7 +7,7 @@ //! missing records. //! //! ```bash -//! cargo run --release --example probe_class_census -- file.dwg [--strict] +//! cargo run --release --example probe_class_census -- file.dwg [--strict] [--twin file.dxf] //! ``` //! //! Exit code is non-zero when any class is under-walked, so the probe @@ -23,6 +23,33 @@ //! the probe reports them as `ALLOWED` instead of failing; `--strict` //! disables it so the numbers stay visible, and is what CI runs over //! the canonical fixtures. +//! +//! # The DXF twin (#230) +//! +//! `--twin` takes AutoCAD's DXF export of the same drawing and adds three +//! columns: how many records of each class the export holds, how many of +//! those have a handle that is no object of the drawing (`export-only`), +//! and the instance count the export's own CLASSES section gives the +//! class (group 91). +//! +//! The export is an independent census of what the drawing contains, so +//! with a twin an allowlisted shortfall is excused only when the export +//! holds no more records of the class than the walk reached, not counting +//! export-only ones, and a missing or unreadable twin fails the run. +//! Export-only records are data the export adds: on the nextgis R2010 +//! saves it gives the TABLESTYLE (`0x65`) an xdictionary `0xD2` holding a +//! round-trip CELLSTYLEMAP `0xD3`. Neither handle is an object of the +//! DWG, and `probe_reference_closure` finds no record of the drawing +//! referring to a handle it does not hold. The export also gives some +//! records handles the DWG does not use (one of the two TABLECONTENT +//! records of each ACadSharp save), so the rule bounds the export from +//! above rather than asking for equality. +//! +//! Wherever the corpus has a twin, no shortfall has the export holding +//! more records of the class than the walk reached, besides export-only +//! ones, while the two class tables disagree with each other: +//! `sample_AC1018.dwg` declares 9 TABLECONTENT, its export's table says +//! 16, and the drawing and the export both hold 2 records. use dwg::error::Result; use dwg::{DwgFile, ObjectType}; @@ -69,16 +96,42 @@ use std::process::ExitCode; /// `ACAD_ROUNDTRIP_2008_TABLESTYLE_CELLSTYLEMAP` dictionary key for one /// to hang from. Run `examples/probe_reference_closure` to reproduce. /// The counts stay pinned per file by `tests/class_census.rs`. +/// +/// The public corpus (#225) added three more, each settled by AutoCAD's +/// DXF export of the same drawing (`--twin`, #230). ACadSharp's R2004 and +/// R2007 saves declare 2 ACDB_TEXTOBJECTCONTEXTDATA_CLASS and contain +/// none, and their exports hold none either, while the R2010-R2018 saves +/// and their exports hold the 2. `geoloc.dwg` declares 2 GEODATA and 1 +/// AcGeoBackupObject and holds 1 and 0; its export holds 1 and 0, and its +/// own class table counts them 1 and 0. The same twins hold exactly 2 +/// TABLECONTENT and 2 TABLEGEOMETRY records on every release from R2004. pub const ALLOWLIST: &[(&str, &str)] = &[ ( "TABLECONTENT", - "declares 5 against 2 ACAD_TABLE entities; the object stream is \ - 99.92 % covered and no handle reference is unresolved (#56)", + "declares 5 to 9 against 2 ACAD_TABLE entities; the object stream is \ + 99.92 % covered, no handle reference is unresolved (#56), and \ + AutoCAD's DXF export of each save holds 2 (#230)", ), ( "TABLEGEOMETRY", - "declares 5 against 2 ACAD_TABLE entities; the object stream is \ - 99.92 % covered and no handle reference is unresolved (#56)", + "declares 5 to 9 against 2 ACAD_TABLE entities; the object stream is \ + 99.92 % covered, no handle reference is unresolved (#56), and \ + AutoCAD's DXF export of each save holds 2 (#230)", + ), + ( + "ACDB_TEXTOBJECTCONTEXTDATA_CLASS", + "the R2004 and R2007 saves declare 2 and hold none, and so do \ + AutoCAD's DXF exports of them; the R2010+ saves and exports hold 2 (#230)", + ), + ( + "GEODATA", + "geoloc.dwg declares 2 and holds 1; AutoCAD's DXF export holds 1 \ + and its class table counts 1 (#230)", + ), + ( + "AcGeoBackupObject", + "geoloc.dwg declares 1 and holds none; AutoCAD's DXF export holds \ + none and its class table counts 0 (#230)", ), ( "DICTIONARYVAR", @@ -94,13 +147,74 @@ pub const ALLOWLIST: &[(&str, &str)] = &[ ), ]; +/// The handle of every record per DXF name in a DXF export, and the +/// instance count its CLASSES section gives each class (group 91), keyed +/// by normalized name. +#[derive(Default)] +struct Twin { + records: BTreeMap>>, + class_counts: BTreeMap, +} + +fn read_twin(path: &str) -> std::io::Result { + let bytes = std::fs::read(path)?; + let text = String::from_utf8_lossy(&bytes); + let lines: Vec<&str> = text.lines().collect(); + let mut twin = Twin::default(); + let mut class_name: Option = None; + let mut in_class = false; + let mut current: Option = None; + for pair in lines.chunks_exact(2) { + let (code, value) = (pair[0].trim(), pair[1].trim()); + if code == "0" { + in_class = value == "CLASS"; + class_name = None; + let name = normalize(value); + twin.records.entry(name.clone()).or_default().push(None); + current = Some(name); + } else if code == "5" || code == "105" { + // The record's own handle is its first `5` (`105` on DIMSTYLE). + if let Some(slot) = current + .as_ref() + .and_then(|n| twin.records.get_mut(n)) + .and_then(|v| v.last_mut()) + .filter(|h| h.is_none()) + { + *slot = u64::from_str_radix(value, 16).ok(); + } + } else if in_class && code == "1" { + class_name = Some(normalize(value)); + } else if in_class && code == "91" { + if let (Some(name), Ok(n)) = (class_name.as_ref(), value.parse()) { + twin.class_counts.insert(name.clone(), n); + } + } + } + Ok(twin) +} + fn main() -> ExitCode { - let Some(path) = env::args().nth(1) else { - eprintln!("usage: probe_class_census [--strict]"); + let args: Vec = env::args().skip(1).collect(); + let Some(path) = args.first().cloned() else { + eprintln!("usage: probe_class_census [--strict] [--twin ]"); return ExitCode::FAILURE; }; - let strict = env::args().any(|a| a == "--strict"); - match run(&path, strict) { + let strict = args.iter().any(|a| a == "--strict"); + let twin = match args.iter().position(|a| a == "--twin") { + Some(i) => match args.get(i + 1).map(|p| read_twin(p)) { + Some(Ok(twin)) => Some(twin), + Some(Err(e)) => { + eprintln!("cannot read the DXF twin: {e}"); + return ExitCode::FAILURE; + } + None => { + eprintln!("--twin needs a DXF path"); + return ExitCode::FAILURE; + } + }, + None => None, + }; + match run(&path, strict, twin.as_ref()) { Ok(true) => ExitCode::SUCCESS, Ok(false) => ExitCode::FAILURE, Err(e) => { @@ -110,7 +224,7 @@ fn main() -> ExitCode { } } -fn run(path: &str, strict: bool) -> Result { +fn run(path: &str, strict: bool, twin: Option<&Twin>) -> Result { let file = DwgFile::open(path)?; println!("file : {path}"); println!("version : {}", file.version()); @@ -225,11 +339,18 @@ fn run(path: &str, strict: bool) -> Result { ); } println!(); + let twin_header = if twin.is_some() { + format!(" {:>6} {:>11} {:>8}", "twin", "export-only", "twin 91") + } else { + String::new() + }; + let in_drawing: std::collections::HashSet = + objects.iter().map(|o| o.handle.value).collect(); println!( - "{:<38} {:>6} {:>9} {:>7} {:>7} {:>7}", + "{:<38} {:>6} {:>9} {:>7} {:>7} {:>7}{twin_header}", "class", "code", "declared", "walked", "fixed", "delta" ); - println!("{}", "-".repeat(80)); + println!("{}", "-".repeat(if twin.is_some() { 108 } else { 80 })); let mut complete = true; let mut allowed = Vec::new(); @@ -244,19 +365,45 @@ fn run(path: &str, strict: bool) -> Result { continue; } let delta = (seen + fixed) as i64 - declared as i64; + let key = normalize(&def.dxf_class_name); + // The export's records of this class, and how many of them have a + // handle that is no object of the drawing: data the export writes + // and the DWG does not hold, such as the round-trip objects AutoCAD + // adds when it writes an older release. + let twin_records = twin.map(|t| { + let handles = t.records.get(&key).map(Vec::as_slice).unwrap_or(&[]); + let export_only = handles + .iter() + .filter(|h| h.is_some_and(|h| !in_drawing.contains(&h))) + .count(); + (handles.len(), export_only) + }); + // With a twin, a shortfall is excused only when the export holds no + // more records of the class than the walk reached, besides its + // export-only ones: the allowlist names the classes, and the twin + // has to agree for this file. let excused = !strict && delta < 0 - && ALLOWLIST - .iter() - .any(|(name, _)| normalize(name) == normalize(&def.dxf_class_name)); + && twin_records.is_none_or(|(n, export_only)| n - export_only <= seen + fixed) + && ALLOWLIST.iter().any(|(name, _)| normalize(name) == key); if delta < 0 && !excused { complete = false; } if excused { allowed.push(def.dxf_class_name.clone()); } + let twin_cells = match (twin, twin_records) { + (Some(t), Some((records, export_only))) => { + let counted = t + .class_counts + .get(&key) + .map_or_else(|| "-".to_string(), |n| n.to_string()); + format!(" {records:>6} {export_only:>11} {counted:>8}") + } + _ => String::new(), + }; println!( - "{:<38} {:>6} {:>9} {:>7} {:>7} {:>+7} {}", + "{:<38} {:>6} {:>9} {:>7} {:>7} {:>+7}{twin_cells} {}", def.dxf_class_name, def.class_number, declared, From 05dfc77dc575b12427781bd03c44c6c15e9288ab Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Mon, 26 Oct 2026 13:44:44 -0400 Subject: [PATCH 2/4] docs(architecture): renumber the new section (7y->7z) after the sections main added --- ARCHITECTURE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2f786c45..ed596b06 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1660,7 +1660,7 @@ strongest of the four checks: an encoder built from the measured layout reproduces every parity byte AutoCAD wrote, so the layout is not merely consistent with the bytes but the one that produced them. -## 7y. The class table is not a census; AutoCAD's export agrees with the walk (2026-10-26, #230) +## 7z. The class table is not a census; AutoCAD's export agrees with the walk (2026-10-26, #230) `AcDb:Classes` gives every class an instance count (DXF group 91), and `examples/probe_class_census` compares it with the walk. #56 and #76 From 72cad8f2202a199a85f95059d5c7a8fcf8b5337b Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:32:17 -0400 Subject: [PATCH 3/4] fix(census): tell export-only records by the handle map and refuse a malformed DXF twin (#230) The export-only column was built from the objects the walker returned, so a record the walker skipped counted as data the export adds and subtracted itself from the independent census: with the sole GEODATA skipped, the walked count and the adjusted twin count were both zero and the shortfall was excused. The set now comes from the drawing's handle map, which the walk cannot shrink, and a run without one fails. read_twin accepted any file and returned a census of zero for an empty or truncated export, which excuses every allowlisted shortfall. It now refuses a file whose lines are not whole group pairs, whose group codes are not numbers, whose sections do not open and close in order, that lacks an ENTITIES or an OBJECTS section or its closing EOF, or that has data after the EOF. The crate's dxf_reader does this too but is behind a feature the CI builds of the probe do not enable. --- examples/probe_class_census.rs | 89 +++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 11 deletions(-) diff --git a/examples/probe_class_census.rs b/examples/probe_class_census.rs index 8e0baf4c..17661b68 100644 --- a/examples/probe_class_census.rs +++ b/examples/probe_class_census.rs @@ -28,14 +28,22 @@ //! //! `--twin` takes AutoCAD's DXF export of the same drawing and adds three //! columns: how many records of each class the export holds, how many of -//! those have a handle that is no object of the drawing (`export-only`), +//! those have a handle the drawing's handle map does not list (`export-only`), //! and the instance count the export's own CLASSES section gives the //! class (group 91). //! //! The export is an independent census of what the drawing contains, so //! with a twin an allowlisted shortfall is excused only when the export //! holds no more records of the class than the walk reached, not counting -//! export-only ones, and a missing or unreadable twin fails the run. +//! export-only ones, and a missing, unreadable or malformed twin fails the run. +//! The handle map, not the walk, says what the drawing holds, so a record the +//! walker skipped cannot pass for export-only data and hide its own shortfall. +//! A malformed twin is one whose lines are not whole group pairs, whose group +//! codes are not numbers, whose sections do not open and close in order, that +//! lacks an ENTITIES or an OBJECTS section or its closing `0 EOF`, or that has +//! data after the EOF; an empty or truncated export would otherwise census +//! every class at zero. The crate's `dxf_reader` parses and validates the +//! grammar but sits behind a feature the CI builds of this probe do not enable. //! Export-only records are data the export adds: on the nextgis R2010 //! saves it gives the TABLESTYLE (`0x65`) an xdictionary `0xD2` holding a //! round-trip CELLSTYLEMAP `0xD3`. Neither handle is an object of the @@ -53,7 +61,7 @@ use dwg::error::Result; use dwg::{DwgFile, ObjectType}; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::env; use std::process::ExitCode; @@ -156,23 +164,71 @@ struct Twin { class_counts: BTreeMap, } -fn read_twin(path: &str) -> std::io::Result { - let bytes = std::fs::read(path)?; +fn read_twin(path: &str) -> Result { + let bytes = std::fs::read(path).map_err(|e| e.to_string())?; let text = String::from_utf8_lossy(&bytes); let lines: Vec<&str> = text.lines().collect(); + if lines.len() % 2 == 1 { + return Err(format!( + "{} lines are not whole group pairs; the file is truncated", + lines.len() + )); + } let mut twin = Twin::default(); let mut class_name: Option = None; let mut in_class = false; let mut current: Option = None; - for pair in lines.chunks_exact(2) { + let mut open_section: Option> = None; + let mut sections = BTreeSet::new(); + let mut saw_eof = false; + for (n, pair) in lines.chunks_exact(2).enumerate() { + let line = 2 * n + 1; let (code, value) = (pair[0].trim(), pair[1].trim()); - if code == "0" { + let numeric: i64 = code + .parse() + .map_err(|_| format!("line {line}: group code {code:?} is not a number"))?; + if saw_eof { + return Err(format!("line {line}: data after EOF")); + } + if numeric == 999 { + continue; + } + if numeric != 0 { + match &mut open_section { + Some(name @ None) if numeric == 2 => { + sections.insert(normalize(value)); + *name = Some(normalize(value)); + } + Some(None) => return Err(format!("line {line}: SECTION is not followed by 2")), + None => return Err(format!("line {line}: group {numeric} outside any section")), + Some(Some(_)) => {} + } + } else { + match (value, &open_section) { + ("SECTION", None) => open_section = Some(None), + ("SECTION", Some(_)) => { + return Err(format!("line {line}: SECTION inside a section")); + } + (_, Some(None)) => { + return Err(format!("line {line}: SECTION is not followed by 2")); + } + ("ENDSEC", Some(Some(_))) => open_section = None, + ("ENDSEC", None) => { + return Err(format!("line {line}: ENDSEC with no open section")); + } + ("EOF", None) => saw_eof = true, + ("EOF", Some(_)) => return Err(format!("line {line}: EOF inside a section")), + (_, Some(Some(_))) => {} + (_, None) => return Err(format!("line {line}: {value} outside any section")), + } in_class = value == "CLASS"; class_name = None; let name = normalize(value); twin.records.entry(name.clone()).or_default().push(None); current = Some(name); - } else if code == "5" || code == "105" { + continue; + } + if code == "5" || code == "105" { // The record's own handle is its first `5` (`105` on DIMSTYLE). if let Some(slot) = current .as_ref() @@ -190,6 +246,14 @@ fn read_twin(path: &str) -> std::io::Result { } } } + if !saw_eof { + return Err("no closing EOF; the file is empty or truncated".to_string()); + } + for needed in ["ENTITIES", "OBJECTS"] { + if !sections.contains(needed) { + return Err(format!("no {needed} section")); + } + } Ok(twin) } @@ -344,8 +408,11 @@ fn run(path: &str, strict: bool, twin: Option<&Twin>) -> Result { } else { String::new() }; - let in_drawing: std::collections::HashSet = - objects.iter().map(|o| o.handle.value).collect(); + let Some(handle_map) = file.handle_map().transpose()? else { + println!("no handle map to tell the export's records from the drawing's"); + return Ok(false); + }; + let in_drawing: std::collections::HashSet = handle_map.iter().map(|e| e.handle).collect(); println!( "{:<38} {:>6} {:>9} {:>7} {:>7} {:>7}{twin_header}", "class", "code", "declared", "walked", "fixed", "delta" @@ -367,7 +434,7 @@ fn run(path: &str, strict: bool, twin: Option<&Twin>) -> Result { let delta = (seen + fixed) as i64 - declared as i64; let key = normalize(&def.dxf_class_name); // The export's records of this class, and how many of them have a - // handle that is no object of the drawing: data the export writes + // handle the drawing's handle map does not list: data the export writes // and the DWG does not hold, such as the round-trip objects AutoCAD // adds when it writes an older release. let twin_records = twin.map(|t| { From 181153eacbe3a1f40eed73432b0d24a973cc46db Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:09:48 -0400 Subject: [PATCH 4/4] docs(architecture): renumber the new section (7z->7bv) after the sections main added --- ARCHITECTURE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 204436f0..e046f6b6 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -2294,7 +2294,7 @@ cargo run --release --example probe_entity_field_list -- $D 0xAA8 BL,BS,BS,BD,BD cargo run --release --example probe_entity_field_list -- $D 0xAA8 2RD,2RD,BL --at=2374 # BL = 298029779 ``` -## 7z. The class table is not a census; AutoCAD's export agrees with the walk (2026-10-26, #230) +## 7bv. The class table is not a census; AutoCAD's export agrees with the walk (2026-10-26, #230) `AcDb:Classes` gives every class an instance count (DXF group 91), and `examples/probe_class_census` compares it with the walk. #56 and #76