diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 263730b..d5f6d0a 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -5260,6 +5260,56 @@ cargo run --release --example probe_vp_ent_hdr -- ../dwg-corpus |---|---|---|---| | R14 | 3 | 3 | no twin | | R2000 | 3 | 3 | the DXF holds no such record | + +## 7bv. The class table is not a census; AutoCAD's export agrees with the walk (2026-10-26, #230) + +`AcDb:Classes` gives every class an instance count (DXF group 91), and +`examples/probe_class_census` compares it with the walk. #56 and #76 +showed the count over-declares on the sample corpus. The public corpus +(#225) added three more shortfalls: ACDB_TEXTOBJECTCONTEXTDATA_CLASS on +ACadSharp's R2004 and R2007 saves, and GEODATA and AcGeoBackupObject on +`geoloc.dwg`. On each, the walk is closed, with zero inter-record bytes +and no unanswered reference. + +The probe's new `--twin` mode settles them against an independent census: +AutoCAD's DXF export of the same drawing. It counts the export's records +of each class, how many of those carry a handle that is no object of the +drawing ("export-only"), and the count the export's own CLASSES section +gives: + +| Drawing | Class | DWG declares | Walked | Export records | Export-only | Export's CLASSES count | +|---|---|---:|---:|---:|---:|---:| +| `sample_AC1018.dwg` (R2004) | ACDB_TEXTOBJECTCONTEXTDATA_CLASS | 2 | 0 | 0 | 0 | 2 | +| `sample_AC1021.dwg` (R2007) | ACDB_TEXTOBJECTCONTEXTDATA_CLASS | 2 | 0 | 0 | 0 | 2 | +| `sample_AC1024/1027/1032.dwg` | ACDB_TEXTOBJECTCONTEXTDATA_CLASS | 2 | 2 | 2 | 0 | 2 | +| `geoloc.dwg` (R2018) | GEODATA | 2 | 1 | 1 | 0 | 1 | +| `geoloc.dwg` | AcGeoBackupObject | 1 | 0 | 0 | 0 | 0 | +| `sample_AC1018` … `sample_AC1032.dwg` | TABLECONTENT, TABLEGEOMETRY | 9 / 8 / 7 / 6 / 5 | 2 | 2 | 1 | 16 / 15 / 14 / 13 / 12 | + +AutoCAD's export of the R2004 and R2007 saves holds no +ACDB_TEXTOBJECTCONTEXTDATA_CLASS record, though the class tables of both +the DWG and the export count 2. `geoloc.dwg`'s export holds one GEODATA +and no AcGeoBackupObject, and its own class table counts 1 and 0 where the +DWG's counts 2 and 1. The two class tables disagree with each other, while +the export's records agree with the walk. So the count is a stale +registration figure, not a census, and the three classes join the +allowlist. + +With `--twin`, an allowlisted shortfall is excused only when the export +holds no more records of the class than the walk reached, besides its +export-only ones, and a missing twin fails the run. All 42 drawings of +the corpus that have a twin pass. The one export-only surplus is the +nextgis R2010 saves' CELLSTYLEMAP: the export gives TABLESTYLE `0x65` +an xdictionary `0xD2` holding a round-trip CELLSTYLEMAP `0xD3`. Neither +handle is an object of the DWG, and nothing in the drawing refers to a +handle it does not hold (`probe_reference_closure`). It is the same kind +of round-trip data the export adds to LTYPE `GAS_LINE` in ACadSharp's +R2004 to R2010 exports (#40). + +```sh +cargo run --release --example probe_class_census -- ../dwg-corpus/acadsharp/sample_AC1018.dwg --twin ../dwg-corpus/acadsharp/sample_AC1018_ascii.dxf +cargo run --release --example probe_class_census -- ../dwg-corpus/acadsharp/geoloc.dwg --twin ../dwg-corpus/acadsharp/geoloc.dxf +``` ## 8. Write pipeline (current scope) The inverse pipeline is partially shipped. Stage-1 (per-section diff --git a/examples/probe_class_census.rs b/examples/probe_class_census.rs index 41f72a2..17661b6 100644 --- a/examples/probe_class_census.rs +++ b/examples/probe_class_census.rs @@ -7,7 +7,7 @@ //! missing records. //! //! ```bash -//! cargo run --release --example probe_class_census -- file.dwg [--strict] +//! cargo run --release --example probe_class_census -- file.dwg [--strict] [--twin file.dxf] //! ``` //! //! Exit code is non-zero when any class is under-walked, so the probe @@ -23,10 +23,45 @@ //! the probe reports them as `ALLOWED` instead of failing; `--strict` //! disables it so the numbers stay visible, and is what CI runs over //! the canonical fixtures. +//! +//! # The DXF twin (#230) +//! +//! `--twin` takes AutoCAD's DXF export of the same drawing and adds three +//! columns: how many records of each class the export holds, how many of +//! those have a handle the drawing's handle map does not list (`export-only`), +//! and the instance count the export's own CLASSES section gives the +//! class (group 91). +//! +//! The export is an independent census of what the drawing contains, so +//! with a twin an allowlisted shortfall is excused only when the export +//! holds no more records of the class than the walk reached, not counting +//! export-only ones, and a missing, unreadable or malformed twin fails the run. +//! The handle map, not the walk, says what the drawing holds, so a record the +//! walker skipped cannot pass for export-only data and hide its own shortfall. +//! A malformed twin is one whose lines are not whole group pairs, whose group +//! codes are not numbers, whose sections do not open and close in order, that +//! lacks an ENTITIES or an OBJECTS section or its closing `0 EOF`, or that has +//! data after the EOF; an empty or truncated export would otherwise census +//! every class at zero. The crate's `dxf_reader` parses and validates the +//! grammar but sits behind a feature the CI builds of this probe do not enable. +//! Export-only records are data the export adds: on the nextgis R2010 +//! saves it gives the TABLESTYLE (`0x65`) an xdictionary `0xD2` holding a +//! round-trip CELLSTYLEMAP `0xD3`. Neither handle is an object of the +//! DWG, and `probe_reference_closure` finds no record of the drawing +//! referring to a handle it does not hold. The export also gives some +//! records handles the DWG does not use (one of the two TABLECONTENT +//! records of each ACadSharp save), so the rule bounds the export from +//! above rather than asking for equality. +//! +//! Wherever the corpus has a twin, no shortfall has the export holding +//! more records of the class than the walk reached, besides export-only +//! ones, while the two class tables disagree with each other: +//! `sample_AC1018.dwg` declares 9 TABLECONTENT, its export's table says +//! 16, and the drawing and the export both hold 2 records. use dwg::error::Result; use dwg::{DwgFile, ObjectType}; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::env; use std::process::ExitCode; @@ -69,16 +104,42 @@ use std::process::ExitCode; /// `ACAD_ROUNDTRIP_2008_TABLESTYLE_CELLSTYLEMAP` dictionary key for one /// to hang from. Run `examples/probe_reference_closure` to reproduce. /// The counts stay pinned per file by `tests/class_census.rs`. +/// +/// The public corpus (#225) added three more, each settled by AutoCAD's +/// DXF export of the same drawing (`--twin`, #230). ACadSharp's R2004 and +/// R2007 saves declare 2 ACDB_TEXTOBJECTCONTEXTDATA_CLASS and contain +/// none, and their exports hold none either, while the R2010-R2018 saves +/// and their exports hold the 2. `geoloc.dwg` declares 2 GEODATA and 1 +/// AcGeoBackupObject and holds 1 and 0; its export holds 1 and 0, and its +/// own class table counts them 1 and 0. The same twins hold exactly 2 +/// TABLECONTENT and 2 TABLEGEOMETRY records on every release from R2004. pub const ALLOWLIST: &[(&str, &str)] = &[ ( "TABLECONTENT", - "declares 5 against 2 ACAD_TABLE entities; the object stream is \ - 99.92 % covered and no handle reference is unresolved (#56)", + "declares 5 to 9 against 2 ACAD_TABLE entities; the object stream is \ + 99.92 % covered, no handle reference is unresolved (#56), and \ + AutoCAD's DXF export of each save holds 2 (#230)", ), ( "TABLEGEOMETRY", - "declares 5 against 2 ACAD_TABLE entities; the object stream is \ - 99.92 % covered and no handle reference is unresolved (#56)", + "declares 5 to 9 against 2 ACAD_TABLE entities; the object stream is \ + 99.92 % covered, no handle reference is unresolved (#56), and \ + AutoCAD's DXF export of each save holds 2 (#230)", + ), + ( + "ACDB_TEXTOBJECTCONTEXTDATA_CLASS", + "the R2004 and R2007 saves declare 2 and hold none, and so do \ + AutoCAD's DXF exports of them; the R2010+ saves and exports hold 2 (#230)", + ), + ( + "GEODATA", + "geoloc.dwg declares 2 and holds 1; AutoCAD's DXF export holds 1 \ + and its class table counts 1 (#230)", + ), + ( + "AcGeoBackupObject", + "geoloc.dwg declares 1 and holds none; AutoCAD's DXF export holds \ + none and its class table counts 0 (#230)", ), ( "DICTIONARYVAR", @@ -94,13 +155,130 @@ pub const ALLOWLIST: &[(&str, &str)] = &[ ), ]; +/// The handle of every record per DXF name in a DXF export, and the +/// instance count its CLASSES section gives each class (group 91), keyed +/// by normalized name. +#[derive(Default)] +struct Twin { + records: BTreeMap>>, + class_counts: BTreeMap, +} + +fn read_twin(path: &str) -> Result { + let bytes = std::fs::read(path).map_err(|e| e.to_string())?; + let text = String::from_utf8_lossy(&bytes); + let lines: Vec<&str> = text.lines().collect(); + if lines.len() % 2 == 1 { + return Err(format!( + "{} lines are not whole group pairs; the file is truncated", + lines.len() + )); + } + let mut twin = Twin::default(); + let mut class_name: Option = None; + let mut in_class = false; + let mut current: Option = None; + let mut open_section: Option> = None; + let mut sections = BTreeSet::new(); + let mut saw_eof = false; + for (n, pair) in lines.chunks_exact(2).enumerate() { + let line = 2 * n + 1; + let (code, value) = (pair[0].trim(), pair[1].trim()); + let numeric: i64 = code + .parse() + .map_err(|_| format!("line {line}: group code {code:?} is not a number"))?; + if saw_eof { + return Err(format!("line {line}: data after EOF")); + } + if numeric == 999 { + continue; + } + if numeric != 0 { + match &mut open_section { + Some(name @ None) if numeric == 2 => { + sections.insert(normalize(value)); + *name = Some(normalize(value)); + } + Some(None) => return Err(format!("line {line}: SECTION is not followed by 2")), + None => return Err(format!("line {line}: group {numeric} outside any section")), + Some(Some(_)) => {} + } + } else { + match (value, &open_section) { + ("SECTION", None) => open_section = Some(None), + ("SECTION", Some(_)) => { + return Err(format!("line {line}: SECTION inside a section")); + } + (_, Some(None)) => { + return Err(format!("line {line}: SECTION is not followed by 2")); + } + ("ENDSEC", Some(Some(_))) => open_section = None, + ("ENDSEC", None) => { + return Err(format!("line {line}: ENDSEC with no open section")); + } + ("EOF", None) => saw_eof = true, + ("EOF", Some(_)) => return Err(format!("line {line}: EOF inside a section")), + (_, Some(Some(_))) => {} + (_, None) => return Err(format!("line {line}: {value} outside any section")), + } + in_class = value == "CLASS"; + class_name = None; + let name = normalize(value); + twin.records.entry(name.clone()).or_default().push(None); + current = Some(name); + continue; + } + if code == "5" || code == "105" { + // The record's own handle is its first `5` (`105` on DIMSTYLE). + if let Some(slot) = current + .as_ref() + .and_then(|n| twin.records.get_mut(n)) + .and_then(|v| v.last_mut()) + .filter(|h| h.is_none()) + { + *slot = u64::from_str_radix(value, 16).ok(); + } + } else if in_class && code == "1" { + class_name = Some(normalize(value)); + } else if in_class && code == "91" { + if let (Some(name), Ok(n)) = (class_name.as_ref(), value.parse()) { + twin.class_counts.insert(name.clone(), n); + } + } + } + if !saw_eof { + return Err("no closing EOF; the file is empty or truncated".to_string()); + } + for needed in ["ENTITIES", "OBJECTS"] { + if !sections.contains(needed) { + return Err(format!("no {needed} section")); + } + } + Ok(twin) +} + fn main() -> ExitCode { - let Some(path) = env::args().nth(1) else { - eprintln!("usage: probe_class_census [--strict]"); + let args: Vec = env::args().skip(1).collect(); + let Some(path) = args.first().cloned() else { + eprintln!("usage: probe_class_census [--strict] [--twin ]"); return ExitCode::FAILURE; }; - let strict = env::args().any(|a| a == "--strict"); - match run(&path, strict) { + let strict = args.iter().any(|a| a == "--strict"); + let twin = match args.iter().position(|a| a == "--twin") { + Some(i) => match args.get(i + 1).map(|p| read_twin(p)) { + Some(Ok(twin)) => Some(twin), + Some(Err(e)) => { + eprintln!("cannot read the DXF twin: {e}"); + return ExitCode::FAILURE; + } + None => { + eprintln!("--twin needs a DXF path"); + return ExitCode::FAILURE; + } + }, + None => None, + }; + match run(&path, strict, twin.as_ref()) { Ok(true) => ExitCode::SUCCESS, Ok(false) => ExitCode::FAILURE, Err(e) => { @@ -110,7 +288,7 @@ fn main() -> ExitCode { } } -fn run(path: &str, strict: bool) -> Result { +fn run(path: &str, strict: bool, twin: Option<&Twin>) -> Result { let file = DwgFile::open(path)?; println!("file : {path}"); println!("version : {}", file.version()); @@ -225,11 +403,21 @@ fn run(path: &str, strict: bool) -> Result { ); } println!(); + let twin_header = if twin.is_some() { + format!(" {:>6} {:>11} {:>8}", "twin", "export-only", "twin 91") + } else { + String::new() + }; + let Some(handle_map) = file.handle_map().transpose()? else { + println!("no handle map to tell the export's records from the drawing's"); + return Ok(false); + }; + let in_drawing: std::collections::HashSet = handle_map.iter().map(|e| e.handle).collect(); println!( - "{:<38} {:>6} {:>9} {:>7} {:>7} {:>7}", + "{:<38} {:>6} {:>9} {:>7} {:>7} {:>7}{twin_header}", "class", "code", "declared", "walked", "fixed", "delta" ); - println!("{}", "-".repeat(80)); + println!("{}", "-".repeat(if twin.is_some() { 108 } else { 80 })); let mut complete = true; let mut allowed = Vec::new(); @@ -244,19 +432,45 @@ fn run(path: &str, strict: bool) -> Result { continue; } let delta = (seen + fixed) as i64 - declared as i64; + let key = normalize(&def.dxf_class_name); + // The export's records of this class, and how many of them have a + // handle the drawing's handle map does not list: data the export writes + // and the DWG does not hold, such as the round-trip objects AutoCAD + // adds when it writes an older release. + let twin_records = twin.map(|t| { + let handles = t.records.get(&key).map(Vec::as_slice).unwrap_or(&[]); + let export_only = handles + .iter() + .filter(|h| h.is_some_and(|h| !in_drawing.contains(&h))) + .count(); + (handles.len(), export_only) + }); + // With a twin, a shortfall is excused only when the export holds no + // more records of the class than the walk reached, besides its + // export-only ones: the allowlist names the classes, and the twin + // has to agree for this file. let excused = !strict && delta < 0 - && ALLOWLIST - .iter() - .any(|(name, _)| normalize(name) == normalize(&def.dxf_class_name)); + && twin_records.is_none_or(|(n, export_only)| n - export_only <= seen + fixed) + && ALLOWLIST.iter().any(|(name, _)| normalize(name) == key); if delta < 0 && !excused { complete = false; } if excused { allowed.push(def.dxf_class_name.clone()); } + let twin_cells = match (twin, twin_records) { + (Some(t), Some((records, export_only))) => { + let counted = t + .class_counts + .get(&key) + .map_or_else(|| "-".to_string(), |n| n.to_string()); + format!(" {records:>6} {export_only:>11} {counted:>8}") + } + _ => String::new(), + }; println!( - "{:<38} {:>6} {:>9} {:>7} {:>7} {:>+7} {}", + "{:<38} {:>6} {:>9} {:>7} {:>7} {:>+7}{twin_cells} {}", def.dxf_class_name, def.class_number, declared,