From 88e6cca7dd9ef43a6385bf062a86aae6091e507c Mon Sep 17 00:00:00 2001 From: warpmaster <32202890+jared-perry@users.noreply.github.com> Date: Tue, 21 Jul 2026 01:54:47 -0400 Subject: [PATCH] feat: wire offsetscan yara subcommand (schema-parity with Invoke-OffsetYaraScan); refactor PE assembly; fix strict-path overlay --- CHANGELOG.md | 32 ++++++++++++++- Cargo.lock | 58 +++++++++++++-------------- Cargo.toml | 7 +++- src/main.rs | 49 +++++++++++++++++++++++ src/pe.rs | 96 ++++++++++++++++++++++++-------------------- src/yara_scan.rs | 102 +++++++++++++++++++++++++++++++++++------------ 6 files changed, 242 insertions(+), 102 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd2c563..7c22744 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,36 @@ All notable changes to OffsetScan are documented in this file. The project follows semantic versioning. +## [0.2.0] - 2026-07-21 + +### Added + +- **`offsetscan yara` subcommand** (feature-gated behind `yara-scan`, off by default). + Matches files against YARA rule files and emits one record per matched string — + `File`/`Rule`/`StringId`/`Offset`/`OffsetHex`/`Data` — **verified byte-identical to + OffsetInspect's `Invoke-OffsetYaraScan`** on the same rules and sample. Supports multiple + `--rules` files, `--recurse`, a `--timeout`, and `--ndjson`. A default build (no feature) + exposes the subcommand but returns a clear rebuild instruction. Schema field names are + locked by a unit test that runs even in the default build. + +### Changed + +- Bumped the `yara` dependency `0.28 -> 0.32` and enabled its `vendored` feature. + `yara-sys 0.28` no longer builds against current `libclang` (bindgen emits a different + anonymous-union layout); 0.32 builds cleanly and vendors `libyara` so no system YARA + install is needed (a C toolchain and `libclang` are still required at build time). The + feature is still excluded from CI, so verify the feature build locally after dep bumps. +- Unified the strict (goblin) and lenient (salvage) PE parse paths onto a single + `assemble_pe_info` builder, removing duplicated struct-assembly and overlay logic. Output + verified unchanged across the 150-file valid corpus and the 34-file malformed corpus. + +### Fixed + +- The strict PE path computed the overlay boundary **without** filtering out zero-raw-size + sections, unlike OffsetInspect's `Get-OIPEOverlayRange` and the lenient path — a latent + inconsistency that could misplace the overlay for PEs with a zero-raw-size section + (e.g. some packers' virtual sections). Both paths now use the filtered computation. + ## [0.1.4] - 2026-07-21 ### Fixed @@ -16,7 +46,7 @@ The project follows semantic versioning. reachable imports/imphash. Valid PEs are unaffected — they always take the goblin path, verified unchanged across a 150-file corpus vs pefile. Across a 34-file malformed corpus (truncations, bit-flips, corrupted directories, bogus section counts, pure garbage), - `IsPE` now agrees with OffsetInspect on all 34 (was 22/34), and neither engine crashes or + `IsPE` now agrees with OffsetInspect on all 34 (was 22/34), and neither engine crashes nor hangs on any input. Genuinely unparseable inputs (no MZ/PE signature, or a section count that cannot fit) are still rejected, matching OffsetInspect. diff --git a/Cargo.lock b/Cargo.lock index 0b19ae6..8da053c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -78,16 +78,14 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "bindgen" -version = "0.68.1" +version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ "bitflags", "cexpr", "clang-sys", - "lazy_static", - "lazycell", - "peeking_take_while", + "itertools", "proc-macro2", "quote", "regex", @@ -306,6 +304,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures-core" version = "0.3.33" @@ -393,6 +397,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -410,18 +423,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "libc" version = "0.2.186" @@ -487,7 +488,7 @@ dependencies = [ [[package]] name = "offsetscan" -version = "0.1.4" +version = "0.2.0" dependencies = [ "chrono", "clap", @@ -516,12 +517,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "pin-project-lite" version = "0.2.17" @@ -603,9 +598,9 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "rustc-hash" -version = "1.1.0" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustversion" @@ -926,9 +921,9 @@ dependencies = [ [[package]] name = "yara" -version = "0.28.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "694bf8660ab8f38b432f114e34f0b2eb0f945da0afc93cb1e985fe63b96d1ea9" +checksum = "b3caead5c2f89f36a59da301b0ad42e6c096b957db0b4adc60e05aae74c2c3ac" dependencies = [ "bitflags", "thiserror", @@ -937,11 +932,14 @@ dependencies = [ [[package]] name = "yara-sys" -version = "0.28.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "694e232eb98af91a4a9575698dc7a54aa60fc4f69dddade6d18251413bf38ee5" +checksum = "ba8598e7b8b80190eddb1e030e87e4051f2fec79df901748a9257a4a59fcc3e5" dependencies = [ "bindgen", + "cc", + "fs_extra", + "glob", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fb8122a..a6a1efd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "offsetscan" -version = "0.1.4" +version = "0.2.0" edition = "2021" authors = ["DreadHost Research"] description = "Standalone native corpus-scale engine for PE parsing, entropy, string extraction, and IOC panels — schema-compatible with OffsetInspect.Result/ThreatScanResult JSON output." @@ -37,8 +37,11 @@ csv = "1" chrono = { version = "0.4", features = ["serde"] } [dependencies.yara] -version = "0.28" +version = "0.32" optional = true +# Build a bundled libyara from source so the feature needs no system YARA install +# (still requires a C toolchain + libclang for bindgen at build time). +features = ["vendored"] [features] default = [] diff --git a/src/main.rs b/src/main.rs index 061b942..f629e8b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -69,6 +69,19 @@ enum Commands { #[arg(long)] recurse: bool, }, + /// Match files against YARA rules; one record per matched string with its offset. + /// Requires a build with `--features yara-scan`. + Yara { + path: String, + /// One or more YARA rule files. + #[arg(long, required = true)] + rules: Vec, + #[arg(long)] + recurse: bool, + /// Per-file scan timeout in seconds. + #[arg(long, default_value_t = 60)] + timeout: i32, + }, } /// Parse a byte offset given as decimal or `0x`-prefixed hexadecimal. @@ -185,6 +198,42 @@ fn main() { }); } } + Commands::Yara { + path, + rules, + recurse, + timeout, + } => { + let files = expand_paths(&path, recurse); + // Rules are recompiled per file (simple and correct); acceptable for the + // interactive/small-corpus use this subcommand targets. + let mut all_hits = Vec::new(); + let mut had_error = false; + for f in &files { + match yara_scan::scan_with_rules(&f.to_string_lossy(), &rules, timeout) { + Ok(hits) => all_hits.extend(hits), + Err(e) => { + eprintln!("offsetscan yara: {}: {e}", f.display()); + had_error = true; + } + } + } + if ndjson { + for hit in &all_hits { + if let Ok(line) = serde_json::to_string(hit) { + println!("{line}"); + } + } + } else { + match serde_json::to_string_pretty(&all_hits) { + Ok(text) => println!("{text}"), + Err(e) => eprintln!("offsetscan yara: serialization failed: {e}"), + } + } + if had_error && all_hits.is_empty() { + std::process::exit(1); + } + } } } diff --git a/src/pe.rs b/src/pe.rs index 0eabfda..07cced7 100644 --- a/src/pe.rs +++ b/src/pe.rs @@ -111,8 +111,7 @@ fn parse_pe_strict(data: &[u8], file_path: &str) -> Result { Some(format!("{:x}", hasher.finalize())) }; - // Resource-directory size (PE data directory index 2). VERIFY goblin API surface: - // recent goblin exposes optional_header.data_directories.get_resource_table(). + // Resource-directory size (PE data directory index 2). let resource_size = pe .header .optional_header @@ -121,30 +120,54 @@ fn parse_pe_strict(data: &[u8], file_path: &str) -> Result { .map(|d| d.size) .unwrap_or(0); - // Overlay = bytes appended after the last section's raw data. - let last_section_end = pe - .sections + Ok(assemble_pe_info( + file_path, + data.len() as u64, + machine, + pe.is_64, + pe.entry as u32, + pe.image_base as u64, + sections, + imports, + imp_hash, + resource_size, + )) +} + +/// Assemble the final `PeInfo`, computing the overlay identically for both parse paths. +/// The overlay boundary uses only sections with non-zero raw data, matching OffsetInspect's +/// `Get-OIPEOverlayRange` — a zero-raw-size section (e.g. a packer's virtual section) never +/// defines it. (The strict path previously omitted this filter, a latent inconsistency.) +#[allow(clippy::too_many_arguments)] +fn assemble_pe_info( + file_path: &str, + file_len: u64, + machine: String, + is_pe32_plus: bool, + entry_point_rva: u32, + image_base: u64, + sections: Vec
, + imports: Vec, + imp_hash: Option, + resource_size: u32, +) -> PeInfo { + let last_section_end = sections .iter() - .map(|s| (s.pointer_to_raw_data as u64) + (s.size_of_raw_data as u64)) + .filter(|s| s.size_of_raw_data > 0) + .map(|s| s.pointer_to_raw_data as u64 + s.size_of_raw_data as u64) .max() .unwrap_or(0); - let file_len = data.len() as u64; let overlay_size = file_len.saturating_sub(last_section_end); let has_overlay = overlay_size > 0; - let overlay_offset = if has_overlay { - Some(last_section_end) - } else { - None - }; - Ok(PeInfo { + PeInfo { file: file_path.to_string(), file_size: file_len, machine, - is_pe32_plus: pe.is_64, - entry_point_rva: pe.entry as u32, - entry_point_hex: format!("0x{:X}", pe.entry), - image_base: pe.image_base as u64, + is_pe32_plus, + entry_point_rva, + entry_point_hex: format!("0x{:X}", entry_point_rva), + image_base, section_count: sections.len() as u32, imported_dll_count: imports.len() as u32, sections, @@ -152,12 +175,16 @@ fn parse_pe_strict(data: &[u8], file_path: &str) -> Result { imp_hash, resource_size, has_overlay, - overlay_offset, + overlay_offset: if has_overlay { + Some(last_section_end) + } else { + None + }, overlay_size, - mapped_offset: None, // populated only when a caller passes an explicit offset + mapped_offset: None, mapped_section: None, warnings: Vec::new(), - }) + } } fn rd_u16(d: &[u8], o: usize) -> Option { @@ -267,37 +294,18 @@ fn parse_pe_lenient(data: &[u8], file_path: &str) -> Result { let (imports, imp_hash) = lenient_imports(data, §ions, import_rva as u64, is_pe32_plus); - let last_end = sections - .iter() - .filter(|s| s.size_of_raw_data > 0) - .map(|s| s.pointer_to_raw_data as u64 + s.size_of_raw_data as u64) - .max() - .unwrap_or(0); - let file_len = data.len() as u64; - let overlay_size = file_len.saturating_sub(last_end); - let has_overlay = overlay_size > 0; - - Ok(PeInfo { - file: file_path.to_string(), - file_size: file_len, - machine: machine_name(machine_id), + Ok(assemble_pe_info( + file_path, + data.len() as u64, + machine_name(machine_id), is_pe32_plus, entry_point_rva, - entry_point_hex: format!("0x{:X}", entry_point_rva), image_base, - section_count: sections.len() as u32, - imported_dll_count: imports.len() as u32, sections, imports, imp_hash, resource_size, - has_overlay, - overlay_offset: if has_overlay { Some(last_end) } else { None }, - overlay_size, - mapped_offset: None, - mapped_section: None, - warnings: Vec::new(), - }) + )) } /// Best-effort import walk for the lenient path, mirroring `Get-OIPEImport` (including the diff --git a/src/yara_scan.rs b/src/yara_scan.rs index 374842d..97c3ca2 100644 --- a/src/yara_scan.rs +++ b/src/yara_scan.rs @@ -1,49 +1,71 @@ -//! Optional YARA rule matching — mirrors `Invoke-OffsetYaraScan`. -//! Only compiled in when the `yara-scan` feature is enabled, matching -//! OffsetInspect's stance that YARA is the one optional external dependency. +//! Optional YARA rule matching — mirrors `Invoke-OffsetYaraScan` +//! (OffsetInspect.YaraMatch). Compiled in only with the `yara-scan` feature, +//! which vendors libyara; the default build omits it entirely. +//! +//! Parity notes vs OffsetInspect's `Invoke-OffsetYaraScan`: +//! - One record per matched string with fields `File`, `Rule`, `StringId` (e.g. `$a`), +//! `Offset` (decimal), `OffsetHex` (lowercase `0x..`), `Data` (matched bytes). +//! - `Data` is decoded lossily from the matched bytes; for non-printable/binary matches +//! the textual form can differ from the YARA CLI's rendering that OffsetInspect parses. -#[cfg(feature = "yara-scan")] use serde::{Deserialize, Serialize}; -#[cfg(feature = "yara-scan")] #[derive(Debug, Serialize, Deserialize, Clone)] pub struct YaraHit { #[serde(rename = "File")] pub file: String, - #[serde(rename = "RuleName")] - pub rule_name: String, + #[serde(rename = "Rule")] + pub rule: String, + #[serde(rename = "StringId")] + pub string_id: String, #[serde(rename = "Offset")] pub offset: u64, - #[serde(rename = "Identifier")] - pub identifier: String, + #[serde(rename = "OffsetHex")] + pub offset_hex: String, + #[serde(rename = "Data")] + pub data: String, } +/// Compile one or more rule files and scan a single file, returning one `YaraHit` per +/// matched string. `timeout_secs` bounds the scan. #[cfg(feature = "yara-scan")] -#[allow(dead_code)] // wired to a CLI subcommand only once YARA integration lands -pub fn scan_with_rules(file_path: &str, rule_path: &str) -> Result, String> { +pub fn scan_with_rules( + file_path: &str, + rule_paths: &[String], + timeout_secs: i32, +) -> Result, String> { use yara::Compiler; - let compiler = Compiler::new() - .map_err(|e| format!("YARA compiler init failed: {e}"))? - .add_rules_file(rule_path) - .map_err(|e| format!("failed to load rules '{rule_path}': {e}"))?; + let mut compiler = Compiler::new().map_err(|e| format!("YARA compiler init failed: {e}"))?; + for rule_path in rule_paths { + compiler = compiler + .add_rules_file(rule_path) + .map_err(|e| format!("failed to load rules '{rule_path}': {e}"))?; + } let rules = compiler .compile_rules() .map_err(|e| format!("rule compilation failed: {e}"))?; - - let results = rules - .scan_file(file_path, 30) + let matches = rules + .scan_file(file_path, timeout_secs) .map_err(|e| format!("scan failed: {e}"))?; let mut hits = Vec::new(); - for rule_match in results { + for rule_match in matches { for string_match in rule_match.strings { + // OffsetInspect reports the id with a leading '$' (e.g. "$a"). + let string_id = if string_match.identifier.starts_with('$') { + string_match.identifier.to_string() + } else { + format!("${}", string_match.identifier) + }; for m in string_match.matches { hits.push(YaraHit { file: file_path.to_string(), - rule_name: rule_match.identifier.to_string(), + rule: rule_match.identifier.to_string(), + string_id: string_id.clone(), offset: m.offset as u64, - identifier: string_match.identifier.to_string(), + offset_hex: format!("0x{:x}", m.offset), + data: String::from_utf8_lossy(&m.data).to_string(), }); } } @@ -51,13 +73,43 @@ pub fn scan_with_rules(file_path: &str, rule_path: &str) -> Result, Ok(hits) } +/// Stub for builds without the `yara-scan` feature: returns a clear rebuild instruction so +/// the `yara` subcommand exists but explains what's missing. #[cfg(not(feature = "yara-scan"))] -#[allow(dead_code)] // wired to a CLI subcommand only once YARA integration lands -pub fn scan_with_rules(_file_path: &str, _rule_path: &str) -> Result, String> { +pub fn scan_with_rules( + _file_path: &str, + _rule_paths: &[String], + _timeout_secs: i32, +) -> Result, String> { Err( "offsetscan was built without the `yara-scan` feature; rebuild with \ - `cargo build --release --features yara-scan` and ensure the YARA \ - engine is installed." + `cargo build --release --features yara-scan` (needs a C toolchain and libclang)." .to_string(), ) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn yara_hit_serializes_with_offsetinspect_field_names() { + // Locks the interchange field names against Invoke-OffsetYaraScan's records, even in + // the default build where the scanner itself isn't compiled. + let hit = YaraHit { + file: "f".to_string(), + rule: "R".to_string(), + string_id: "$a".to_string(), + offset: 100, + offset_hex: "0x64".to_string(), + data: "x".to_string(), + }; + let json = serde_json::to_string(&hit).unwrap(); + for field in ["File", "Rule", "StringId", "Offset", "OffsetHex", "Data"] { + assert!( + json.contains(&format!("\"{field}\"")), + "missing field {field}" + ); + } + } +}