From 5c1fa713d8c8d2e14b151f37aeb0a3a17d6ad376 Mon Sep 17 00:00:00 2001 From: Brigs Date: Fri, 7 Aug 2026 01:23:33 -0400 Subject: [PATCH] Fix MemoryError during HTML report generation Report generation loaded each artifact's entire .temphtml page into memory and rebuilt it via string concatenation to inject the navigation sidebar. For large extractions an artifact page can reach several GB, so this peaked at roughly 3x the page size and raised MemoryError. Replace the whole-file read/concat with stream_insert_sidebar_code(), which copies the page to its final location in fixed-size chunks. The sidebar placeholder is near the top of every page, so only a small head buffer is retained while searching for it (handling a placeholder split across a chunk boundary); the large table body that follows is streamed with shutil.copyfileobj. Output is byte-identical to the previous approach. Ported from iLEAPP commit 0a4aee10 (issue #1746, authored by JSup). Co-Authored-By: JSup Co-Authored-By: Claude Fable 5 --- scripts/report.py | 64 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/scripts/report.py b/scripts/report.py index dc559d4..2afd493 100755 --- a/scripts/report.py +++ b/scripts/report.py @@ -116,13 +116,11 @@ def generate_report(reportfolderbase, time_in_secs, time_hms, extraction_type, i filename = old_filename.replace(".temphtml", ".html").replace(" ", "_") # search for it in nav_list_data, then mark that one as 'active' tab active_nav_list_data = mark_item_active(nav_list_data, filename) + nav_bar_script - artifact_data = get_file_content(path) - - # Now write out entire html page for artifact - f = open(os.path.join(reportfolderbase, '_HTML', filename), 'w', encoding='utf8') - artifact_data = insert_sidebar_code(artifact_data, active_nav_list_data, path) - f.write(artifact_data) - f.close() + # Stream the (potentially very large) artifact page to its final + # location, injecting the sidebar navigation without loading the + # whole file into memory (iLEAPP issue #1746). + dest_path = os.path.join(reportfolderbase, '_HTML', filename) + stream_insert_sidebar_code(path, dest_path, active_nav_list_data) # Now delete .temphtml os.remove(path) @@ -322,15 +320,49 @@ def generate_key_val_table_without_headings(title, data_list, agency_logo_mimety return code -def insert_sidebar_code(data, sidebar_code, filename): - """Replaces the sidebar placeholder in the page HTML with the fully built navigation sidebar code.""" - pos = data.find(body_sidebar_dynamic_data_placeholder) - if pos < 0: - logfunc(f'Error, could not find {body_sidebar_dynamic_data_placeholder} in file {filename}') - return data - else: - ret = data[0: pos] + sidebar_code + data[pos + len(body_sidebar_dynamic_data_placeholder):] - return ret +def stream_insert_sidebar_code(src_path, dest_path, sidebar_code): + """Copy the artifact page from src_path to dest_path, replacing the first + sidebar placeholder with sidebar_code, without loading the whole file into + memory. + + Artifact pages can grow to several GB for large extractions, so reading an + entire page and concatenating strings (the previous approach) could exhaust + memory and raise MemoryError during report generation (iLEAPP issue #1746). + The placeholder is written near the top of every page, so only a small head + buffer is retained while searching for it; the large table body that + follows is streamed to the destination in fixed-size chunks.""" + placeholder = body_sidebar_dynamic_data_placeholder + marker_len = len(placeholder) + chunk_size = 1024 * 1024 # 1 MiB + keep = marker_len - 1 # bytes a placeholder split across a chunk could span + with open(src_path, 'r', encoding='utf8') as src, \ + open(dest_path, 'w', encoding='utf8') as dst: + buffer = '' + inserted = False + while True: + chunk = src.read(chunk_size) + if not chunk: + break + buffer += chunk + pos = buffer.find(placeholder) + if pos >= 0: + dst.write(buffer[:pos]) + dst.write(sidebar_code) + dst.write(buffer[pos + marker_len:]) + buffer = '' + inserted = True + # Copy the remainder of the (large) file in bounded chunks. + shutil.copyfileobj(src, dst, chunk_size) + break + # Placeholder not found yet: flush everything except a small tail + # that could still hold a placeholder split across the boundary. + if len(buffer) > keep: + dst.write(buffer[:-keep]) + buffer = buffer[-keep:] + if not inserted: + if buffer: + dst.write(buffer) + logfunc(f'Error, could not find {placeholder} in file {src_path}') def mark_item_active(data, itemname):