diff --git a/application/single_app/config.py b/application/single_app/config.py index aaf7ab33f..14e7f949f 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -101,7 +101,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.313" +VERSION = "0.261.314" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_document_analysis_results.py b/application/single_app/functions_document_analysis_results.py index abdb31f25..72302747a 100644 --- a/application/single_app/functions_document_analysis_results.py +++ b/application/single_app/functions_document_analysis_results.py @@ -584,21 +584,17 @@ def _locate_analysis_evidence(passage, contents, forms): quote = passage.get('quote') if not isinstance(quote, str) or not quote.strip(): return None, 'missing_quote' - label = _EVIDENCE_PROMPT_LABEL.match(quote) - needles = { - tier: form[0] - for tier, form in _evidence_forms(quote[label.end():] if label else quote).items() - } - if not needles['normalized']: - return None, 'missing_quote' if not any(passage.get(key) is not None for key in _EVIDENCE_SELECTORS): return None, 'missing_location' + needles = {} def occurrence(index, tier): text = contents[index]['text'] if tier == 'exact': offset = text.find(quote) return (offset, offset + len(quote)) if offset >= 0 else None + if not needles[tier]: + return None if index not in forms: forms[index] = _evidence_forms(text) return _evidence_occurrence(forms[index][tier], needles[tier], text) @@ -611,6 +607,13 @@ def occurrence(index, tier): ) } for tier in EVIDENCE_MATCH_TIERS: + if tier == 'normalized': + # Normalization may erase literal source content, such as extraction comments. + label = _EVIDENCE_PROMPT_LABEL.match(quote) + needles = { + name: form[0] + for name, form in _evidence_forms(quote[label.end():] if label else quote).items() + } hits = [ (index, span) for index in sorted(selected) for span in [occurrence(index, tier)] if span is not None @@ -627,7 +630,7 @@ def occurrence(index, tier): for tier in EVIDENCE_MATCH_TIERS ): return None, 'not_in_cited_location' - return None, 'not_in_window' + return None, 'not_in_window' if needles['normalized'] else 'missing_quote' def collect_analysis_window_candidates(analysis_text, source, work_unit, window_payload): diff --git a/docs/explanation/fixes/DOCUMENT_ANALYSIS_LITERAL_EVIDENCE_FIX.md b/docs/explanation/fixes/DOCUMENT_ANALYSIS_LITERAL_EVIDENCE_FIX.md new file mode 100644 index 000000000..0026c2446 --- /dev/null +++ b/docs/explanation/fixes/DOCUMENT_ANALYSIS_LITERAL_EVIDENCE_FIX.md @@ -0,0 +1,106 @@ +# Document analysis literal evidence fix + +**Version: 0.261.314** + +Fixed in version: **0.261.314**, recorded in +`application/single_app/config.py`. + +## Issue and root cause + +A document-derived report could stop before drafting even after every source +window was read. An evidence quote existed verbatim in its cited chunk, but the +shared validator removed its contents during presentation normalization before +trying exact matching. HTML comments, including Document Intelligence extraction +annotations such as page footers, normalize to an empty string. + +The investigated run read all 20 pages in four windows. One finding had three +matched citations and a fourth quoting a source footer marked "Test Data Only". +The validator classified that fourth quote as `missing_quote`, rejected the +finding, and retained only 24 of 25 findings. Compose then refused the partial +input; Word rendering was never reached. + +This was a source-representation problem, not a missing document, an inability +to read the PDF, or a Word renderer failure. + +## Shared exact-first matching + +`functions_document_analysis_results.py::_locate_analysis_evidence` now checks +the literal quote against the original cited chunks before computing normalized +forms. A source-backed comment, markup fragment, code example or extraction +annotation follows the same exact-match rule as ordinary prose. There is no +footer-specific exception or list of approved annotation formats. + +If exact matching fails, the existing normalized and case-folded comparisons +remain available. Empty normalized needles are skipped so they cannot match +unrelated text. Evidence retains the original source text and character offsets. +The existing result contract, matching tiers and `evidence-matcher-v2` identifier +are unchanged. + +The shared collector serves explicit narrative Analyze in chat, workflows and +orchestration. Ordinary search/RAG chat is unchanged. The fix introduces no model +calls, additional retries, settings, routes or source reprocessing. + +## Safeguards and limitations + +A quote must still be a nonblank string with a source-location selector. +All supplied selectors must identify the cited chunk. A quote found only in +another chunk fails; multiple matching cited chunks remain ambiguous. +Normalized matching retains its existing word, number and sign protections. +Other unsupported citations, conflicting findings and incomplete source coverage +still prevent complete-only composition. Findings or failed citations are not +silently discarded to complete a report. + +Locating a quote proves that the passage exists in the cited source, not that the +model's entire finding is factually correct or logically entailed by it. This +fix corrects the locator's unnecessary representation restriction; it does not +add a factual-accuracy evaluation or relax the completeness contract. + +Old saved attempts and their recovery behavior are unchanged. After deploying +this version, test with a fresh request; this change does not repair or migrate +existing failed runs. + +## Files and regression coverage + +- `application/single_app/functions_document_analysis_results.py`: exact-first + matching and an empty-normalized-needle guard. +- `functional_tests/test_document_analysis_evidence_matching.py`: synthetic + source comments, headers, footers, literal markup and label text; original + offsets; missing, wrong-location and ambiguous citations; empty fallbacks; + and real Analyze finalization without repair calls. +- `functional_tests/test_orchestration_document_derivation_reliability.py`: + fresh Analyze to Compose to Word execution, durable publication, and actual + downloaded Word content, using the existing offline harness. +- `application/single_app/config.py`: application patch version update. + +The existing evidence-matching documentation also links to this correction. +Private production source chunks and model responses are not regression fixtures +and are not committed to the repository. + +## Validation + +The unchanged saved model responses were replayed locally against the original +20 source chunks. Their recomputed fingerprint matched the failed run. + +| Replay | Accepted findings | Unresolved | Status | +| --- | --- | --- | --- | +| Before the fix | 24 | 1 | Partial | +| After the fix, with no response or source edits | 25 | 0 | Valid | + +Normal and optimized Python both produced the fixed result. Replay did not invoke +a model, update a saved result or write to Azure. + +Local offline validation: + +| Coverage | Result | +| --- | --- | +| Evidence matching, final results and document derivation suites | 201 tests passed | +| Saved analysis, workflow durability and execution-fence compatibility suites | 74 tests passed | +| New Word publication cases | Both completed with two extraction calls and one composition call; no correction calls | +| Documentation surface coverage and site quality | 7/7 and 6/6 checks passed; surface inventory unchanged | +| `git diff --check` | Passed | + +Tests used an isolated environment with repository-pinned Flask/Werkzeug and +compatible cryptography to avoid the machine's existing OpenSSL import failure. +Global packages and application dependency manifests were not changed. + +This validation is not a production deployment or live model replay. diff --git a/docs/explanation/fixes/V2_COMPARISON_ANALYZE_EVIDENCE_MATCHING_FIX.md b/docs/explanation/fixes/V2_COMPARISON_ANALYZE_EVIDENCE_MATCHING_FIX.md index b680ec2b0..446e502cb 100644 --- a/docs/explanation/fixes/V2_COMPARISON_ANALYZE_EVIDENCE_MATCHING_FIX.md +++ b/docs/explanation/fixes/V2_COMPARISON_ANALYZE_EVIDENCE_MATCHING_FIX.md @@ -100,6 +100,12 @@ tier with a match: differences are removed. 3. `normalized_casefold`: the same comparison, ignoring case. +As of **0.261.314**, exact matching runs before any normalization. A quote that +exists verbatim in its cited chunk remains evidence even if it consists of source +comments, markup or an extraction annotation. Normalization is only a fallback; +an empty normalized quote cannot match arbitrary source text. See the +[literal evidence fix](DOCUMENT_ANALYSIS_LITERAL_EVIDENCE_FIX.md). + The normalized comparison keeps every word and number separate, and drops only what renders as nothing: diff --git a/functional_tests/test_document_analysis_evidence_matching.py b/functional_tests/test_document_analysis_evidence_matching.py index 723ffa46a..b56ddc91e 100644 --- a/functional_tests/test_document_analysis_evidence_matching.py +++ b/functional_tests/test_document_analysis_evidence_matching.py @@ -1,9 +1,11 @@ # test_document_analysis_evidence_matching.py """ Functional tests for general Analyze evidence matching, caveats, notes and validation logging. -Version: 0.261.191 +Version: 0.261.314 Implemented in: 0.261.191 +Literal source matching precedes lossy normalization as of 0.261.314. + Refs #1540. A model quoting table-heavy or formatted source text must be located in its original chunk when the only differences are presentation: markup, table rules, entities, typographic quotes and dashes, compatibility forms, invisible characters, the amount of @@ -117,6 +119,10 @@ def test_evidence_matching_fix_is_in_the_application_version(): assert_app_version_at_least('0.261.191') +def test_literal_evidence_fix_is_in_the_application_version(): + assert_app_version_at_least('0.261.314') + + @pytest.mark.parametrize('chunk_text,quote,expected_text,tier', [ pytest.param( '
RegionRevenue
North$1,200
', @@ -233,6 +239,76 @@ def test_presentation_only_differences_locate_the_verbatim_source_span(chunk_tex assert result['evidence_matching'] == match_counts(**{tier: 1}) +@pytest.mark.parametrize('quote', [ + 'Test Data Only', + '', + '', + '', + '', + '', + '[Page 1, Chunk 1]', + '---', +]) +def test_literal_source_content_is_matched_before_normalization(quote, monkeypatch): + text = f'Source begins here.\n{quote}\nSource ends here.' + with document_analysis_runtime({}) as runtime: + def normalization_must_not_run(*args): + raise AssertionError('An exact source match must not depend on presentation normalization.') + + monkeypatch.setattr(runtime.results, '_evidence_forms', normalization_must_not_run) + result = collect(runtime, window_chunks(text), supported_finding({'chunk_sequence': 1, 'quote': quote})) + [candidate] = result['candidates'] + assert candidate['status'] == 'candidate' + assert candidate['issues'] == [] + [passage] = result['evidence'] + location = passage['location'] + assert passage['text'] == quote + assert (location['start_char'], location['end_char']) == (text.index(quote), text.index(quote) + len(quote)) + assert location['match'] == 'exact' + assert candidate['evidence_refs'] == [passage['evidence_id']] + assert result['evidence_matching'] == match_counts(exact=1) + + +@pytest.mark.parametrize('quote', ['', '']) +@pytest.mark.parametrize('selectors,reason', [ + ({}, 'missing_location'), + ({'chunk_sequence': 9}, 'not_in_cited_location'), + ({'chunk_sequence': 2, 'page_number': 2}, 'not_in_cited_location'), + ({'page_number': 1}, 'ambiguous'), +]) +def test_literal_source_content_still_requires_an_unambiguous_cited_chunk(quote, selectors, reason): + with document_analysis_runtime({}) as runtime: + result = collect( + runtime, window_chunks(quote, quote, pages=[1, 1]), + supported_finding({**selectors, 'quote': quote}), + ) + [candidate] = result['candidates'] + assert candidate['status'] == 'unresolved' + assert candidate['evidence_refs'] == [] + assert result['evidence'] == [] + assert [issue.get('reason') for issue in candidate['issues']] == [reason] + assert result['evidence_matching'] == match_counts(**{reason: 1}) + + +@pytest.mark.parametrize('quote', [ + '', + '', + '[Page 1, Chunk 1]', + '---', +]) +def test_empty_normalized_quotes_cannot_match_unrelated_source_text(quote): + with document_analysis_runtime({}) as runtime: + result = collect( + runtime, window_chunks('Ordinary source text.'), + supported_finding({'chunk_sequence': 1, 'quote': quote}), + ) + [candidate] = result['candidates'] + assert candidate['status'] == 'unresolved' + assert candidate['evidence_refs'] == [] + assert result['evidence'] == [] + assert result['evidence_matching'] == match_counts(missing_quote=1) + + @pytest.mark.parametrize('passage,reason', [ pytest.param({'chunk_sequence': 3, 'quote': 'Early cancellation carries a penalty.'}, 'not_in_window', id='paraphrase'), pytest.param({'chunk_sequence': 3, 'quote': 'An exit penalty ... cancellation.'}, 'not_in_window', id='ellipsis'), @@ -248,6 +324,8 @@ def test_presentation_only_differences_locate_the_verbatim_source_span(chunk_tex pytest.param({'quote': 'sole supplier'}, 'missing_location', id='no_selector'), pytest.param({'chunk_sequence': 1, 'quote': ' '}, 'missing_quote', id='blank_quote'), pytest.param({'chunk_sequence': 1}, 'missing_quote', id='no_quote'), + pytest.param({'chunk_sequence': 1, 'quote': None}, 'missing_quote', id='null_quote'), + pytest.param({'chunk_sequence': 1, 'quote': ['sole supplier']}, 'missing_quote', id='non_text_quote'), pytest.param({'chunk_sequence': 1, 'quote': ''}, 'missing_quote', id='markup_only_quote'), pytest.param({'chunk_sequence': 1, 'quote': '[Page 1, Chunk 1]'}, 'missing_quote', id='label_only_quote'), pytest.param('sole supplier', 'missing_quote', id='passage_is_not_an_object'), @@ -355,6 +433,37 @@ def test_exact_offsets_are_unchanged_and_one_unsupported_passage_still_unresolve assert result['evidence_matching'] == match_counts(exact=1, normalized_casefold=1, not_in_window=1) +@pytest.mark.parametrize('annotation', [ + 'Example report - Test Data Only', + '', + '', +]) +def test_source_annotation_and_ordinary_citations_finalize_without_repair_calls(annotation): + source = 'The service owner is Mira.' + documents = {'example': original_document('example', [f'{source}\n{annotation}'])} + + def response(prompt): + payload = json.loads(extract_fixture_findings(prompt)) + payload['findings'][0]['evidence'].append({'chunk_sequence': 1, 'quote': annotation}) + return json.dumps(payload) + + with document_analysis_runtime(documents) as runtime: + result, client = run_analysis( + runtime, documents, FixtureAnalysisClient(response), max_retries_per_window=1, + ) + assert len(client.calls) == 1 + assert result['coverage']['retries'] == 0 + validation = result['analysis_validation'] + assert validation['status'] == 'valid' + assert validation['issues'] == [] + assert validation['unresolved_candidate_count'] == 0 + [record] = result['authoritative_result']['value'] + assert record['values'] == {'owner': 'Mira'} + assert len(record['evidence_refs']) == 2 + assert {item['text'] for item in result['analysis_evidence']} == {source, annotation} + assert all(item['location']['match'] == 'exact' for item in result['analysis_evidence']) + + def test_formatted_table_quote_finalizes_with_caveats_and_notes_outside_validation(monkeypatch): table = '
RegionRevenue
North$1,200
' documents = {'revenue': original_document('revenue', [table])} diff --git a/functional_tests/test_orchestration_document_derivation_reliability.py b/functional_tests/test_orchestration_document_derivation_reliability.py index bc4d41b48..786a217d9 100644 --- a/functional_tests/test_orchestration_document_derivation_reliability.py +++ b/functional_tests/test_orchestration_document_derivation_reliability.py @@ -1,9 +1,11 @@ # test_orchestration_document_derivation_reliability.py """Source-grounded schema correction and producer-aware document generation recovery. -Version: 0.261.309 +Version: 0.261.314 Implemented in: 0.261.309 +Literal source annotations reach Word publication without correction as of 0.261.314. + Real collectors, producers, checkpoints, orchestration and renderers run offline. Malformed metadata must not erase uncertainty or rewrite accepted source findings. """ @@ -385,3 +387,51 @@ def repaired_reply(): else: assert not projection['eligible'], projection assert projection['reason_code'] == 'input_partial_not_recoverable' + + +@pytest.mark.parametrize('annotation', [ + '', + '', +]) +def test_literal_source_annotation_reaches_composition_and_published_word_without_repairs( + harness, narrative_io, monkeypatch, annotation, +): + search = importlib.import_module('functions_search_service') + original_read = search.get_ordered_document_chunks + + def read_chunks(*args, **kwargs): + chunks = original_read(*args, **kwargs) + chunks[0]['chunk_text'] += f'\n{annotation}' + return chunks + + monkeypatch.setattr(search, 'get_ordered_document_chunks', read_chunks) + + def reply(): + prompt = harness.model_calls[-1]['messages'][-1]['content'] + if '' in prompt: + payload = json.loads(narrative_io['reply']()) + if '[Page 1, Chunk 1]' in prompt: + payload['findings'][0]['evidence'].append({'chunk_sequence': 1, 'quote': annotation}) + return json.dumps(payload) + return '# Source-backed report\n\nComplete finding 006. This report contains test data.' + + harness.create( + document_steps('docx'), replies=[reply] * 3, + seeds={'document_ids': ['document-1'], 'doc_scope': 'personal'}, + original_seeds={'document_ids': ['document-1'], 'doc_scope': 'personal'}, + ) + execution = harness.prepare() + frames = execution.execute() + done = decoded_frames(frames)[-1] + assert done['status'] == 'completed', done + analysis = execution.context.task_results['analyze'] + assert analysis.status == 'complete' + assert analysis.output('findings').item_count == 7 + outputs, artifacts, payloads = download_outputs(harness) + assert len(outputs) == len(artifacts) == harness.blobs.file_uploads == 1 + text = '\n'.join(paragraph.text for paragraph in Document(io.BytesIO(payloads['report.docx'])).paragraphs) + assert 'Complete finding 006.' in text and 'This report contains test data.' in text + calls = harness.model_calls + assert len(calls) == 3 + assert sum('' in str(call['messages']) for call in calls) == 2 + assert all('' not in str(call['messages']) for call in calls)