Fix document evidence validation rejecting exact source quotes

application/single_app/config.py

@@ -101,7 +101,7 @@
 EXECUTOR_TYPE = 'thread'
 EXECUTOR_MAX_WORKERS = 30
 SESSION_TYPE = 'filesystem'
-VERSION = "0.261.313"
+VERSION = "0.261.314"
 IS_DEVELOPMENT = is_development_env_enabled()
 
 # Opt-out for deployments where App Service Easy Auth is active but the platform

application/single_app/functions_document_analysis_results.py

@@ -584,21 +584,17 @@
     quote = passage.get('quote')
     if not isinstance(quote, str) or not quote.strip():
         return None, 'missing_quote'
-    label = _EVIDENCE_PROMPT_LABEL.match(quote)
-    needles = {
-        tier: form[0]
-        for tier, form in _evidence_forms(quote[label.end():] if label else quote).items()
-    }
-    if not needles['normalized']:
-        return None, 'missing_quote'
     if not any(passage.get(key) is not None for key in _EVIDENCE_SELECTORS):
         return None, 'missing_location'
+    needles = {}
 
     def occurrence(index, tier):
         text = contents[index]['text']
         if tier == 'exact':
             offset = text.find(quote)
             return (offset, offset + len(quote)) if offset >= 0 else None
+        if not needles[tier]:
+            return None
         if index not in forms:
             forms[index] = _evidence_forms(text)
         return _evidence_occurrence(forms[index][tier], needles[tier], text)
@@ -611,6 +607,13 @@
         )
     }
     for tier in EVIDENCE_MATCH_TIERS:
+        if tier == 'normalized':
+            # Normalization may erase literal source content, such as extraction comments.
+            label = _EVIDENCE_PROMPT_LABEL.match(quote)
+            needles = {
+                name: form[0]
+                for name, form in _evidence_forms(quote[label.end():] if label else quote).items()
+            }
         hits = [
             (index, span) for index in sorted(selected)
             for span in [occurrence(index, tier)] if span is not None
@@ -627,7 +630,7 @@
         for tier in EVIDENCE_MATCH_TIERS
     ):
         return None, 'not_in_cited_location'
-    return None, 'not_in_window'
+    return None, 'not_in_window' if needles['normalized'] else 'missing_quote'
 
 
 def collect_analysis_window_candidates(analysis_text, source, work_unit, window_payload):

docs/explanation/fixes/DOCUMENT_ANALYSIS_LITERAL_EVIDENCE_FIX.md

@@ -0,0 +1,106 @@
+# Document analysis literal evidence fix
+
+**Version: 0.261.314**
+
+Fixed in version: **0.261.314**, recorded in
+`application/single_app/config.py`.
+
+## Issue and root cause
+
+A document-derived report could stop before drafting even after every source
+window was read. An evidence quote existed verbatim in its cited chunk, but the
+shared validator removed its contents during presentation normalization before
+trying exact matching. HTML comments, including Document Intelligence extraction
+annotations such as page footers, normalize to an empty string.
+
+The investigated run read all 20 pages in four windows. One finding had three
+matched citations and a fourth quoting a source footer marked "Test Data Only".
+The validator classified that fourth quote as `missing_quote`, rejected the
+finding, and retained only 24 of 25 findings. Compose then refused the partial
+input; Word rendering was never reached.
+
+This was a source-representation problem, not a missing document, an inability
+to read the PDF, or a Word renderer failure.
+
+## Shared exact-first matching
+
+`functions_document_analysis_results.py::_locate_analysis_evidence` now checks
+the literal quote against the original cited chunks before computing normalized
+forms. A source-backed comment, markup fragment, code example or extraction
+annotation follows the same exact-match rule as ordinary prose. There is no
+footer-specific exception or list of approved annotation formats.
+
+If exact matching fails, the existing normalized and case-folded comparisons
+remain available. Empty normalized needles are skipped so they cannot match
+unrelated text. Evidence retains the original source text and character offsets.
+The existing result contract, matching tiers and `evidence-matcher-v2` identifier
+are unchanged.
+
+The shared collector serves explicit narrative Analyze in chat, workflows and
+orchestration. Ordinary search/RAG chat is unchanged. The fix introduces no model
+calls, additional retries, settings, routes or source reprocessing.
+
+## Safeguards and limitations
+
+A quote must still be a nonblank string with a source-location selector.
+All supplied selectors must identify the cited chunk. A quote found only in
+another chunk fails; multiple matching cited chunks remain ambiguous.
+Normalized matching retains its existing word, number and sign protections.
+Other unsupported citations, conflicting findings and incomplete source coverage
+still prevent complete-only composition. Findings or failed citations are not
+silently discarded to complete a report.
+
+Locating a quote proves that the passage exists in the cited source, not that the
+model's entire finding is factually correct or logically entailed by it. This
+fix corrects the locator's unnecessary representation restriction; it does not
+add a factual-accuracy evaluation or relax the completeness contract.
+
+Old saved attempts and their recovery behavior are unchanged. After deploying
+this version, test with a fresh request; this change does not repair or migrate
+existing failed runs.
+
+## Files and regression coverage
+
+- `application/single_app/functions_document_analysis_results.py`: exact-first
+  matching and an empty-normalized-needle guard.
+- `functional_tests/test_document_analysis_evidence_matching.py`: synthetic
+  source comments, headers, footers, literal markup and label text; original
+  offsets; missing, wrong-location and ambiguous citations; empty fallbacks;
+  and real Analyze finalization without repair calls.
+- `functional_tests/test_orchestration_document_derivation_reliability.py`:
+  fresh Analyze to Compose to Word execution, durable publication, and actual
+  downloaded Word content, using the existing offline harness.
+- `application/single_app/config.py`: application patch version update.
+
+The existing evidence-matching documentation also links to this correction.
+Private production source chunks and model responses are not regression fixtures
+and are not committed to the repository.
+
+## Validation
+
+The unchanged saved model responses were replayed locally against the original
+20 source chunks. Their recomputed fingerprint matched the failed run.
+
+| Replay | Accepted findings | Unresolved | Status |
+| --- | --- | --- | --- |
+| Before the fix | 24 | 1 | Partial |
+| After the fix, with no response or source edits | 25 | 0 | Valid |
+
+Normal and optimized Python both produced the fixed result. Replay did not invoke
+a model, update a saved result or write to Azure.
+
+Local offline validation:
+
+| Coverage | Result |
+| --- | --- |
+| Evidence matching, final results and document derivation suites | 201 tests passed |
+| Saved analysis, workflow durability and execution-fence compatibility suites | 74 tests passed |
+| New Word publication cases | Both completed with two extraction calls and one composition call; no correction calls |
+| Documentation surface coverage and site quality | 7/7 and 6/6 checks passed; surface inventory unchanged |
+| `git diff --check` | Passed |
+
+Tests used an isolated environment with repository-pinned Flask/Werkzeug and
+compatible cryptography to avoid the machine's existing OpenSSL import failure.
+Global packages and application dependency manifests were not changed.
+
+This validation is not a production deployment or live model replay.

docs/explanation/fixes/V2_COMPARISON_ANALYZE_EVIDENCE_MATCHING_FIX.md

@@ -100,6 +100,12 @@ tier with a match:
    differences are removed.
 3. `normalized_casefold`: the same comparison, ignoring case.
 
+As of **0.261.314**, exact matching runs before any normalization. A quote that
+exists verbatim in its cited chunk remains evidence even if it consists of source
+comments, markup or an extraction annotation. Normalization is only a fallback;
+an empty normalized quote cannot match arbitrary source text. See the
+[literal evidence fix](DOCUMENT_ANALYSIS_LITERAL_EVIDENCE_FIX.md).
+
 The normalized comparison keeps every word and number separate, and drops only what
 renders as nothing:
 

functional_tests/test_document_analysis_evidence_matching.py

@@ -1,9 +1,11 @@
 # test_document_analysis_evidence_matching.py
 """
 Functional tests for general Analyze evidence matching, caveats, notes and validation logging.
-Version: 0.261.191
+Version: 0.261.314
 Implemented in: 0.261.191
 
+Literal source matching precedes lossy normalization as of 0.261.314.
+
 Refs #1540. A model quoting table-heavy or formatted source text must be located in its
 original chunk when the only differences are presentation: markup, table rules, entities,
 typographic quotes and dashes, compatibility forms, invisible characters, the amount of
@@ -117,6 +119,10 @@ def test_evidence_matching_fix_is_in_the_application_version():
     assert_app_version_at_least('0.261.191')
 
 
+def test_literal_evidence_fix_is_in_the_application_version():
+    assert_app_version_at_least('0.261.314')
+
+
 @pytest.mark.parametrize('chunk_text,quote,expected_text,tier', [
     pytest.param(
         '<table><tr><th>Region</th><th>Revenue</th></tr><tr><td>North</td><td>$1,200</td></tr></table>',
@@ -233,6 +239,76 @@ def test_presentation_only_differences_locate_the_verbatim_source_span(chunk_tex
     assert result['evidence_matching'] == match_counts(**{tier: 1})
 
 
+@pytest.mark.parametrize('quote', [
+    'Test Data Only',
+    '<!-- PageFooter: Example report - Test Data Only -->',
+    '<!-- PageHeader: Internal report -->',
+    '<!-- This example is not audited financial data. -->',
+    '<settings mode="preview" />',
+    '<td></td>',
+    '[Page 1, Chunk 1]',
+    '---',
+])
+def test_literal_source_content_is_matched_before_normalization(quote, monkeypatch):
+    text = f'Source begins here.\n{quote}\nSource ends here.'
+    with document_analysis_runtime({}) as runtime:
+        def normalization_must_not_run(*args):
+            raise AssertionError('An exact source match must not depend on presentation normalization.')
+
+        monkeypatch.setattr(runtime.results, '_evidence_forms', normalization_must_not_run)
+        result = collect(runtime, window_chunks(text), supported_finding({'chunk_sequence': 1, 'quote': quote}))
+    [candidate] = result['candidates']
+    assert candidate['status'] == 'candidate'
+    assert candidate['issues'] == []
+    [passage] = result['evidence']
+    location = passage['location']
+    assert passage['text'] == quote
+    assert (location['start_char'], location['end_char']) == (text.index(quote), text.index(quote) + len(quote))
+    assert location['match'] == 'exact'
+    assert candidate['evidence_refs'] == [passage['evidence_id']]
+    assert result['evidence_matching'] == match_counts(exact=1)
+
+
+@pytest.mark.parametrize('quote', ['<!-- Internal test data -->', '<settings mode="preview" />'])
+@pytest.mark.parametrize('selectors,reason', [
+    ({}, 'missing_location'),
+    ({'chunk_sequence': 9}, 'not_in_cited_location'),
+    ({'chunk_sequence': 2, 'page_number': 2}, 'not_in_cited_location'),
+    ({'page_number': 1}, 'ambiguous'),
+])
+def test_literal_source_content_still_requires_an_unambiguous_cited_chunk(quote, selectors, reason):
+    with document_analysis_runtime({}) as runtime:
+        result = collect(
+            runtime, window_chunks(quote, quote, pages=[1, 1]),
+            supported_finding({**selectors, 'quote': quote}),
+        )
+    [candidate] = result['candidates']
+    assert candidate['status'] == 'unresolved'
+    assert candidate['evidence_refs'] == []
+    assert result['evidence'] == []
+    assert [issue.get('reason') for issue in candidate['issues']] == [reason]
+    assert result['evidence_matching'] == match_counts(**{reason: 1})
+
+
+@pytest.mark.parametrize('quote', [
+    '<!-- Absent source annotation -->',
+    '<td></td>',
+    '[Page 1, Chunk 1]',
+    '---',
+])
+def test_empty_normalized_quotes_cannot_match_unrelated_source_text(quote):
+    with document_analysis_runtime({}) as runtime:
+        result = collect(
+            runtime, window_chunks('Ordinary source text.'),
+            supported_finding({'chunk_sequence': 1, 'quote': quote}),
+        )
+    [candidate] = result['candidates']
+    assert candidate['status'] == 'unresolved'
+    assert candidate['evidence_refs'] == []
+    assert result['evidence'] == []
+    assert result['evidence_matching'] == match_counts(missing_quote=1)
+
+
 @pytest.mark.parametrize('passage,reason', [
     pytest.param({'chunk_sequence': 3, 'quote': 'Early cancellation carries a penalty.'}, 'not_in_window', id='paraphrase'),
     pytest.param({'chunk_sequence': 3, 'quote': 'An exit penalty ... cancellation.'}, 'not_in_window', id='ellipsis'),
@@ -248,6 +324,8 @@ def test_presentation_only_differences_locate_the_verbatim_source_span(chunk_tex
     pytest.param({'quote': 'sole supplier'}, 'missing_location', id='no_selector'),
     pytest.param({'chunk_sequence': 1, 'quote': '   '}, 'missing_quote', id='blank_quote'),
     pytest.param({'chunk_sequence': 1}, 'missing_quote', id='no_quote'),
+    pytest.param({'chunk_sequence': 1, 'quote': None}, 'missing_quote', id='null_quote'),
+    pytest.param({'chunk_sequence': 1, 'quote': ['sole supplier']}, 'missing_quote', id='non_text_quote'),
     pytest.param({'chunk_sequence': 1, 'quote': '<td></td>'}, 'missing_quote', id='markup_only_quote'),
     pytest.param({'chunk_sequence': 1, 'quote': '[Page 1, Chunk 1]'}, 'missing_quote', id='label_only_quote'),
     pytest.param('sole supplier', 'missing_quote', id='passage_is_not_an_object'),
@@ -355,6 +433,37 @@ def test_exact_offsets_are_unchanged_and_one_unsupported_passage_still_unresolve
     assert result['evidence_matching'] == match_counts(exact=1, normalized_casefold=1, not_in_window=1)
 
 
+@pytest.mark.parametrize('annotation', [
+    'Example report - Test Data Only',
+    '<!-- PageFooter: Example report - Test Data Only -->',
+    '<!-- Example report - Test Data Only -->',
+])
+def test_source_annotation_and_ordinary_citations_finalize_without_repair_calls(annotation):
+    source = 'The service owner is Mira.'
+    documents = {'example': original_document('example', [f'{source}\n{annotation}'])}
+
+    def response(prompt):
+        payload = json.loads(extract_fixture_findings(prompt))
+        payload['findings'][0]['evidence'].append({'chunk_sequence': 1, 'quote': annotation})
+        return json.dumps(payload)
+
+    with document_analysis_runtime(documents) as runtime:
+        result, client = run_analysis(
+            runtime, documents, FixtureAnalysisClient(response), max_retries_per_window=1,
+        )
+    assert len(client.calls) == 1
+    assert result['coverage']['retries'] == 0
+    validation = result['analysis_validation']
+    assert validation['status'] == 'valid'
+    assert validation['issues'] == []
+    assert validation['unresolved_candidate_count'] == 0
+    [record] = result['authoritative_result']['value']
+    assert record['values'] == {'owner': 'Mira'}
+    assert len(record['evidence_refs']) == 2
+    assert {item['text'] for item in result['analysis_evidence']} == {source, annotation}
+    assert all(item['location']['match'] == 'exact' for item in result['analysis_evidence'])
+
+
 def test_formatted_table_quote_finalizes_with_caveats_and_notes_outside_validation(monkeypatch):
     table = '<table><tr><th>Region</th><th>Revenue</th></tr><tr><td>North</td><td>$1,200</td></tr></table>'
     documents = {'revenue': original_document('revenue', [table])}

functional_tests/test_orchestration_document_derivation_reliability.py

@@ -1,9 +1,11 @@
 # test_orchestration_document_derivation_reliability.py
 """Source-grounded schema correction and producer-aware document generation recovery.
 
-Version: 0.261.309
+Version: 0.261.314
 Implemented in: 0.261.309
 
+Literal source annotations reach Word publication without correction as of 0.261.314.
+
 Real collectors, producers, checkpoints, orchestration and renderers run offline.
 Malformed metadata must not erase uncertainty or rewrite accepted source findings.
 """
@@ -385,3 +387,51 @@ def repaired_reply():
     else:
         assert not projection['eligible'], projection
         assert projection['reason_code'] == 'input_partial_not_recoverable'
+
+
+@pytest.mark.parametrize('annotation', [
+    '<!-- PageFooter: Example report - Test Data Only -->',
+    '<!-- This report contains test data. -->',
+])
+def test_literal_source_annotation_reaches_composition_and_published_word_without_repairs(
+    harness, narrative_io, monkeypatch, annotation,
+):
+    search = importlib.import_module('functions_search_service')
+    original_read = search.get_ordered_document_chunks
+
+    def read_chunks(*args, **kwargs):
+        chunks = original_read(*args, **kwargs)
+        chunks[0]['chunk_text'] += f'\n{annotation}'
+        return chunks
+
+    monkeypatch.setattr(search, 'get_ordered_document_chunks', read_chunks)
+
+    def reply():
+        prompt = harness.model_calls[-1]['messages'][-1]['content']
+        if '<DocumentSlice>' in prompt:
+            payload = json.loads(narrative_io['reply']())
+            if '[Page 1, Chunk 1]' in prompt:
+                payload['findings'][0]['evidence'].append({'chunk_sequence': 1, 'quote': annotation})
+            return json.dumps(payload)
+        return '# Source-backed report\n\nComplete finding 006. This report contains test data.'
+
+    harness.create(
+        document_steps('docx'), replies=[reply] * 3,
+        seeds={'document_ids': ['document-1'], 'doc_scope': 'personal'},
+        original_seeds={'document_ids': ['document-1'], 'doc_scope': 'personal'},
+    )
+    execution = harness.prepare()
+    frames = execution.execute()
+    done = decoded_frames(frames)[-1]
+    assert done['status'] == 'completed', done
+    analysis = execution.context.task_results['analyze']
+    assert analysis.status == 'complete'
+    assert analysis.output('findings').item_count == 7
+    outputs, artifacts, payloads = download_outputs(harness)
+    assert len(outputs) == len(artifacts) == harness.blobs.file_uploads == 1
+    text = '\n'.join(paragraph.text for paragraph in Document(io.BytesIO(payloads['report.docx'])).paragraphs)
+    assert 'Complete finding 006.' in text and 'This report contains test data.' in text
+    calls = harness.model_calls
+    assert len(calls) == 3
+    assert sum('<DocumentSlice>' in str(call['messages']) for call in calls) == 2
+    assert all('<ResponseShapeCorrection>' not in str(call['messages']) for call in calls)