From 9a17a134c42556baad3382793ff73af740decfe6 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Mon, 17 Aug 2026 22:45:09 -0400 Subject: [PATCH 1/2] Distinguish retrieved sources from cited references Separate the complete set of retrieved workspace and web search results from the smaller set of references the final assistant response actually cites, so Used documents, conversation details, and export references stop presenting every retrieval candidate as a citation. New assistant responses persist exact cited document and web subsets alongside the unchanged full source arrays. Conversation reads use a compact persisted aggregate instead of reparsing message history, and conversations without tracking keep their existing document list as a no-migration compatibility fallback. - Add shared citation tracking with exact citation IDs plus strict filename/location matching for visible legacy-style source references - Persist cited subsets across non-streaming, streaming, mixed-source, assigned-knowledge, document-action, partial, and workflow responses - Rebuild exact used documents after retry, edit, attempt switching, deletion, and conversation fork, and repair a failed rebuild on read - Propagate tracking through collaboration conversion, synchronization, serialization, and stream mirroring, including persisted partials - Show all source documents in conversation details with a Cited badge, and drive Used documents from exact usage plus legacy fallback - Use cited document/web subsets plus executed tools for conversation Markdown/PDF and per-message Word, PowerPoint, and email references while JSON exports retain full source arrays for audit - Render sheet, page, and workbook schema locations consistently Fixes #1249 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../single_app/collaboration_models.py | 3 + application/single_app/config.py | 2 +- .../single_app/functions_citation_tracking.py | 686 ++++++++++++++++++ .../single_app/functions_collaboration.py | 78 +- .../functions_simplechat_operations.py | 8 + .../single_app/functions_workflow_runner.py | 72 +- application/single_app/route_backend_chats.py | 238 ++++-- .../single_app/route_backend_collaboration.py | 16 +- .../route_backend_conversation_export.py | 81 ++- .../single_app/route_backend_conversations.py | 87 +++ .../static/js/chat/chat-citations.js | 9 +- .../static/js/chat/chat-collaboration.js | 29 + .../js/chat/chat-conversation-contents.js | 102 ++- .../js/chat/chat-conversation-details.js | 82 ++- .../single_app/static/js/chat/chat-edit.js | 9 + .../static/js/chat/chat-messages.js | 11 + .../single_app/static/js/chat/chat-retry.js | 18 + .../static/js/chat/chat-streaming.js | 55 +- .../features/CONVERSATION_CONTENTS_DRAWER.md | 14 +- .../v0.238.024/CITATION_IMPROVEMENTS.md | 20 + ...RCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md | 95 +++ docs/explanation/release_notes.md | 10 + .../test_chat_cited_source_tracking.py | 630 ++++++++++++++++ functional_tests/test_conversation_export.py | 131 +++- functional_tests/test_per_message_export.py | 6 +- .../test_per_message_powerpoint_export.py | 5 +- .../test_chat_conversation_contents_drawer.py | 117 ++- ...t_chat_markdown_citation_lookup_payload.py | 85 ++- ..._lock_and_conversation_details_escaping.py | 44 +- 29 files changed, 2596 insertions(+), 147 deletions(-) create mode 100644 application/single_app/functions_citation_tracking.py create mode 100644 docs/explanation/fixes/SOURCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md create mode 100644 functional_tests/test_chat_cited_source_tracking.py diff --git a/application/single_app/collaboration_models.py b/application/single_app/collaboration_models.py index 0c98dd940..e2f322ea2 100644 --- a/application/single_app/collaboration_models.py +++ b/application/single_app/collaboration_models.py @@ -678,6 +678,9 @@ def build_collaboration_message_doc_from_legacy( 'augmented', 'hybrid_citations', 'web_search_citations', + 'citation_tracking_version', + 'cited_hybrid_citations', + 'cited_web_search_citations', 'agent_citations', 'agent_display_name', 'agent_name', diff --git a/application/single_app/config.py b/application/single_app/config.py index 534b9a812..3661cf2b0 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -96,7 +96,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.214" +VERSION = "0.250.215" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_citation_tracking.py b/application/single_app/functions_citation_tracking.py new file mode 100644 index 000000000..e1d90aa04 --- /dev/null +++ b/application/single_app/functions_citation_tracking.py @@ -0,0 +1,686 @@ +# functions_citation_tracking.py +"""Track retrieved sources separately from explicitly cited references.""" + +import html +import re +from copy import deepcopy +from typing import Any, Dict, Iterable, List, Mapping, Optional +from urllib.parse import urlsplit, urlunsplit + + +CITATION_TRACKING_VERSION = 1 +USED_DOCUMENTS_TRACKING_VERSION = 1 + +_INLINE_DOCUMENT_CITATION_GROUP_PATTERN = re.compile( + r"\[\s*#([^\]\r\n]+)\]", + re.IGNORECASE, +) +_INLINE_DOCUMENT_SOURCE_PATTERN = re.compile( + r"\(Source:\s*(.+?),\s*(Page(?:s)?|Sheet(?:s)?|Location):\s*(.*?)\)" + r"(?=\s*(?:\[#|$|[.;,!? \n]))", + re.IGNORECASE, +) +_HTTP_URL_PATTERN = re.compile(r"https?://[^\s<>\"']+", re.IGNORECASE) +_TRAILING_URL_PUNCTUATION = ".,;:!?" +_MERGED_DOCUMENT_LIST_FIELDS = ( + "chunk_ids", + "citation_ids", + "page_numbers", + "sheet_names", +) + + +def _as_dict_list(value: Any) -> List[Dict[str, Any]]: + if not isinstance(value, list): + return [] + return [deepcopy(item) for item in value if isinstance(item, dict)] + + +def _append_unique(values: List[Any], value: Any) -> None: + if value in (None, "") or value in values: + return + values.append(deepcopy(value)) + + +def _append_unique_mapping(values: List[Dict[str, Any]], value: Dict[str, Any]) -> None: + if not value or value in values: + return + values.append(deepcopy(value)) + + +def _normalize_reference_id(value: Any) -> str: + return str(value or "").strip().lstrip("#").strip() + + +def extract_explicit_document_citation_ids(content: Any) -> List[str]: + """Return ordered, unique citation IDs from explicit ``[#id]`` groups.""" + citation_ids: List[str] = [] + for citation_group in _INLINE_DOCUMENT_CITATION_GROUP_PATTERN.findall( + str(content or "") + ): + for raw_citation_id in re.split(r"[;,]", citation_group): + citation_id = _normalize_reference_id(raw_citation_id) + if citation_id and citation_id not in citation_ids: + citation_ids.append(citation_id) + return citation_ids + + +def _normalize_source_text(value: Any) -> str: + return re.sub(r"\s+", " ", html.unescape(str(value or ""))).strip().lower() + + +def _extract_explicit_document_source_references( + content: Any, +) -> List[Dict[str, str]]: + references = [] + for file_name, location_label, location_value in ( + _INLINE_DOCUMENT_SOURCE_PATTERN.findall(str(content or "")) + ): + normalized_file_name = _normalize_source_text(file_name) + normalized_location_label = ( + _normalize_source_text(location_label).rstrip("s") + ) + normalized_location_value = _normalize_source_text(location_value) + if ( + normalized_file_name + and normalized_location_label + and normalized_location_value + ): + references.append({ + "file_name": normalized_file_name, + "location_label": normalized_location_label, + "location_value": normalized_location_value, + }) + return references + + +def _parse_numeric_location(value: Any) -> Optional[int]: + normalized_value = _normalize_source_text(value) + if not normalized_value or not normalized_value.isdigit(): + return None + return int(normalized_value) + + +def _location_reference_matches( + explicit_location: str, + citation_location: Any, +) -> bool: + normalized_citation_location = _normalize_source_text(citation_location) + if not normalized_citation_location: + return False + if normalized_citation_location == explicit_location: + return True + + explicit_tokens = [ + token.strip() + for token in re.split(r"[,;]", explicit_location) + if token.strip() + ] + if normalized_citation_location in explicit_tokens: + return True + + citation_number = _parse_numeric_location(normalized_citation_location) + if citation_number is None: + return False + + for token in explicit_tokens: + range_match = re.fullmatch(r"(\d+)\s*[-\u2013\u2014]\s*(\d+)", token) + if not range_match: + continue + range_start, range_end = ( + int(range_match.group(1)), + int(range_match.group(2)), + ) + lower_bound = min(range_start, range_end) + upper_bound = max(range_start, range_end) + if lower_bound <= citation_number <= upper_bound: + return True + return False + + +def resolve_citation_location( + page_number: Any = None, + chunk_text: Any = None, + sheet_name: Any = None, + is_tabular: bool = False, +) -> tuple[str, str]: + """Return a schema-, sheet-, or page-aware citation location.""" + if sheet_name not in (None, ""): + return "Sheet", str(sheet_name) + + normalized_chunk_text = str(chunk_text or "").strip() + if is_tabular and ( + normalized_chunk_text.startswith("Tabular workbook:") + or normalized_chunk_text.startswith("Tabular data file:") + ): + return "Location", "Workbook Schema" + + return "Page", str(page_number or 1) + + +def _source_reference_matches_citation( + source_reference: Mapping[str, str], + citation: Mapping[str, Any], +) -> bool: + citation_file_name = _normalize_source_text( + citation.get("file_name") + or citation.get("title") + ) + if ( + not citation_file_name + or citation_file_name != source_reference.get("file_name") + ): + return False + + explicit_label = source_reference.get("location_label") + citation_label = _normalize_source_text(citation.get("location_label")) + if not citation_label: + citation_label = "sheet" if citation.get("sheet_name") else "page" + citation_label = citation_label.rstrip("s") + if explicit_label != "location" and citation_label != explicit_label: + return False + + location_candidates = ( + citation.get("location_value"), + citation.get("sheet_name"), + citation.get("page_number"), + citation.get("chunk_sequence"), + ) + return any( + _location_reference_matches( + source_reference.get("location_value", ""), + candidate, + ) + for candidate in location_candidates + ) + + +def _trim_url_candidate(value: Any) -> str: + candidate = html.unescape(str(value or "")).strip() + while candidate and candidate[-1] in _TRAILING_URL_PUNCTUATION: + candidate = candidate[:-1] + while candidate.endswith(")") and candidate.count(")") > candidate.count("("): + candidate = candidate[:-1] + while candidate.endswith("]") and candidate.count("]") > candidate.count("["): + candidate = candidate[:-1] + return candidate + + +def normalize_http_url(value: Any) -> str: + """Normalize an HTTP(S) URL for exact response-to-source matching.""" + candidate = _trim_url_candidate(value) + if not candidate: + return "" + + try: + parsed = urlsplit(candidate) + except ValueError: + return "" + + scheme = parsed.scheme.lower() + if scheme not in {"http", "https"} or not parsed.hostname: + return "" + + hostname = parsed.hostname.lower() + try: + port = parsed.port + except ValueError: + return "" + if port and not ( + (scheme == "http" and port == 80) + or (scheme == "https" and port == 443) + ): + hostname = f"{hostname}:{port}" + + path = parsed.path or "" + if path == "/": + path = "" + + return urlunsplit((scheme, hostname, path, parsed.query, "")) + + +def extract_explicit_web_urls(content: Any) -> List[str]: + """Return ordered, unique normalized HTTP(S) URLs from final response text.""" + normalized_urls: List[str] = [] + for raw_url in _HTTP_URL_PATTERN.findall(str(content or "")): + normalized_url = normalize_http_url(raw_url) + if normalized_url and normalized_url not in normalized_urls: + normalized_urls.append(normalized_url) + return normalized_urls + + +def _get_document_citation_id(citation: Mapping[str, Any]) -> str: + return _normalize_reference_id( + citation.get("citation_id") + or citation.get("id") + ) + + +def _get_web_citation_url(citation: Mapping[str, Any]) -> str: + return normalize_http_url( + citation.get("url") + or citation.get("href") + or citation.get("link") + ) + + +def build_cited_source_subsets( + content: Any, + hybrid_citations: Any = None, + web_search_citations: Any = None, +) -> Dict[str, Any]: + """Match explicit final-response references to returned source records.""" + document_citation_ids = set(extract_explicit_document_citation_ids(content)) + document_source_references = _extract_explicit_document_source_references( + content + ) + explicit_web_urls = set(extract_explicit_web_urls(content)) + + hybrid_source_records = _as_dict_list(hybrid_citations) + exact_document_citations = [ + citation + for citation in hybrid_source_records + if _get_document_citation_id(citation) in document_citation_ids + ] + unmatched_source_references = [ + source_reference + for source_reference in document_source_references + if not any( + _source_reference_matches_citation( + source_reference, + exact_citation, + ) + for exact_citation in exact_document_citations + ) + ] + + cited_hybrid_citations: List[Dict[str, Any]] = [] + seen_document_citations = set() + for citation in hybrid_source_records: + citation_id = _get_document_citation_id(citation) + matched_by_id = bool( + citation_id + and citation_id in document_citation_ids + ) + matched_by_source = any( + _source_reference_matches_citation(source_reference, citation) + for source_reference in unmatched_source_references + ) + if not matched_by_id and not matched_by_source: + continue + if matched_by_id: + identity = ( + "citation_id", + citation_id, + str(citation.get("document_id") or ""), + str(citation.get("chunk_id") or ""), + ) + else: + identity = ( + "source_location", + _extract_document_id(citation), + _normalize_source_text( + citation.get("file_name") + or citation.get("title") + ), + _normalize_source_text( + citation.get("location_value") + or citation.get("sheet_name") + or citation.get("page_number") + or citation.get("chunk_sequence") + ), + ) + if identity in seen_document_citations: + continue + seen_document_citations.add(identity) + cited_hybrid_citations.append(citation) + + cited_web_search_citations: List[Dict[str, Any]] = [] + seen_web_urls = set() + for citation in _as_dict_list(web_search_citations): + normalized_url = _get_web_citation_url(citation) + if ( + not normalized_url + or normalized_url not in explicit_web_urls + or normalized_url in seen_web_urls + ): + continue + seen_web_urls.add(normalized_url) + cited_web_search_citations.append(citation) + + return { + "citation_tracking_version": CITATION_TRACKING_VERSION, + "cited_hybrid_citations": cited_hybrid_citations, + "cited_web_search_citations": cited_web_search_citations, + } + + +def _message_has_citation_tracking(message: Mapping[str, Any]) -> bool: + tracking_version = message.get("citation_tracking_version") + if isinstance(tracking_version, int) and tracking_version >= 1: + return True + return ( + "cited_hybrid_citations" in message + or "cited_web_search_citations" in message + ) + + +def get_message_source_citation_buckets( + message: Optional[Mapping[str, Any]], +) -> Dict[str, List[Any]]: + """Return complete stored source/execution buckets for audit and Sources UI.""" + safe_message = message if isinstance(message, Mapping) else {} + + def ensure_list(value: Any) -> List[Any]: + if not value: + return [] + return deepcopy(value) if isinstance(value, list) else [deepcopy(value)] + + return { + "legacy": ensure_list(safe_message.get("citations")), + "hybrid": ensure_list(safe_message.get("hybrid_citations")), + "web": ensure_list(safe_message.get("web_search_citations")), + "agent": ensure_list(safe_message.get("agent_citations")), + } + + +def get_message_reference_citation_buckets( + message: Optional[Mapping[str, Any]], +) -> Dict[str, List[Any]]: + """Return exact references, with broad source fallback for legacy messages.""" + safe_message = message if isinstance(message, Mapping) else {} + source_buckets = get_message_source_citation_buckets(safe_message) + if not _message_has_citation_tracking(safe_message): + return source_buckets + + referenced_agent_citations = [] + for citation in source_buckets["agent"]: + if ( + isinstance(citation, Mapping) + and str(citation.get("function_name") or "").strip().lower() + == "azure_ai_foundry_web_search" + ): + continue + referenced_agent_citations.append(deepcopy(citation)) + + return { + "legacy": source_buckets["legacy"], + "hybrid": _as_dict_list(safe_message.get("cited_hybrid_citations")), + "web": _as_dict_list(safe_message.get("cited_web_search_citations")), + "agent": referenced_agent_citations, + } + + +def _extract_document_id(citation: Mapping[str, Any]) -> str: + document_id = str(citation.get("document_id") or "").strip() + if document_id: + return document_id + + citation_id = _get_document_citation_id(citation) + if not citation_id: + return "" + if "_" in citation_id: + return citation_id.rsplit("_", 1)[0] + return citation_id + + +def _get_document_tags(conversation: Mapping[str, Any]) -> List[Dict[str, Any]]: + tags = conversation.get("tags") + if not isinstance(tags, list): + return [] + return [ + deepcopy(tag) + for tag in tags + if isinstance(tag, dict) + and tag.get("category") == "document" + and str(tag.get("document_id") or "").strip() + ] + + +def _scope_from_citation(citation: Mapping[str, Any]) -> Dict[str, Any]: + raw_scope = citation.get("scope") + if isinstance(raw_scope, dict): + scope_type = str( + raw_scope.get("type") + or raw_scope.get("scope") + or "" + ).strip() + scope_id = str( + raw_scope.get("id") + or raw_scope.get("scope_id") + or "" + ).strip() + if scope_type and scope_id: + scope = { + "type": scope_type, + "id": scope_id, + } + scope_name = str(raw_scope.get("name") or "").strip() + if scope_name: + scope["name"] = scope_name + return scope + + public_workspace_id = str( + citation.get("public_workspace_id") or "" + ).strip() + if public_workspace_id: + return {"type": "public", "id": public_workspace_id} + + group_id = str(citation.get("group_id") or "").strip() + if group_id: + return {"type": "group", "id": group_id} + + user_id = str(citation.get("user_id") or "").strip() + if user_id: + return {"type": "personal", "id": user_id} + return {} + + +def _merge_document_record( + target: Dict[str, Any], + incoming: Mapping[str, Any], +) -> Dict[str, Any]: + for field_name in _MERGED_DOCUMENT_LIST_FIELDS: + target_values = target.setdefault(field_name, []) + incoming_values = incoming.get(field_name) + if not isinstance(incoming_values, list): + continue + for value in incoming_values: + _append_unique(target_values, value) + + target_locations = target.setdefault("citation_locations", []) + incoming_locations = incoming.get("citation_locations") + if isinstance(incoming_locations, list): + for location in incoming_locations: + if isinstance(location, dict): + _append_unique_mapping(target_locations, location) + + for field_name in ( + "title", + "file_name", + "classification", + "scope", + ): + current_value = target.get(field_name) + incoming_value = incoming.get(field_name) + if current_value in (None, "", {}, "Unknown Document") and incoming_value not in ( + None, + "", + {}, + ): + target[field_name] = deepcopy(incoming_value) + return target + + +def build_used_documents( + cited_hybrid_citations: Any, + source_document_tags: Any = None, +) -> List[Dict[str, Any]]: + """Collapse exact cited chunks into document-level used-document records.""" + tag_by_document_id = { + str(tag.get("document_id") or "").strip(): tag + for tag in _as_dict_list(source_document_tags) + if str(tag.get("document_id") or "").strip() + } + documents_by_id: Dict[str, Dict[str, Any]] = {} + + for citation in _as_dict_list(cited_hybrid_citations): + document_id = _extract_document_id(citation) + if not document_id: + continue + + document = documents_by_id.get(document_id) + if document is None: + source_tag = tag_by_document_id.get(document_id) + document = deepcopy(source_tag) if source_tag else { + "category": "document", + "document_id": document_id, + } + document["category"] = "document" + document["document_id"] = document_id + document["chunk_ids"] = [] + document["citation_ids"] = [] + document["page_numbers"] = [] + document["sheet_names"] = [] + document["citation_locations"] = [] + documents_by_id[document_id] = document + + citation_id = _get_document_citation_id(citation) + _append_unique(document["citation_ids"], citation_id) + _append_unique( + document["chunk_ids"], + citation_id or citation.get("chunk_id"), + ) + _append_unique(document["page_numbers"], citation.get("page_number")) + _append_unique(document["sheet_names"], citation.get("sheet_name")) + + location = { + key: deepcopy(citation.get(key)) + for key in ( + "citation_id", + "chunk_id", + "page_number", + "sheet_name", + "location_label", + "location_value", + ) + if citation.get(key) not in (None, "") + } + _append_unique_mapping(document["citation_locations"], location) + + incoming_metadata = { + "title": citation.get("title") or citation.get("file_name"), + "file_name": citation.get("file_name") or citation.get("title"), + "classification": ( + citation.get("classification") + or citation.get("document_classification") + ), + "scope": _scope_from_citation(citation), + } + _merge_document_record(document, incoming_metadata) + + return list(documents_by_id.values()) + + +def merge_used_documents( + existing_documents: Any, + incoming_documents: Any, +) -> List[Dict[str, Any]]: + """Merge exact used-document aggregates without losing cited locations.""" + merged_by_id: Dict[str, Dict[str, Any]] = {} + for document in [ + *_as_dict_list(existing_documents), + *_as_dict_list(incoming_documents), + ]: + document_id = str(document.get("document_id") or "").strip() + if not document_id: + continue + if document_id not in merged_by_id: + merged_by_id[document_id] = deepcopy(document) + continue + _merge_document_record(merged_by_id[document_id], document) + return list(merged_by_id.values()) + + +def initialize_conversation_used_document_tracking( + conversation: Dict[str, Any], +) -> bool: + """Snapshot legacy document tags before the first strictly tracked turn.""" + if not isinstance(conversation, dict): + return False + tracking_version = conversation.get("used_documents_tracking_version") + if isinstance(tracking_version, int) and tracking_version >= 1: + return False + + conversation["used_documents_tracking_version"] = ( + USED_DOCUMENTS_TRACKING_VERSION + ) + conversation["legacy_used_documents"] = _get_document_tags(conversation) + conversation["used_documents"] = [] + return True + + +def merge_cited_documents_into_conversation( + conversation: Dict[str, Any], + cited_hybrid_citations: Any, +) -> List[Dict[str, Any]]: + """Merge one finalized assistant response into exact conversation usage.""" + initialize_conversation_used_document_tracking(conversation) + source_document_tags = _get_document_tags(conversation) + incoming_documents = build_used_documents( + cited_hybrid_citations, + source_document_tags=source_document_tags, + ) + conversation["used_documents"] = merge_used_documents( + conversation.get("used_documents"), + incoming_documents, + ) + return deepcopy(conversation["used_documents"]) + + +def _message_is_active_assistant(message: Mapping[str, Any]) -> bool: + if message.get("role") != "assistant": + return False + metadata = message.get("metadata") + metadata = metadata if isinstance(metadata, Mapping) else {} + if metadata.get("is_deleted") is True: + return False + if metadata.get("is_generated_chat_artifact") is True: + return False + thread_info = metadata.get("thread_info") + thread_info = thread_info if isinstance(thread_info, Mapping) else {} + return thread_info.get("active_thread") is not False + + +def rebuild_conversation_used_documents( + conversation: Dict[str, Any], + messages: Iterable[Mapping[str, Any]], + rebuild_legacy: bool = False, +) -> List[Dict[str, Any]]: + """Rebuild exact usage after a retry, switch, deletion, or fork mutation.""" + initialize_conversation_used_document_tracking(conversation) + cited_hybrid_citations: List[Dict[str, Any]] = [] + legacy_hybrid_citations: List[Dict[str, Any]] = [] + for message in messages or []: + if not isinstance(message, Mapping) or not _message_is_active_assistant(message): + continue + if _message_has_citation_tracking(message): + cited_hybrid_citations.extend( + _as_dict_list(message.get("cited_hybrid_citations")) + ) + elif rebuild_legacy: + legacy_hybrid_citations.extend( + _as_dict_list(message.get("hybrid_citations")) + ) + + source_document_tags = _get_document_tags(conversation) + conversation["used_documents"] = build_used_documents( + cited_hybrid_citations, + source_document_tags=source_document_tags, + ) + if rebuild_legacy: + conversation["legacy_used_documents"] = build_used_documents( + legacy_hybrid_citations, + source_document_tags=source_document_tags, + ) + return deepcopy(conversation["used_documents"]) diff --git a/application/single_app/functions_collaboration.py b/application/single_app/functions_collaboration.py index d434cb62a..75e9e78c0 100644 --- a/application/single_app/functions_collaboration.py +++ b/application/single_app/functions_collaboration.py @@ -40,6 +40,11 @@ log_conversation_deletion, ) from functions_appinsights import log_event +from functions_citation_tracking import ( + initialize_conversation_used_document_tracking, + merge_cited_documents_into_conversation, + rebuild_conversation_used_documents, +) from functions_conversation_cache import bump_conversation_cache_version, invalidate_conversation_cache_for_item from functions_documents import sync_chat_upload_workspace_document_sharing_for_collaboration from functions_group import ( @@ -64,6 +69,21 @@ COLLABORATION_EVENT_PUBLISHERS = [] +CITATION_TRACKING_CONVERSATION_FIELDS = ( + 'used_documents_tracking_version', + 'legacy_used_documents', + 'used_documents', +) + + +def _copy_citation_tracking_conversation_fields(target, source): + if not isinstance(target, dict) or not isinstance(source, dict): + return target + for field_name in CITATION_TRACKING_CONVERSATION_FIELDS: + if field_name in source: + target[field_name] = deepcopy(source.get(field_name)) + return target + def register_collaboration_event_publisher(event_publisher): if callable(event_publisher) and event_publisher not in COLLABORATION_EVENT_PUBLISHERS: @@ -355,7 +375,7 @@ def serialize_collaboration_message(message_doc): message_doc.get('id'), ) or serialized_content - return { + payload = { 'id': message_doc.get('id'), 'conversation_id': message_doc.get('conversation_id'), 'role': serialized_role, @@ -382,6 +402,14 @@ def serialize_collaboration_message(message_doc): 'extracted_text': message_doc.get('extracted_text'), 'vision_analysis': message_doc.get('vision_analysis'), } + for field_name in ( + 'citation_tracking_version', + 'cited_hybrid_citations', + 'cited_web_search_citations', + ): + if field_name in message_doc: + payload[field_name] = deepcopy(message_doc.get(field_name)) + return payload def _get_collaboration_source_message(message_doc): @@ -729,6 +757,15 @@ def build_collaboration_message_metadata_payload(message_doc, conversation_doc): 'extracted_text': message_doc.get('extracted_text') or payload.get('extracted_text'), 'vision_analysis': message_doc.get('vision_analysis') or payload.get('vision_analysis'), }) + for field_name in ( + 'citation_tracking_version', + 'cited_hybrid_citations', + 'cited_web_search_citations', + ): + if field_name in message_doc: + payload[field_name] = deepcopy(message_doc.get(field_name)) + elif field_name in (source_message_doc or {}): + payload[field_name] = deepcopy(source_message_doc.get(field_name)) payload['metadata'] = merged_metadata return payload @@ -820,6 +857,9 @@ def serialize_collaboration_conversation(conversation_doc, current_user_id, user 'is_hidden': bool((user_state or {}).get('is_hidden', False)), 'classification': list(conversation_doc.get('classification', []) or []), 'tags': list(conversation_doc.get('tags', []) or []), + 'used_documents_tracking_version': conversation_doc.get('used_documents_tracking_version'), + 'legacy_used_documents': deepcopy(list(conversation_doc.get('legacy_used_documents', []) or [])), + 'used_documents': deepcopy(list(conversation_doc.get('used_documents', []) or [])), 'strict': bool(conversation_doc.get('strict', False)), 'summary': conversation_doc.get('summary'), 'has_unread_assistant_response': False, @@ -959,6 +999,10 @@ def ensure_personal_collaboration_for_legacy_conversation(source_conversation_id collaboration_conversation_doc['source_conversation_id'] = source_conversation_id collaboration_conversation_doc['classification'] = list(source_conversation_doc.get('classification', []) or []) collaboration_conversation_doc['tags'] = list(source_conversation_doc.get('tags', []) or []) + _copy_citation_tracking_conversation_fields( + collaboration_conversation_doc, + source_conversation_doc, + ) collaboration_conversation_doc['strict'] = bool(source_conversation_doc.get('strict', False)) collaboration_conversation_doc['summary'] = source_conversation_doc.get('summary') @@ -1145,6 +1189,10 @@ def ensure_group_collaboration_for_legacy_conversation(source_conversation_id, o collaboration_conversation_doc['classification'] = list(source_conversation_doc.get('classification', []) or []) collaboration_conversation_doc['tags'] = list(source_conversation_doc.get('tags', []) or []) + _copy_citation_tracking_conversation_fields( + collaboration_conversation_doc, + source_conversation_doc, + ) collaboration_conversation_doc['strict'] = bool(source_conversation_doc.get('strict', False)) collaboration_conversation_doc['summary'] = source_conversation_doc.get('summary') collaboration_conversation_doc['legacy_source_conversation_id'] = source_conversation_id @@ -1749,6 +1797,19 @@ def _save_collaboration_message_doc(conversation_doc, message_doc): cosmos_collaboration_messages_container.upsert_item(message_doc) + if ( + str(message_doc.get('role') or '').strip().lower() == 'assistant' + and ( + message_doc.get('citation_tracking_version') is not None + or 'cited_hybrid_citations' in message_doc + ) + ): + initialize_conversation_used_document_tracking(conversation_doc) + merge_cited_documents_into_conversation( + conversation_doc, + message_doc.get('cited_hybrid_citations'), + ) + conversation_doc['last_message_at'] = message_doc.get('timestamp') conversation_doc['last_message_preview'] = ( message_doc.get('metadata', {}).get('last_message_preview', '') @@ -1814,6 +1875,11 @@ def sync_collaboration_conversation_metadata_from_source(conversation_doc, sourc 'classification': deepcopy(list(source_conversation_doc.get('classification', []) or [])), 'summary': deepcopy(source_conversation_doc.get('summary')), } + for field_name in CITATION_TRACKING_CONVERSATION_FIELDS: + if field_name in source_conversation_doc: + metadata_fields[field_name] = deepcopy( + source_conversation_doc.get(field_name) + ) updated = False for field_name, field_value in metadata_fields.items(): @@ -1868,6 +1934,10 @@ def ensure_collaboration_source_conversation(conversation_doc, current_user): 'collaboration_conversation_id': (conversation_doc or {}).get('id'), 'is_hidden': True, } + _copy_citation_tracking_conversation_fields( + source_conversation_doc, + conversation_doc, + ) source_updated = True else: synchronized_values = { @@ -1884,6 +1954,11 @@ def ensure_collaboration_source_conversation(conversation_doc, current_user): 'collaboration_conversation_id': (conversation_doc or {}).get('id'), 'is_hidden': True, } + for field_name in CITATION_TRACKING_CONVERSATION_FIELDS: + if field_name in (conversation_doc or {}): + synchronized_values[field_name] = deepcopy( + conversation_doc.get(field_name) + ) for field_name, field_value in synchronized_values.items(): if source_conversation_doc.get(field_name) != field_value: source_conversation_doc[field_name] = field_value @@ -1952,6 +2027,7 @@ def _refresh_collaboration_conversation_message_summary(conversation_doc): raise ValueError('conversation_id is required') remaining_messages = list_collaboration_messages(conversation_id) + rebuild_conversation_used_documents(conversation_doc, remaining_messages) conversation_doc['message_count'] = len(remaining_messages) if remaining_messages: diff --git a/application/single_app/functions_simplechat_operations.py b/application/single_app/functions_simplechat_operations.py index 03da3a6e5..2aff415c3 100644 --- a/application/single_app/functions_simplechat_operations.py +++ b/application/single_app/functions_simplechat_operations.py @@ -35,6 +35,7 @@ log_workflow_creation, ) from functions_appinsights import log_event +from functions_citation_tracking import rebuild_conversation_used_documents from functions_authentication import ( get_current_user_info, get_graph_endpoint, @@ -93,6 +94,8 @@ PERSONAL_FORK_CONVERSATION_FIELDS = ( "context", "tags", + "used_documents_tracking_version", + "legacy_used_documents", "strict", "classification", "scope_locked", @@ -808,6 +811,11 @@ def fork_personal_conversation_for_user( normalized_user_id, destination_chat_type, ) + rebuild_conversation_used_documents( + fork_conversation, + fork_documents, + rebuild_legacy=True, + ) written_message_ids = [] created_blob_targets: List[Tuple[str, str]] = [] diff --git a/application/single_app/functions_workflow_runner.py b/application/single_app/functions_workflow_runner.py index 83ccfd1ad..2c36b99f0 100644 --- a/application/single_app/functions_workflow_runner.py +++ b/application/single_app/functions_workflow_runner.py @@ -39,6 +39,7 @@ ) from config import ( SECRET_KEY, + TABULAR_EXTENSIONS, VERSION, cognitive_services_scope, cosmos_conversations_container, @@ -54,6 +55,12 @@ build_conversation_context_system_message, serialize_conversation_context_snapshot, ) +from functions_citation_tracking import ( + build_cited_source_subsets, + initialize_conversation_used_document_tracking, + merge_cited_documents_into_conversation, + resolve_citation_location, +) from functions_activity_logging import log_conversation_creation, log_token_usage, log_workflow_run from functions_appinsights import log_event from functions_assistant_table_exports import ( @@ -4470,8 +4477,26 @@ def _mirror_assistant_message_to_personal_conversation( previous_thread_id, ), } + for field_name in ( + 'citation_tracking_version', + 'cited_hybrid_citations', + 'cited_web_search_citations', + ): + if field_name in source_assistant_doc: + if field_name == 'citation_tracking_version': + mirrored_assistant_doc[field_name] = source_assistant_doc.get(field_name) + else: + mirrored_assistant_doc[field_name] = list( + source_assistant_doc.get(field_name) or [] + ) cosmos_messages_container.upsert_item(mirrored_assistant_doc) + if 'citation_tracking_version' in mirrored_assistant_doc: + initialize_conversation_used_document_tracking(conversation_doc) + merge_cited_documents_into_conversation( + conversation_doc, + mirrored_assistant_doc.get('cited_hybrid_citations'), + ) conversation_doc['last_updated'] = timestamp conversation_doc['has_unread_assistant_response'] = True conversation_doc['last_unread_assistant_message_id'] = mirrored_message_id @@ -5833,6 +5858,12 @@ def _create_assistant_message(conversation, workflow, result, trigger_source, ru if generated_file_output.get('output_format') == 'csv': generated_tabular_outputs.append(generated_file_output) web_search_citations = list(result.get('web_search_citations') or []) + hybrid_citations = list(result.get('hybrid_citations') or []) + citation_tracking = build_cited_source_subsets( + result.get('reply', ''), + hybrid_citations=hybrid_citations, + web_search_citations=web_search_citations, + ) source_review_metadata = result.get('source_review') if isinstance(result.get('source_review'), dict) else {} url_access_metadata = result.get('url_access') if isinstance(result.get('url_access'), dict) else {} prepared_agent_citations = _persist_agent_citation_artifacts( @@ -5851,10 +5882,11 @@ def _create_assistant_message(conversation, workflow, result, trigger_source, ru 'content': result.get('reply', ''), 'timestamp': timestamp, 'model_deployment_name': result.get('model_deployment_name'), - 'augmented': bool(result.get('augmented') or result.get('hybrid_citations')), - 'hybrid_citations': list(result.get('hybrid_citations') or []), + 'augmented': bool(result.get('augmented') or hybrid_citations), + 'hybrid_citations': hybrid_citations, 'agent_citations': prepared_agent_citations, 'web_search_citations': web_search_citations, + **citation_tracking, 'agent_display_name': result.get('agent_display_name'), 'agent_name': result.get('agent_name'), 'workspace_type': workspace_type, @@ -5927,6 +5959,11 @@ def _create_assistant_message(conversation, workflow, result, trigger_source, ru conversation['has_unread_assistant_response'] = True conversation['last_unread_assistant_message_id'] = assistant_message_id conversation['last_unread_assistant_at'] = timestamp + initialize_conversation_used_document_tracking(conversation) + merge_cited_documents_into_conversation( + conversation, + citation_tracking['cited_hybrid_citations'], + ) cosmos_conversations_container.upsert_item(conversation) return assistant_doc @@ -6418,12 +6455,25 @@ def _build_workflow_search_citation(result): document_id = str(result.get('document_id') or '').strip() if not document_id: document_id = '_'.join(str(citation_id).split('_')[:-1]) if '_' in str(citation_id) else str(citation_id) + sheet_name = result.get('sheet_name') + page_number = result.get('page_number') or result.get('chunk_sequence') or 1 + file_name = result.get('file_name') or result.get('title') or 'Unknown document' + _, extension = os.path.splitext(str(file_name).strip().lower()) + location_label, location_value = resolve_citation_location( + page_number=page_number, + chunk_text=result.get('chunk_text'), + sheet_name=sheet_name, + is_tabular=extension.lstrip('.') in TABULAR_EXTENSIONS, + ) return { - 'file_name': result.get('file_name') or result.get('title') or 'Unknown document', + 'file_name': file_name, 'document_id': document_id, 'citation_id': citation_id, - 'page_number': result.get('page_number'), + 'page_number': page_number, + 'sheet_name': sheet_name, + 'location_label': location_label, + 'location_value': location_value, 'chunk_id': result.get('chunk_id'), 'chunk_sequence': result.get('chunk_sequence'), 'score': result.get('score'), @@ -6445,9 +6495,18 @@ def _format_workflow_search_results(results): file_name = str(result.get('file_name') or result.get('title') or 'Unknown document').strip() or 'Unknown document' page_number = result.get('page_number') or result.get('chunk_sequence') or 1 + sheet_name = result.get('sheet_name') + _, extension = os.path.splitext(file_name.lower()) + location_label, location_value = resolve_citation_location( + page_number=page_number, + chunk_text=chunk_text, + sheet_name=sheet_name, + is_tabular=extension.lstrip('.') in TABULAR_EXTENSIONS, + ) citation_id = result.get('id') or result.get('chunk_id') or f'workflow-search-{index}' result_lines.append( - f'[{index}] {file_name}, page {page_number}, citation #{citation_id}\n{chunk_text}' + f'[{index}] {chunk_text}\n' + f'(Source: {file_name}, {location_label}: {location_value}) [#{citation_id}]' ) citations.append(_build_workflow_search_citation(result)) @@ -6468,7 +6527,8 @@ def _build_workflow_search_prompt(task_prompt, search_context): prompt_sections = [ '[Workflow document search context]\n' 'Use the bounded native-engine evidence below as grounding for the workflow task. ' - 'When the evidence is insufficient or source coverage is partial, say what is missing instead of guessing.' + 'When the evidence is insufficient or source coverage is partial, say what is missing instead of guessing. ' + 'Preserve the exact (Source: ...) [#citation-id] reference for every excerpt used in the response.' ] if retrieved_content: prompt_sections.append( diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index d675d7292..bad4c20dc 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -179,6 +179,12 @@ inject_conversation_context_message, serialize_conversation_context_snapshot, ) +from functions_citation_tracking import ( + build_cited_source_subsets, + initialize_conversation_used_document_tracking, + merge_cited_documents_into_conversation, + resolve_citation_location, +) from functions_conversation_metadata import collect_conversation_metadata, update_conversation_with_metadata from functions_conversation_unread import mark_conversation_unread from functions_image_messages import build_image_message_documents, decode_image_content @@ -4298,7 +4304,7 @@ def build_tabular_fallback_system_message(tabular_filenames_str, execution_mode= def build_search_augmentation_system_prompt(retrieved_content): """Build the retrieval augmentation prompt without blocking later tool-backed results.""" - return f"""You are an AI assistant. Use the following retrieved document excerpts to answer the user's question. Cite sources using the format (Source: filename, Page: page number). + return f"""You are an AI assistant. Use the following retrieved document excerpts to answer the user's question. Cite sources using the format (Source: filename, Page: page number) [#citation-id]. Copy the exact bracketed citation ID shown after the supporting excerpt. Retrieved Excerpts: {retrieved_content} @@ -4308,7 +4314,7 @@ def build_search_augmentation_system_prompt(retrieved_content): Example User: What is the policy on double dipping? - Assistant: The policy prohibits entities from using federal funds received through one program to apply for additional funds through another program, commonly known as 'double dipping' (Source: PolicyDocument.pdf, Page: 12) + Assistant: The policy prohibits entities from using federal funds received through one program to apply for additional funds through another program, commonly known as 'double dipping' (Source: PolicyDocument.pdf, Page: 12) [#example-citation-id] """ @@ -13046,17 +13052,12 @@ def is_tabular_filename(filename): def get_citation_location(file_name, page_number=None, chunk_text=None, sheet_name=None): """Return a display label/value pair for a citation location.""" - if sheet_name: - return 'Sheet', str(sheet_name) - - normalized_chunk_text = (chunk_text or '').strip() - if is_tabular_filename(file_name) and ( - normalized_chunk_text.startswith('Tabular workbook:') - or normalized_chunk_text.startswith('Tabular data file:') - ): - return 'Location', 'Workbook Schema' - - return 'Page', str(page_number or 1) + return resolve_citation_location( + page_number=page_number, + chunk_text=chunk_text, + sheet_name=sheet_name, + is_tabular=is_tabular_filename(file_name), + ) def get_document_container_for_scope(document_scope): @@ -14220,6 +14221,9 @@ def normalize_terminal_chat_payload(payload): 'augmented': payload.get('augmented', False), 'hybrid_citations': payload.get('hybrid_citations', []), 'web_search_citations': payload.get('web_search_citations', []), + 'citation_tracking_version': payload.get('citation_tracking_version'), + 'cited_hybrid_citations': payload.get('cited_hybrid_citations', []), + 'cited_web_search_citations': payload.get('cited_web_search_citations', []), 'agent_citations': payload.get('agent_citations', []), 'agent_display_name': payload.get('agent_display_name'), 'agent_name': payload.get('agent_name'), @@ -15400,6 +15404,11 @@ def execute_document_action_chat_request( generated_analysis_artifacts=document_generated_analysis_artifacts, generated_tabular_outputs=document_generated_tabular_outputs, ) + document_action_citation_tracking = build_cited_source_subsets( + document_action_reply_content, + hybrid_citations=hybrid_citations_list, + web_search_citations=[], + ) document_action_capability_usage = _build_capability_usage_metadata( workspace_search_used=True, workspace_search_result_count=len(hybrid_citations_list or []), @@ -15419,6 +15428,7 @@ def execute_document_action_chat_request( 'augmented': False, 'hybrid_citations': hybrid_citations_list, 'web_search_citations': [], + **document_action_citation_tracking, 'hybridsearch_query': None, 'agent_citations': prepared_agent_citations, 'model_deployment_name': execution_result.get('model_deployment_name'), @@ -15521,6 +15531,7 @@ def execute_document_action_chat_request( conversation_item['last_updated'] = datetime.utcnow().isoformat() conversation_item['chat_type'] = data.get('chat_type') or conversation_item.get('chat_type') or 'new' + initialize_conversation_used_document_tracking(conversation_item) try: conversation_item = collect_conversation_metadata( @@ -15548,6 +15559,10 @@ def execute_document_action_chat_request( except Exception as exc: debug_print(f'[CHAT_DOCUMENT_ANALYSIS] Conversation metadata update failed: {exc}') + merge_cited_documents_into_conversation( + conversation_item, + document_action_citation_tracking['cited_hybrid_citations'], + ) cosmos_conversations_container.upsert_item(conversation_item) invalidate_conversation_cache_for_item(conversation_item, reason="document_action_chat_completed") debug_print( @@ -15580,6 +15595,7 @@ def execute_document_action_chat_request( 'augmented': False, 'hybrid_citations': hybrid_citations_list, 'web_search_citations': [], + **document_action_citation_tracking, 'agent_citations': prepared_agent_citations, 'reload_messages': False, 'kernel_fallback_notice': None, @@ -17610,6 +17626,9 @@ def result_requires_message_reload(result: Any) -> bool: "document_id": source_doc.get("document_id"), "citation_id": source_doc.get("citation_id"), # Seems like a useful identifier "page_number": source_doc.get("page_number"), + "sheet_name": source_doc.get("sheet_name"), + "location_label": source_doc.get("location_label"), + "location_value": source_doc.get("location_value"), "chunk_id": source_doc.get("chunk_id"), # Specific chunk identifier "chunk_sequence": source_doc.get("chunk_sequence"), # Order within document/group "score": source_doc.get("score"), # Relevance score from search @@ -19807,6 +19826,11 @@ def gpt_error(e): deep_research_query_count=_deep_research_query_count(deep_research_query_plan, deep_research_web_search_runs), ) agent_runtime_metadata = _build_foundry_runtime_metadata(selected_agent) if selected_agent else {} + citation_tracking = build_cited_source_subsets( + ai_message, + hybrid_citations=hybrid_citations_list, + web_search_citations=web_search_citations_list, + ) assistant_doc = make_json_serializable({ 'id': assistant_message_id, @@ -19817,6 +19841,7 @@ def gpt_error(e): 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, # <--- SIMPLIFIED: Directly use the list 'web_search_citations': web_search_citations_list, + **citation_tracking, 'hybridsearch_query': search_query if search_results else None, # Log query when any bounded document retrieval produced results 'agent_citations': prepared_agent_citations, 'model_deployment_name': actual_model_used, @@ -19928,6 +19953,7 @@ def gpt_error(e): # Update conversation's last_updated timestamp one last time conversation_item['last_updated'] = datetime.utcnow().isoformat() + initialize_conversation_used_document_tracking(conversation_item) # Collect comprehensive conversation metadata try: @@ -19971,6 +19997,10 @@ def gpt_error(e): debug_print(f"Error collecting conversation metadata: {e}") # Continue even if metadata collection fails + merge_cited_documents_into_conversation( + conversation_item, + citation_tracking['cited_hybrid_citations'], + ) # Add any other final updates to conversation_item if needed (like classifications if not done earlier) cosmos_conversations_container.upsert_item(conversation_item) invalidate_conversation_cache_for_item(conversation_item, reason="chat_completed") @@ -19999,6 +20029,7 @@ def gpt_error(e): 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, 'web_search_citations': web_search_citations_list, + **citation_tracking, 'source_review': compact_source_review_result_for_metadata(source_review_result), 'deep_research': deep_research_result, 'agent_citations': prepared_agent_citations, @@ -20134,6 +20165,9 @@ def normalize_legacy_chat_payload(payload): 'augmented': payload.get('augmented', False), 'hybrid_citations': payload.get('hybrid_citations', []), 'web_search_citations': payload.get('web_search_citations', []), + 'citation_tracking_version': payload.get('citation_tracking_version'), + 'cited_hybrid_citations': payload.get('cited_hybrid_citations', []), + 'cited_web_search_citations': payload.get('cited_web_search_citations', []), 'agent_citations': payload.get('agent_citations', []), 'agent_display_name': payload.get('agent_display_name'), 'agent_name': payload.get('agent_name'), @@ -20634,6 +20668,39 @@ def build_streaming_capability_usage(): deep_research_query_count=_deep_research_query_count(deep_research_query_plan, deep_research_web_search_runs), ) + def collect_stream_response_conversation_metadata(): + nonlocal conversation_item + source_continuity_refs = None + if ( + is_mixed_source_conversation_continuity_enabled(settings) + and mixed_source_manifest + ): + source_continuity_refs = _build_mixed_source_continuity_refs( + mixed_source_manifest, + mixed_source_evidence_envelopes, + effective_mixed_source_selection_mode, + ) + conversation_item = collect_conversation_metadata( + user_message=user_message, + conversation_id=conversation_id, + user_id=user_id, + active_group_id=effective_active_group_id, + active_group_ids=effective_active_group_ids, + document_scope=effective_document_scope, + selected_document_id=effective_selected_document_id, + model_deployment=final_model_used if use_agent_streaming else gpt_model, + hybrid_search_enabled=mixed_source_document_context_active, + image_gen_enabled=False, + selected_documents=combined_documents if combined_documents else None, + selected_agent=agent_name_used if use_agent_streaming else None, + selected_agent_details=selected_agent_metadata if use_agent_streaming else None, + search_results=search_results if search_results else None, + conversation_item=conversation_item, + active_public_workspace_id=effective_active_public_workspace_id, + active_public_workspace_ids=effective_active_public_workspace_ids, + source_continuity_refs=source_continuity_refs, + ) + # Initialize GPT client (simplified version) gpt_model = "" gpt_client = None @@ -21876,6 +21943,9 @@ def record_and_publish_streaming_thought(thought_payload): "document_id": document_id, "citation_id": citation_id, "page_number": page_number, + "sheet_name": sheet_name, + "location_label": location_label, + "location_value": location_value, "chunk_id": chunk_id, "chunk_sequence": chunk_sequence, "score": score, @@ -22934,6 +23004,7 @@ def finalize_cancelled_stream_response(): cancel_reason = stream_session.get_cancel_reason() if stream_session else 'user_requested' partial_content = accumulated_content.strip() message_persisted = False + partial_citation_tracking = {} cancel_metadata = { 'incomplete': True, 'canceled': True, @@ -22960,6 +23031,11 @@ def finalize_cancelled_stream_response(): if partial_content: assistant_timestamp = datetime.utcnow().isoformat() + partial_citation_tracking = build_cited_source_subsets( + partial_content, + hybrid_citations=hybrid_citations_list, + web_search_citations=web_search_citations_list, + ) prepared_agent_citations = persist_agent_citation_artifacts( conversation_id=conversation_id, assistant_message_id=assistant_message_id, @@ -22980,6 +23056,7 @@ def finalize_cancelled_stream_response(): 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, 'web_search_citations': web_search_citations_list, + **partial_citation_tracking, 'hybridsearch_query': search_query if hybrid_search_enabled and search_results else None, 'agent_citations': prepared_agent_citations, 'model_deployment_name': final_model_used if use_agent_streaming else gpt_model, @@ -23016,6 +23093,24 @@ def finalize_cancelled_stream_response(): }) cosmos_messages_container.upsert_item(assistant_doc) conversation_item['last_updated'] = datetime.utcnow().isoformat() + initialize_conversation_used_document_tracking(conversation_item) + try: + collect_stream_response_conversation_metadata() + except Exception as metadata_error: + log_event( + '[STREAMING] Failed to collect canceled response metadata', + extra={ + 'conversation_id': conversation_id, + 'message_id': assistant_message_id, + 'error_type': type(metadata_error).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + merge_cited_documents_into_conversation( + conversation_item, + partial_citation_tracking['cited_hybrid_citations'], + ) cosmos_conversations_container.upsert_item(conversation_item) invalidate_conversation_cache_for_item(conversation_item, reason="chat_stream_stopped") message_persisted = True @@ -23043,6 +23138,7 @@ def finalize_cancelled_stream_response(): 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, 'web_search_citations': web_search_citations_list, + **partial_citation_tracking, 'agent_citations': agent_citations_list, 'model_deployment_name': final_model_used if use_agent_streaming else gpt_model, 'model_icon': gpt_model_icon, @@ -23666,6 +23762,11 @@ def finalize_cancelled_agent_stream_response(): generated_tabular_outputs=generated_tabular_outputs_list, ) agent_runtime_metadata = _build_foundry_runtime_metadata(selected_agent) if use_agent_streaming else {} + stream_citation_tracking = build_cited_source_subsets( + accumulated_content, + hybrid_citations=hybrid_citations_list, + web_search_citations=web_search_citations_list, + ) assistant_doc = make_json_serializable({ 'id': assistant_message_id, @@ -23676,6 +23777,7 @@ def finalize_cancelled_agent_stream_response(): 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, 'web_search_citations': web_search_citations_list, + **stream_citation_tracking, 'hybridsearch_query': search_query if search_results else None, 'agent_citations': prepared_agent_citations, 'model_deployment_name': final_model_used if use_agent_streaming else gpt_model, @@ -23778,6 +23880,7 @@ def finalize_cancelled_agent_stream_response(): # Update conversation conversation_item['last_updated'] = datetime.utcnow().isoformat() + initialize_conversation_used_document_tracking(conversation_item) try: user_message_doc = cosmos_messages_container.read_item( @@ -23795,39 +23898,14 @@ def finalize_cancelled_agent_stream_response(): debug_print(f"Warning: Could not update streaming user message metadata: {e}") try: - source_continuity_refs = None - if ( - is_mixed_source_conversation_continuity_enabled(settings) - and mixed_source_manifest - ): - source_continuity_refs = _build_mixed_source_continuity_refs( - mixed_source_manifest, - mixed_source_evidence_envelopes, - effective_mixed_source_selection_mode, - ) - conversation_item = collect_conversation_metadata( - user_message=user_message, - conversation_id=conversation_id, - user_id=user_id, - active_group_id=effective_active_group_id, - active_group_ids=effective_active_group_ids, - document_scope=effective_document_scope, - selected_document_id=effective_selected_document_id, - model_deployment=final_model_used if use_agent_streaming else gpt_model, - hybrid_search_enabled=mixed_source_document_context_active, - image_gen_enabled=False, - selected_documents=combined_documents if combined_documents else None, - selected_agent=agent_name_used if use_agent_streaming else None, - selected_agent_details=selected_agent_metadata if use_agent_streaming else None, - search_results=search_results if search_results else None, - conversation_item=conversation_item, - active_public_workspace_id=effective_active_public_workspace_id, - active_public_workspace_ids=effective_active_public_workspace_ids, - source_continuity_refs=source_continuity_refs, - ) + collect_stream_response_conversation_metadata() except Exception as e: debug_print(f"Error collecting conversation metadata: {e}") + merge_cited_documents_into_conversation( + conversation_item, + stream_citation_tracking['cited_hybrid_citations'], + ) if is_personal_chat_conversation(conversation_item): conversation_item = mark_conversation_unread( conversation_item, @@ -23871,6 +23949,7 @@ def finalize_cancelled_agent_stream_response(): 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, 'web_search_citations': web_search_citations_list, + **stream_citation_tracking, 'source_review': compact_source_review_result_for_metadata(source_review_result), 'deep_research': deep_research_result, 'agent_citations': prepared_agent_citations, @@ -23942,10 +24021,18 @@ def finalize_cancelled_agent_stream_response(): debug_print(f"Error during streaming: {error_msg}") # Save partial response if we have content + interrupted_message_persisted = False + interrupted_citation_tracking = {} + interrupted_agent_citations = [] if accumulated_content: current_assistant_thread_id = str(uuid.uuid4()) assistant_timestamp = datetime.utcnow().isoformat() - prepared_agent_citations = persist_agent_citation_artifacts( + interrupted_citation_tracking = build_cited_source_subsets( + accumulated_content, + hybrid_citations=hybrid_citations_list, + web_search_citations=web_search_citations_list, + ) + interrupted_agent_citations = persist_agent_citation_artifacts( conversation_id=conversation_id, assistant_message_id=assistant_message_id, agent_citations=agent_citations_list, @@ -23966,8 +24053,9 @@ def finalize_cancelled_agent_stream_response(): 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, 'web_search_citations': web_search_citations_list, + **interrupted_citation_tracking, 'hybridsearch_query': search_query if hybrid_search_enabled and search_results else None, - 'agent_citations': prepared_agent_citations, + 'agent_citations': interrupted_agent_citations, 'model_deployment_name': final_model_used if use_agent_streaming else gpt_model, 'agent_display_name': agent_display_name_used if use_agent_streaming else None, 'agent_name': agent_name_used if use_agent_streaming else None, @@ -23991,12 +24079,68 @@ def finalize_cancelled_agent_stream_response(): }) try: cosmos_messages_container.upsert_item(assistant_doc) - except Exception as ex: - pass + interrupted_message_persisted = True + conversation_item['last_updated'] = assistant_timestamp + initialize_conversation_used_document_tracking( + conversation_item + ) + try: + collect_stream_response_conversation_metadata() + except Exception as metadata_error: + log_event( + '[STREAMING] Failed to collect interrupted response metadata', + extra={ + 'conversation_id': conversation_id, + 'message_id': assistant_message_id, + 'error_type': type( + metadata_error + ).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + merge_cited_documents_into_conversation( + conversation_item, + interrupted_citation_tracking[ + 'cited_hybrid_citations' + ], + ) + cosmos_conversations_container.upsert_item( + conversation_item + ) + invalidate_conversation_cache_for_item( + conversation_item, + reason="chat_stream_interrupted", + ) + except Exception as persistence_error: + log_event( + '[STREAMING] Failed to persist interrupted response metadata', + extra={ + 'conversation_id': conversation_id, + 'message_id': assistant_message_id, + 'error_type': type( + persistence_error + ).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) yield build_stream_error_event( CLIENT_SAFE_STREAM_ERROR_MESSAGE, partial_content=accumulated_content, + conversation_id=conversation_id, + user_message_id=user_message_id, + message_id=( + assistant_message_id + if interrupted_message_persisted + else None + ), + message_persisted=interrupted_message_persisted, + hybrid_citations=hybrid_citations_list, + web_search_citations=web_search_citations_list, + agent_citations=interrupted_agent_citations, + **interrupted_citation_tracking, ) except Exception as e: diff --git a/application/single_app/route_backend_collaboration.py b/application/single_app/route_backend_collaboration.py index bf1f20af3..6b45af1bc 100644 --- a/application/single_app/route_backend_collaboration.py +++ b/application/single_app/route_backend_collaboration.py @@ -1507,7 +1507,13 @@ def transform_event_block(event_block): except json.JSONDecodeError: return normalized_event_block + '\n\n' - if stream_payload.get('error'): + if ( + stream_payload.get('error') + and not ( + stream_payload.get('message_persisted') + and stream_payload.get('message_id') + ) + ): return _serialize_stream_error( stream_payload.get('error'), partial_content=stream_payload.get('partial_content'), @@ -1515,6 +1521,8 @@ def transform_event_block(event_block): message_persisted=True, conversation_id=conversation_id, ) + if stream_payload.get('error'): + stream_payload['done'] = True if stream_payload.get('type') == USER_MESSAGE_PERSISTED_EVENT_TYPE: return None @@ -1627,6 +1635,7 @@ def transform_event_block(event_block): **stream_payload, 'conversation_id': conversation_id, 'conversation_title': serialized_final_conversation.get('title'), + 'conversation_kind': serialized_final_conversation.get('conversation_kind'), 'chat_type': serialized_final_conversation.get('chat_type'), 'classification': serialized_final_conversation.get('classification', []), 'context': serialized_final_conversation.get('context', []), @@ -1638,12 +1647,15 @@ def transform_event_block(event_block): 'augmented': serialized_assistant_message.get('augmented', False), 'hybrid_citations': serialized_assistant_message.get('hybrid_citations', []), 'web_search_citations': serialized_assistant_message.get('web_search_citations', []), + 'citation_tracking_version': serialized_assistant_message.get('citation_tracking_version'), + 'cited_hybrid_citations': serialized_assistant_message.get('cited_hybrid_citations', []), + 'cited_web_search_citations': serialized_assistant_message.get('cited_web_search_citations', []), 'agent_citations': serialized_assistant_message.get('agent_citations', []), 'agent_display_name': serialized_assistant_message.get('agent_display_name'), 'agent_name': serialized_assistant_message.get('agent_name'), 'full_content': serialized_assistant_message.get('content') if serialized_assistant_message.get('role') != 'image' else stream_payload.get('full_content', ''), 'image_url': serialized_assistant_message.get('content') if serialized_assistant_message.get('role') == 'image' else stream_payload.get('image_url'), - 'reload_messages': False, + 'reload_messages': bool(stream_payload.get('error')), } return f'data: {json.dumps(make_json_serializable(transformed_payload))}\n\n' diff --git a/application/single_app/route_backend_conversation_export.py b/application/single_app/route_backend_conversation_export.py index 68b2b855d..56af5c671 100644 --- a/application/single_app/route_backend_conversation_export.py +++ b/application/single_app/route_backend_conversation_export.py @@ -30,6 +30,10 @@ is_collaboration_conversation, list_collaboration_messages, ) +from functions_citation_tracking import ( + get_message_reference_citation_buckets, + get_message_source_citation_buckets, +) from functions_conversation_metadata import update_conversation_with_metadata from functions_debug import debug_print from functions_group import get_group_model_endpoints, get_user_groups @@ -565,8 +569,9 @@ def _sanitize_message( ) -> Dict[str, Any]: role = message.get('role', '') content = message.get('content', '') - raw_citation_buckets = _collect_raw_citation_buckets(message) - normalized_citations = _normalize_citations(raw_citation_buckets) + reference_citation_buckets = _collect_raw_citation_buckets(message) + source_citation_buckets = _collect_source_citation_buckets(message) + normalized_citations = _normalize_citations(reference_citation_buckets) citation_counts = _build_citation_counts(normalized_citations) details = _curate_message_details(message, citation_counts, len(thoughts)) @@ -585,10 +590,21 @@ def _sanitize_message( 'citations': normalized_citations, 'citation_counts': citation_counts, 'thoughts': thoughts, - 'legacy_citations': raw_citation_buckets['legacy'], - 'hybrid_citations': raw_citation_buckets['hybrid'], - 'web_search_citations': raw_citation_buckets['web'], - 'agent_citations': raw_citation_buckets['agent'] + 'citation_tracking_version': message.get('citation_tracking_version'), + 'cited_hybrid_citations': ( + list(message.get('cited_hybrid_citations')) + if isinstance(message.get('cited_hybrid_citations'), list) + else [] + ), + 'cited_web_search_citations': ( + list(message.get('cited_web_search_citations')) + if isinstance(message.get('cited_web_search_citations'), list) + else [] + ), + 'legacy_citations': source_citation_buckets['legacy'], + 'hybrid_citations': source_citation_buckets['hybrid'], + 'web_search_citations': source_citation_buckets['web'], + 'agent_citations': source_citation_buckets['agent'] } @@ -604,17 +620,13 @@ def _sanitize_thought(thought: Dict[str, Any]) -> Dict[str, Any]: def _collect_raw_citation_buckets(message: Dict[str, Any]) -> Dict[str, List[Any]]: - def ensure_list(value: Any) -> List[Any]: - if not value: - return [] - return value if isinstance(value, list) else [value] + """Return citation records that are valid references for this message.""" + return get_message_reference_citation_buckets(message) - return { - 'legacy': ensure_list(message.get('citations')), - 'hybrid': ensure_list(message.get('hybrid_citations')), - 'web': ensure_list(message.get('web_search_citations')), - 'agent': ensure_list(message.get('agent_citations')) - } + +def _collect_source_citation_buckets(message: Dict[str, Any]) -> Dict[str, List[Any]]: + """Return complete source and execution records for JSON audit output.""" + return get_message_source_citation_buckets(message) def _normalize_citations(raw_citation_buckets: Dict[str, List[Any]]) -> List[Dict[str, Any]]: @@ -627,7 +639,11 @@ def _normalize_citations(raw_citation_buckets: Dict[str, List[Any]]) -> List[Dic 'label': _build_document_citation_label(citation), 'file_name': citation.get('file_name'), 'title': citation.get('title') or citation.get('file_name'), + 'document_id': citation.get('document_id'), 'page_number': citation.get('page_number'), + 'sheet_name': citation.get('sheet_name'), + 'location_label': citation.get('location_label'), + 'location_value': citation.get('location_value'), 'citation_id': citation.get('citation_id'), 'chunk_id': citation.get('chunk_id'), 'metadata_type': citation.get('metadata_type'), @@ -668,6 +684,9 @@ def _normalize_citations(raw_citation_buckets: Dict[str, List[Any]]) -> List[Dic 'tool_name': citation.get('tool_name'), 'function_name': citation.get('function_name'), 'plugin_name': citation.get('plugin_name'), + 'function_arguments': citation.get('function_arguments'), + 'function_result': citation.get('function_result'), + 'error_message': citation.get('error_message'), 'success': citation.get('success'), 'timestamp': citation.get('timestamp') }) @@ -701,12 +720,20 @@ def _normalize_citations(raw_citation_buckets: Dict[str, List[Any]]) -> List[Dic def _build_document_citation_label(citation: Dict[str, Any]) -> str: file_name = citation.get('file_name') or citation.get('title') or 'Document source' metadata_type = citation.get('metadata_type') - page_number = citation.get('page_number') + location_label = ( + citation.get('location_label') + or ('Sheet' if citation.get('sheet_name') else 'Page') + ) + location_value = ( + citation.get('location_value') + or citation.get('sheet_name') + or citation.get('page_number') + ) if metadata_type: return f"{file_name} — {metadata_type.replace('_', ' ').title()}" - if page_number not in (None, ''): - return f"{file_name} — Page {page_number}" + if location_value not in (None, ''): + return f"{file_name} — {location_label}: {location_value}" return file_name @@ -1624,7 +1651,7 @@ def _conversation_to_markdown(entry: Dict[str, Any]) -> str: def _append_citations_markdown(lines: List[str], message: Dict[str, Any]): document_citations = [citation for citation in message.get('citations', []) if citation.get('citation_type') == 'document'] web_citations = [citation for citation in message.get('citations', []) if citation.get('citation_type') == 'web'] - agent_citations = message.get('agent_citations', []) or [] + agent_citations = [citation for citation in message.get('citations', []) if citation.get('citation_type') == 'agent_tool'] legacy_citations = [citation for citation in message.get('citations', []) if citation.get('citation_type') == 'legacy'] if not any([document_citations, web_citations, agent_citations, legacy_citations]): @@ -1632,13 +1659,16 @@ def _append_citations_markdown(lines: List[str], message: Dict[str, Any]): return if document_citations: - lines.append('#### Document Sources') + lines.append('#### Document References') lines.append('') for index, citation in enumerate(document_citations, start=1): lines.append(f"{index}. **{citation.get('label', 'Document source')}**") detail_mapping = _remove_empty_values({ 'citation_id': citation.get('citation_id'), 'page_number': citation.get('page_number'), + 'sheet_name': citation.get('sheet_name'), + 'location_label': citation.get('location_label'), + 'location_value': citation.get('location_value'), 'classification': citation.get('classification'), 'score': citation.get('score'), 'metadata_type': citation.get('metadata_type') @@ -1650,7 +1680,7 @@ def _append_citations_markdown(lines: List[str], message: Dict[str, Any]): lines.append('') if web_citations: - lines.append('#### Web Sources') + lines.append('#### Web References') lines.append('') for index, citation in enumerate(web_citations, start=1): title = citation.get('title') or citation.get('label') or 'Web source' @@ -5565,7 +5595,7 @@ def _append_html_citations(parts: List[str], message: Dict[str, Any]): legacy_citations = [c for c in citations if c.get('citation_type') == 'legacy'] if doc_citations: - parts.append('

Document Sources

') + parts.append('

Document References

') parts.append('
    ') for citation in doc_citations: parts.append( @@ -5574,6 +5604,9 @@ def _append_html_citations(parts: List[str], message: Dict[str, Any]): detail_items = _remove_empty_values({ 'citation_id': citation.get('citation_id'), 'page_number': citation.get('page_number'), + 'sheet_name': citation.get('sheet_name'), + 'location_label': citation.get('location_label'), + 'location_value': citation.get('location_value'), 'classification': citation.get('classification'), 'score': citation.get('score'), 'metadata_type': citation.get('metadata_type') @@ -5591,7 +5624,7 @@ def _append_html_citations(parts: List[str], message: Dict[str, Any]): parts.append('
') if web_citations: - parts.append('

Web Sources

') + parts.append('

Web References

') parts.append('
    ') for citation in web_citations: title = _escape_html( diff --git a/application/single_app/route_backend_conversations.py b/application/single_app/route_backend_conversations.py index b286afc04..d00b78534 100644 --- a/application/single_app/route_backend_conversations.py +++ b/application/single_app/route_backend_conversations.py @@ -31,6 +31,7 @@ tag_conversation_feed_source, ) from functions_conversation_metadata import get_conversation_metadata, update_conversation_with_metadata +from functions_citation_tracking import rebuild_conversation_used_documents from functions_conversation_unread import clear_conversation_unread, normalize_conversation_unread_state from functions_conversation_cache import ( build_conversation_cache_key, @@ -822,6 +823,63 @@ def _authorize_personal_conversation_read(user_id, conversation_id): return conversation_item +def _rebuild_authorized_personal_conversation_used_documents( + user_id, + conversation_id, +): + """Persist exact used documents after an authorized message mutation.""" + try: + conversation_item = _authorize_personal_conversation_read( + user_id, + conversation_id, + ) + messages = list(cosmos_messages_container.query_items( + query=( + "SELECT * FROM c " + "WHERE c.conversation_id = @conversation_id" + ), + parameters=[ + { + "name": "@conversation_id", + "value": conversation_id, + }, + ], + partition_key=conversation_id, + )) + rebuild_conversation_used_documents(conversation_item, messages) + conversation_item.pop('used_documents_rebuild_required', None) + conversation_item['last_updated'] = datetime.utcnow().isoformat() + cosmos_conversations_container.upsert_item(conversation_item) + return conversation_item + except Exception as rebuild_error: + if 'conversation_item' in locals() and isinstance(conversation_item, dict): + conversation_item['used_documents_rebuild_required'] = True + try: + cosmos_conversations_container.upsert_item(conversation_item) + except Exception as marker_error: + log_event( + "[CONVERSATION_METADATA] Failed to mark used documents for rebuild", + extra={ + "conversation_id": conversation_id, + "user_id": user_id, + "error_type": type(marker_error).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + log_event( + "[CONVERSATION_METADATA] Failed to rebuild used documents after message mutation", + extra={ + "conversation_id": conversation_id, + "user_id": user_id, + "error_type": type(rebuild_error).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + return None + + def _invalidate_conversation_cache_after_message_mutation(conversation_id, user_id, reason): """Invalidate conversation caches after message-level changes without failing the caller.""" try: @@ -1800,6 +1858,16 @@ def get_conversation_metadata_api(conversation_id): # Ensure that the conversation belongs to the current user if conversation_item.get('user_id') != user_id: return jsonify({'error': 'Forbidden'}), 403 + + if conversation_item.get('used_documents_rebuild_required') is True: + rebuilt_conversation = ( + _rebuild_authorized_personal_conversation_used_documents( + user_id, + conversation_id, + ) + ) + if rebuilt_conversation: + conversation_item = rebuilt_conversation _, updated = normalize_chat_type(conversation_item) if updated: @@ -1828,6 +1896,9 @@ def get_conversation_metadata_api(conversation_id): "classification": conversation_item.get('classification', []), "context": conversation_item.get('context', []), "tags": conversation_item.get('tags', []), + "used_documents_tracking_version": conversation_item.get('used_documents_tracking_version'), + "legacy_used_documents": conversation_item.get('legacy_used_documents', []), + "used_documents": conversation_item.get('used_documents', []), "strict": conversation_item.get('strict', False), "is_pinned": conversation_item.get('is_pinned', False), "is_hidden": conversation_item.get('is_hidden', False), @@ -2609,6 +2680,10 @@ def delete_message(message_id): deleted_message_ids.append(msg_id) + _rebuild_authorized_personal_conversation_used_documents( + user_id, + conversation_id, + ) _invalidate_conversation_cache_after_message_mutation( conversation_id, user_id, @@ -2787,6 +2862,10 @@ def retry_message(message_id): } cosmos_messages_container.upsert_item(new_user_message) + _rebuild_authorized_personal_conversation_used_documents( + user_id, + conversation_id, + ) _invalidate_conversation_cache_after_message_mutation( conversation_id, user_id, @@ -3006,6 +3085,10 @@ def edit_message(message_id): } cosmos_messages_container.upsert_item(new_user_message) + _rebuild_authorized_personal_conversation_used_documents( + user_id, + conversation_id, + ) _invalidate_conversation_cache_after_message_mutation( conversation_id, user_id, @@ -3170,6 +3253,10 @@ def switch_attempt(message_id): msg['metadata']['thread_info']['active_thread'] = (msg_attempt == target_attempt) cosmos_messages_container.upsert_item(msg) + _rebuild_authorized_personal_conversation_used_documents( + user_id, + conversation_id, + ) _invalidate_conversation_cache_after_message_mutation( conversation_id, user_id, diff --git a/application/single_app/static/js/chat/chat-citations.js b/application/single_app/static/js/chat/chat-citations.js index c51927233..edd6ac293 100644 --- a/application/single_app/static/js/chat/chat-citations.js +++ b/application/single_app/static/js/chat/chat-citations.js @@ -42,7 +42,7 @@ export function parseDocIdAndPage(citationId) { export function parseCitations(message) { // ... (keep existing implementation) - const citationRegex = /\(Source:\s*([^,]+),\s*(Page(?:s)?|Sheet(?:s)?|Location):\s*([^)]+)\)\s*((?:\[#.*?\]\s*)+)/gi; + const citationRegex = /\(Source:\s*((?:(?!\(Source:).)+?),\s*(Page(?:s)?|Sheet(?:s)?|Location):\s*((?:(?!\(Source:).)+?)\)\s*((?:\[#.*?\]\s*)+)/gi; let result = message.replace(citationRegex, (whole, filename, locationLabel, locations, bracketSection) => { const trimmedFilename = filename.trim(); @@ -80,7 +80,12 @@ export function parseCitations(message) { } const normalizedLocationLabel = locationLabel.toLowerCase(); - const locationTokens = locations.split(/,/).map(tok => tok.trim()); + const locationTokens = ( + normalizedLocationLabel === 'sheet' + || normalizedLocationLabel === 'location' + ) + ? [locations.trim()] + : locations.split(/,/).map(tok => tok.trim()); const linkedTokens = locationTokens.map((token, index) => { if (!normalizedLocationLabel.startsWith('page')) { const ref = orderedRefs[index] || orderedRefs[0]; diff --git a/application/single_app/static/js/chat/chat-collaboration.js b/application/single_app/static/js/chat/chat-collaboration.js index 1dff3c9f1..d2ae11ac5 100644 --- a/application/single_app/static/js/chat/chat-collaboration.js +++ b/application/single_app/static/js/chat/chat-collaboration.js @@ -1074,6 +1074,24 @@ function handleConversationEvent(eventEnvelope = {}) { const decoratedMessage = decorateReplyMessage(payload.message); cacheCollaborationMessage(payload.message); + if (String(payload.message.role || '').trim().toLowerCase() === 'assistant') { + const conversationId = ( + eventEnvelope.conversation_id + || payload.message.conversation_id + || payload.conversation?.id + || '' + ); + window.dispatchEvent(new CustomEvent( + 'chat:conversation-documents-refresh', + { + detail: { + conversationId, + autoOpen: false, + conversationKind: 'collaborative', + }, + } + )); + } if (reconcilePendingCollaborativeUserMessage(payload.message)) { const messageKind = String( payload.message.message_kind || payload.message.metadata?.message_kind || '' @@ -1112,6 +1130,16 @@ function handleConversationEvent(eventEnvelope = {}) { if (eventEnvelope.event_type === 'collaboration.message.deleted' && payload.message_id) { removeCollaborationMessage(payload.message_id); + window.dispatchEvent(new CustomEvent( + 'chat:conversation-documents-refresh', + { + detail: { + conversationId: eventEnvelope.conversation_id || '', + autoOpen: false, + conversationKind: 'collaborative', + }, + } + )); if (payload.deleted_by_user_id && payload.deleted_by_user_id !== getCurrentUserId()) { showToast('A shared message was deleted.', 'info'); } @@ -1937,6 +1965,7 @@ window.chatCollaboration = { deactivateConversation, fetchCollaborationConversationList, fetchConversationMetadata, + loadConversationMessages, getPendingMessageContext, handleComposerBlur, handleComposerInput, diff --git a/application/single_app/static/js/chat/chat-conversation-contents.js b/application/single_app/static/js/chat/chat-conversation-contents.js index 97d1a9016..3034d119e 100644 --- a/application/single_app/static/js/chat/chat-conversation-contents.js +++ b/application/single_app/static/js/chat/chat-conversation-contents.js @@ -3,7 +3,10 @@ import { extractPageNumbers, fetchConversationMetadata, - getConversationDocumentTags, + getConversationExactUsedDocuments, + getConversationLegacyUsedDocuments, + getConversationUsedDocuments, + hasConversationUsedDocumentTracking, } from "./chat-conversation-details.js"; import { isColorLight } from "./chat-utils.js"; @@ -47,6 +50,9 @@ let restoreFocusAfterClose = true; let metadataRequestToken = 0; let documentLoadState = "idle"; let activeDocumentConversationId = ""; +let documentUsageTrackingAvailable = false; +let exactDocumentUsageAvailable = false; +let legacyDocumentFallbackAvailable = false; const autoOpenedDocumentConversationIds = new Set(); export function normalizeConversationContentsLabel(value, fallbackLabel = "User message") { @@ -178,9 +184,15 @@ function syncDrawerModeControls() { drawerTitle.textContent = contentsActive ? "Conversation contents" : "Used documents"; } if (drawerSubtitle) { + let documentsSubtitle = "Documents associated with this historical conversation"; + if (documentUsageTrackingAvailable && exactDocumentUsageAvailable) { + documentsSubtitle = legacyDocumentFallbackAvailable + ? "Cited documents and historical associations" + : "Documents cited in active responses"; + } drawerSubtitle.textContent = contentsActive ? "Jump to a user message" - : "Documents cited in this conversation"; + : documentsSubtitle; } contentsPanel?.classList.toggle("d-none", !contentsActive); @@ -428,9 +440,62 @@ function createDocumentMetaLine(iconName, text) { return line; } +function getDocumentCitationLocationLines(doc) { + const groupedLocations = new Map(); + const appendLocation = (label, value) => { + const normalizedLabel = String(label || "Location").trim() || "Location"; + const normalizedValue = String(value ?? "").trim(); + if (!normalizedValue) { + return; + } + const values = groupedLocations.get(normalizedLabel) || []; + if (!values.includes(normalizedValue)) { + values.push(normalizedValue); + groupedLocations.set(normalizedLabel, values); + } + }; + + const citationLocations = Array.isArray(doc?.citation_locations) + ? doc.citation_locations + : []; + citationLocations.forEach(location => { + if (!location || typeof location !== "object") { + return; + } + const label = location.location_label + || (location.sheet_name ? "Sheet" : "Page"); + const value = location.location_value + || location.sheet_name + || location.page_number; + appendLocation(label, value); + }); + + if (groupedLocations.size === 0) { + (Array.isArray(doc?.sheet_names) ? doc.sheet_names : []) + .forEach(value => appendLocation("Sheet", value)); + (Array.isArray(doc?.page_numbers) ? doc.page_numbers : []) + .forEach(value => appendLocation("Page", value)); + } + + if (groupedLocations.size === 0) { + const chunkIds = Array.isArray(doc?.chunk_ids) ? doc.chunk_ids : []; + extractPageNumbers(chunkIds).forEach(value => appendLocation("Page", value)); + } + + return Array.from(groupedLocations.entries()).map(([label, values]) => { + const normalizedLabel = label.toLowerCase(); + const displayLabel = normalizedLabel === "page" + ? "Pages" + : normalizedLabel === "sheet" + ? "Sheets" + : normalizedLabel === "location" + ? "Locations" + : label; + return `${displayLabel}: ${values.join(", ")}`; + }); +} + function createDocumentEntry(doc) { - const chunkIds = Array.isArray(doc?.chunk_ids) ? doc.chunk_ids : []; - const chunkPages = extractPageNumbers(chunkIds); const documentId = String(doc?.document_id || "Unknown Document"); const documentTitle = String(doc?.title || doc?.file_name || documentId); const documentFileName = String(doc?.file_name || ""); @@ -460,9 +525,9 @@ function createDocumentEntry(doc) { ) { entry.appendChild(createDocumentMetaLine("file-earmark", documentFileName)); } - if (chunkPages.length > 0) { - entry.appendChild(createDocumentMetaLine("file-earmark-text", `Pages: ${chunkPages.join(", ")}`)); - } + getDocumentCitationLocationLines(doc).forEach(locationLine => { + entry.appendChild(createDocumentMetaLine("file-earmark-text", locationLine)); + }); entry.appendChild(createDocumentMetaLine( getScopeIcon(scopeType), `${scopeType} scope: ${scopeName}` @@ -503,6 +568,7 @@ function renderConversationDocuments(documents) { async function refreshConversationDocuments(options = {}) { const conversationId = String(options.conversationId || getCurrentConversationId()).trim(); + const conversationKind = String(options.conversationKind || "").trim(); const requestToken = ++metadataRequestToken; const autoOpen = Boolean(options.autoOpen); const conversationChanged = conversationId !== activeDocumentConversationId; @@ -510,6 +576,9 @@ async function refreshConversationDocuments(options = {}) { activeDocumentConversationId = conversationId; if (!conversationId) { documentLoadState = "idle"; + documentUsageTrackingAvailable = false; + exactDocumentUsageAvailable = false; + legacyDocumentFallbackAvailable = false; renderConversationDocuments([]); return; } @@ -527,18 +596,26 @@ async function refreshConversationDocuments(options = {}) { } try { - const metadata = await fetchConversationMetadata(conversationId); + const metadata = await fetchConversationMetadata( + conversationId, + { conversationKind } + ); if (requestToken !== metadataRequestToken || conversationId !== activeDocumentConversationId) { return; } documentLoadState = "ready"; - const documents = getConversationDocumentTags(metadata); + documentUsageTrackingAvailable = hasConversationUsedDocumentTracking(metadata); + const exactDocuments = getConversationExactUsedDocuments(metadata); + const legacyDocuments = getConversationLegacyUsedDocuments(metadata); + exactDocumentUsageAvailable = exactDocuments.length > 0; + legacyDocumentFallbackAvailable = legacyDocuments.length > 0; + const documents = getConversationUsedDocuments(metadata); renderConversationDocuments(documents); if ( autoOpen - && documents.length > 0 + && exactDocuments.length > 0 && conversationId === getCurrentConversationId() && !autoOpenedDocumentConversationIds.has(conversationId) ) { @@ -551,6 +628,9 @@ async function refreshConversationDocuments(options = {}) { } documentLoadState = "error"; + documentUsageTrackingAvailable = false; + exactDocumentUsageAvailable = false; + legacyDocumentFallbackAvailable = false; documentEntries = []; documentsList?.replaceChildren(); documentsEmptyState?.classList.add("d-none"); @@ -658,6 +738,7 @@ function initializeConversationContents() { void refreshConversationDocuments({ conversationId: event.detail?.conversationId || "", autoOpen: false, + conversationKind: event.detail?.conversationKind || "", }); }); window.addEventListener("chat:conversation-documents-refresh", event => { @@ -668,6 +749,7 @@ function initializeConversationContents() { void refreshConversationDocuments({ conversationId, autoOpen: Boolean(event.detail?.autoOpen), + conversationKind: event.detail?.conversationKind || "", }); }); diff --git a/application/single_app/static/js/chat/chat-conversation-details.js b/application/single_app/static/js/chat/chat-conversation-details.js index da3f195b3..3101e0791 100644 --- a/application/single_app/static/js/chat/chat-conversation-details.js +++ b/application/single_app/static/js/chat/chat-conversation-details.js @@ -41,14 +41,17 @@ function getConversationMetadataDomItem(conversationId) { || document.querySelector(`.sidebar-conversation-item[data-conversation-id="${escapedConversationId}"]`); } -export async function fetchConversationMetadata(conversationId) { +export async function fetchConversationMetadata(conversationId, options = {}) { const normalizedConversationId = String(conversationId || '').trim(); if (!normalizedConversationId) { throw new Error('Conversation ID is required'); } const conversationItem = getConversationMetadataDomItem(normalizedConversationId); - const isCollaborativeConversation = conversationItem?.dataset?.conversationKind === 'collaborative'; + const isCollaborativeConversation = ( + options.conversationKind === 'collaborative' + || conversationItem?.dataset?.conversationKind === 'collaborative' + ); if (isCollaborativeConversation && window.chatCollaboration?.fetchConversationMetadata) { return window.chatCollaboration.fetchConversationMetadata(normalizedConversationId); @@ -68,6 +71,57 @@ export function getConversationDocumentTags(metadata = {}) { return tags.filter(tag => tag?.category === 'document'); } +export function hasConversationUsedDocumentTracking(metadata = {}) { + return Number(metadata.used_documents_tracking_version || 0) >= 1; +} + +export function getConversationExactUsedDocuments(metadata = {}) { + if (!hasConversationUsedDocumentTracking(metadata)) { + return []; + } + return Array.isArray(metadata.used_documents) ? metadata.used_documents : []; +} + +export function getConversationLegacyUsedDocuments(metadata = {}) { + if (!hasConversationUsedDocumentTracking(metadata)) { + return getConversationDocumentTags(metadata); + } + return Array.isArray(metadata.legacy_used_documents) + ? metadata.legacy_used_documents + : []; +} + +function mergeConversationDocumentCollections(...collections) { + const documentsById = new Map(); + collections.forEach(collection => { + if (!Array.isArray(collection)) { + return; + } + collection.forEach(doc => { + const documentId = String(doc?.document_id || '').trim(); + if (!documentId) { + return; + } + const existingDocument = documentsById.get(documentId) || {}; + documentsById.set(documentId, { + ...existingDocument, + ...doc, + }); + }); + }); + return Array.from(documentsById.values()); +} + +export function getConversationUsedDocuments(metadata = {}) { + if (!hasConversationUsedDocumentTracking(metadata)) { + return getConversationDocumentTags(metadata); + } + return mergeConversationDocumentCollections( + getConversationLegacyUsedDocuments(metadata), + getConversationExactUsedDocuments(metadata), + ); +} + export function hideConversationDetails() { const { modal } = getConversationDetailsModalElements(); const modalInstance = getConversationDetailsModalInstance(); @@ -355,6 +409,14 @@ function renderConversationMetadata(metadata, conversationId) { } const documentTags = getConversationDocumentTags(metadata); + const exactUsedDocumentIds = new Set( + getConversationExactUsedDocuments(metadata) + .map(doc => String(doc?.document_id || '').trim()) + .filter(Boolean) + ); + const documentUsageNote = hasConversationUsedDocumentTracking(metadata) + ? 'All returned source documents are listed. Cited marks documents referenced by active responses.' + : 'All associated source documents are listed. Exact citation use was not tracked for this historical conversation.'; // Documents Section if (documentTags.length > 0) { @@ -362,10 +424,11 @@ function renderConversationMetadata(metadata, conversationId) {
    -
    Documents
    +
    Source documents
    - ${renderDocumentsSection(documentTags)} +

    ${documentUsageNote}

    + ${renderDocumentsSection(documentTags, exactUsedDocumentIds)}
    @@ -748,7 +811,7 @@ function renderModelsAndAgentsSection(models, agents) { /** * Render documents section */ -function renderDocumentsSection(documents) { +function renderDocumentsSection(documents, citedDocumentIds = new Set()) { let html = ''; documents.forEach(doc => { @@ -761,6 +824,10 @@ function renderDocumentsSection(documents) { const safeDocumentTitle = escapeHtml(documentTitle); const safeScopeName = escapeHtml(scopeName); const safeScopeType = escapeHtml(doc.scope?.type || 'Unknown'); + const isCited = citedDocumentIds.has(String(doc.document_id || '').trim()); + const citedBadgeHtml = isCited + ? 'Cited' + : ''; // Format document classification with custom colors const allCategories = window.classification_categories || []; @@ -778,7 +845,10 @@ function renderDocumentsSection(documents) {
    ${safeDocumentTitle}
    - ${classificationHtml} +
    + ${citedBadgeHtml} + ${classificationHtml} +
    diff --git a/application/single_app/static/js/chat/chat-edit.js b/application/single_app/static/js/chat/chat-edit.js index f8d109a77..f4173cc6f 100644 --- a/application/single_app/static/js/chat/chat-edit.js +++ b/application/single_app/static/js/chat/chat-edit.js @@ -147,6 +147,15 @@ window.executeMessageEdit = function() { console.log(' retry_thread_id:', data.chat_request.retry_thread_id); console.log(' retry_thread_attempt:', data.chat_request.retry_thread_attempt); console.log(' Full chat_request:', data.chat_request); + window.dispatchEvent(new CustomEvent( + 'chat:conversation-documents-refresh', + { + detail: { + conversationId: data.chat_request.conversation_id, + autoOpen: false, + }, + } + )); sendMessageWithStreaming( data.chat_request, diff --git a/application/single_app/static/js/chat/chat-messages.js b/application/single_app/static/js/chat/chat-messages.js index ae90c097e..da6310144 100644 --- a/application/single_app/static/js/chat/chat-messages.js +++ b/application/single_app/static/js/chat/chat-messages.js @@ -9571,6 +9571,17 @@ function executeMessageDeletion(deleteThread = false) { if (typeof loadConversations === 'function') { loadConversations(); } + if (window.currentConversationId) { + window.dispatchEvent(new CustomEvent( + 'chat:conversation-documents-refresh', + { + detail: { + conversationId: window.currentConversationId, + autoOpen: false, + }, + } + )); + } } else { showToast('Failed to delete message', 'error'); } diff --git a/application/single_app/static/js/chat/chat-retry.js b/application/single_app/static/js/chat/chat-retry.js index 6dcdebd9e..99d3cfdf3 100644 --- a/application/single_app/static/js/chat/chat-retry.js +++ b/application/single_app/static/js/chat/chat-retry.js @@ -320,6 +320,15 @@ window.executeMessageRetry = function() { console.log(' retry_thread_id:', data.chat_request.retry_thread_id); console.log(' retry_thread_attempt:', data.chat_request.retry_thread_attempt); console.log(' Full chat_request:', data.chat_request); + window.dispatchEvent(new CustomEvent( + 'chat:conversation-documents-refresh', + { + detail: { + conversationId: data.chat_request.conversation_id, + autoOpen: false, + }, + } + )); sendMessageWithStreaming( data.chat_request, @@ -393,6 +402,15 @@ export function handleCarouselNavigation(messageDiv, messageId, direction) { }) .then(data => { console.log(`✅ Switched to attempt ${data.new_active_attempt}:`, data); + window.dispatchEvent(new CustomEvent( + 'chat:conversation-documents-refresh', + { + detail: { + conversationId: window.currentConversationId || '', + autoOpen: false, + }, + } + )); // Reload messages to show new active attempt if (window.currentConversationId) { diff --git a/application/single_app/static/js/chat/chat-streaming.js b/application/single_app/static/js/chat/chat-streaming.js index 15ab2aeff..a2a09b493 100644 --- a/application/single_app/static/js/chat/chat-streaming.js +++ b/application/single_app/static/js/chat/chat-streaming.js @@ -1,6 +1,7 @@ // chat-streaming.js import { appendMessage, + loadMessages, markUserMessageMetadataFinalizationUnconfirmed, markUserMessageMetadataUnconfirmed, refreshUserMessageMetadata, @@ -374,6 +375,11 @@ function notifyConversationDocumentsMayHaveChanged(conversationId, autoOpen = fa })); } +function hasCitedWorkspaceDocuments(payload) { + return Array.isArray(payload?.cited_hybrid_citations) + && payload.cited_hybrid_citations.length > 0; +} + async function requestStreamCancellation(streamContext = currentStreamContext) { if (!streamContext || streamContext.cancellationRequested) { return false; @@ -697,6 +703,36 @@ function consumeStreamingResponse(requestFactory, tempAiMessageId, tempUserMessa error_message: data.error, }); handleStreamError(tempAiMessageId, data.partial_content || accumulatedContent, data.error, data); + if ( + data.message_persisted + && data.message_id + && data.conversation_id + ) { + notifyConversationDocumentsMayHaveChanged( + data.conversation_id, + false + ); + if ( + data.conversation_kind === 'collaborative' + && typeof window.chatCollaboration?.loadConversationMessages === 'function' + ) { + void window.chatCollaboration + .loadConversationMessages(data.conversation_id) + .catch(error => { + console.warn( + 'Failed to reload persisted collaborative response:', + error + ); + }); + } else { + void loadMessages(data.conversation_id).catch(error => { + console.warn( + 'Failed to reload persisted response:', + error + ); + }); + } + } clearCurrentStreamController(abortController); if (typeof onError === 'function') { onError(data.error, data); @@ -1276,6 +1312,10 @@ function finalizeCancelledStreamingMessage(messageId, userMessageId, finalData, document.querySelector(`[data-message-id="${finalData.message_id}"]`), Boolean(String(partialContent || '').trim()) ); + notifyConversationDocumentsMayHaveChanged( + finalData.conversation_id, + false + ); } else if (messageElement) { renderStoppedContent(messageElement, partialContent); } @@ -1377,7 +1417,10 @@ function finalizeStreamingMessage(messageId, userMessageId, finalData, fallbackA if (existingFinalMessage) { markStreamingConversationReadIfActive(finalData.conversation_id, 'live streaming completion'); - notifyConversationDocumentsMayHaveChanged(finalData.conversation_id, Boolean(finalData.augmented)); + notifyConversationDocumentsMayHaveChanged( + finalData.conversation_id, + hasCitedWorkspaceDocuments(finalData) + ); return; } @@ -1406,7 +1449,10 @@ function finalizeStreamingMessage(messageId, userMessageId, finalData, fallbackA if (finalData.reload_messages && finalData.conversation_id && typeof window.chatMessages?.loadMessages === 'function') { window.chatMessages.loadMessages(finalData.conversation_id); } - notifyConversationDocumentsMayHaveChanged(finalData.conversation_id, Boolean(finalData.augmented)); + notifyConversationDocumentsMayHaveChanged( + finalData.conversation_id, + hasCitedWorkspaceDocuments(finalData) + ); return; } @@ -1470,7 +1516,10 @@ function finalizeStreamingMessage(messageId, userMessageId, finalData, fallbackA window.chatMessages.loadMessages(finalData.conversation_id); } - notifyConversationDocumentsMayHaveChanged(finalData.conversation_id, Boolean(finalData.augmented)); + notifyConversationDocumentsMayHaveChanged( + finalData.conversation_id, + hasCitedWorkspaceDocuments(finalData) + ); markStreamingConversationReadIfActive(finalData.conversation_id, 'live streaming completion'); } diff --git a/docs/explanation/features/CONVERSATION_CONTENTS_DRAWER.md b/docs/explanation/features/CONVERSATION_CONTENTS_DRAWER.md index 6b8cbccbd..bd7b03b01 100644 --- a/docs/explanation/features/CONVERSATION_CONTENTS_DRAWER.md +++ b/docs/explanation/features/CONVERSATION_CONTENTS_DRAWER.md @@ -7,6 +7,7 @@ The conversation contents drawer provides a compact index of persisted user mess Implemented in version: **0.250.074** Used documents mode added in version: **0.250.159** Compact overflow-safe layout added in version: **0.250.171** +Source/cited distinction added in version: **0.250.215** ### Dependencies @@ -21,7 +22,9 @@ Compact overflow-safe layout added in version: **0.250.171** The contents mode is derived entirely from messages already loaded into the chat DOM. It does not add a conversation-index API or a second datastore. `chat-conversation-contents.js` observes persisted user-message elements, renders labels with `textContent`, and tracks the message nearest the current scroll position. -The documents mode uses the same conversation metadata source as the details modal. `functions_conversation_metadata.py` retains each document's user-facing title and source filename in the conversation tag. `chat-conversation-details.js` exposes shared helpers for fetching conversation metadata and filtering `tags` where `category === "document"`. `chat-conversation-contents.js` renders those document tags into a vertical row list, so selected-but-unused documents are excluded and the side pane stays aligned with the details modal document card. +The documents mode uses the same conversation metadata source as the details modal but applies a different selector. Document tags remain the complete source inventory. New assistant messages persist exact cited document records, and conversation metadata maintains a compact `used_documents` aggregate for active responses. `chat-conversation-contents.js` renders exact used documents plus any explicit legacy fallback, while `chat-conversation-details.js` keeps the complete source list and marks exact matches with a **Cited** badge. + +`functions_citation_tracking.py` matches final-response citation IDs without mutating full source arrays. Historical conversations without versioned tracking continue to use their existing document tags in the drawer; they are not falsely marked as exact citations in the details modal. The application setting `enable_conversation_contents_drawer` is the authoritative global gate and defaults to `true`. The user setting `conversationContentsDrawerEnabled` also defaults to `true`; it is only effective while the admin gate is enabled. @@ -38,6 +41,8 @@ The application setting `enable_conversation_contents_drawer` is the authoritati - Contents and Documents share the same right-side drawer. Opening either mode swaps the drawer content instead of creating side-by-side panes. - After a streamed response completes, the drawer refreshes full conversation metadata and auto-opens Documents once per conversation when cited documents first appear. - Document rows show title, classification, source filename when it differs from the title, cited page references, and workspace scope. +- The full conversation details modal lists every source document and marks only exact tracked use with a **Cited** badge. +- Retrieved-only documents remain available under each response's **Sources** disclosure but do not enter the strict Used documents list. - Chunk counts and backend document IDs are intentionally omitted from the user-facing drawer. - Both lists use compact typography and constrained grid/flex tracks so long labels and metadata truncate instead of creating horizontal scrolling. @@ -46,6 +51,8 @@ The application setting `enable_conversation_contents_drawer` is the authoritati - `application/single_app/functions_settings.py` - `application/single_app/functions_conversation_contents.py` - `application/single_app/functions_conversation_metadata.py` +- `application/single_app/functions_citation_tracking.py` +- `application/single_app/route_backend_conversations.py` - `application/single_app/route_frontend_admin_settings.py` - `application/single_app/route_backend_users.py` - `application/single_app/route_frontend_chats.py` @@ -69,12 +76,15 @@ The application setting `enable_conversation_contents_drawer` is the authoritati - `functional_tests/test_conversation_contents_drawer_settings.py` validates global and user defaults, persistence wiring, validation, and gate precedence. - `functional_tests/test_document_action_conversation_scope_metadata.py` validates that source filenames survive conversation document-tag creation and refresh. +- `functional_tests/test_chat_cited_source_tracking.py` validates exact citation matching, source preservation, legacy compatibility, and active-attempt aggregation. - `ui_tests/test_chat_conversation_contents_drawer.py` covers filtering, ordering, safe 30-character labels, fallback labels, compact cited-document rendering, documents-mode swapping, auto-open behavior, navigation, focus, live updates, conversation replacement, keyboard closing, and desktop/mobile horizontal-overflow measurements. +- `ui_tests/test_chat_scope_lock_and_conversation_details_escaping.py` verifies the full source inventory and safe exact Cited badge. - The contents list is generated locally from authorized messages and inserts user-authored labels only through safe text APIs. The documents list also renders metadata through DOM APIs and `textContent`. ### Known limitations - Contents mode indexes user messages only. -- Documents mode includes cited/used conversation metadata documents only; documents merely selected in the picker but not used for citations are intentionally excluded. +- New tracked responses include exact cited/used documents only; retrieved-only documents are intentionally excluded. +- Historical conversations retain the prior document list as a compatibility fallback because exact citation use was not persisted before version `0.250.215`. - Search, grouping, generated summaries, assistant-message entries, and document filtering are not included. - Only messages currently loaded into the active timeline can appear in the contents list. diff --git a/docs/explanation/features/v0.238.024/CITATION_IMPROVEMENTS.md b/docs/explanation/features/v0.238.024/CITATION_IMPROVEMENTS.md index dd2029c02..66ca169c9 100644 --- a/docs/explanation/features/v0.238.024/CITATION_IMPROVEMENTS.md +++ b/docs/explanation/features/v0.238.024/CITATION_IMPROVEMENTS.md @@ -2,6 +2,7 @@ **Feature**: Citation Improvements **Version**: v0.238.024 +**Updated**: v0.250.215 **Dependencies**: Enhanced Citations System, chat-citations.js ## Overview and Purpose @@ -15,6 +16,7 @@ This release includes several improvements to the citation system in the chat in - **Improved Error Handling**: Graceful JSON parsing with explicit error messages in citation processing - **Robust Citation Links**: Support for both inline citation links and hybrid citation buttons - **Metadata Citations**: New metadata citation type for viewing extracted metadata (OCR text, vision analysis, detected objects) +- **Source and Reference Separation**: Complete returned sources remain inspectable while exact document and web citations are persisted separately for Used documents, details badges, and exports ## Technical Specifications @@ -37,6 +39,18 @@ The citation system supports multiple citation types: | **Agent Citations** | Tool execution details from Semantic Kernel agents | Expandable section with function args and results | | **Metadata Citations** | Extracted metadata from document processing | Expandable section with OCR text, vision analysis, detected objects | +### Retrieved sources and cited references + +Starting in version `0.250.215`, SimpleChat distinguishes retrieval disclosure from final-response references: + +- `hybrid_citations` and `web_search_citations` retain all returned sources for the **Sources** control and JSON audit output. +- `cited_hybrid_citations` contains only returned document records whose citation IDs appear in the final response, with strict filename/location matching for explicit legacy-style source references that omit the ID. +- `cited_web_search_citations` contains only returned web records whose normalized URLs appear in the final response. +- Agent citations continue to represent tool executions that actually occurred; Foundry web-source proxy entries are deduplicated into web references for exports. +- Conversation details list all source documents and mark exact use with **Cited**. +- Used documents and reference-producing exports consume exact cited subsets for new responses. +- Historical records without versioned tracking use the previous broad fallback and require no migration. + ### Enhanced Citation Modal Citations that reference documents open specialized viewers based on file type: @@ -62,6 +76,10 @@ The document selection dropdown in the chat interface was widened to accommodate | `static/js/chat/chat-citations.js` | Core citation parsing, link generation, and rendering | | `static/js/chat/chat-enhanced-citations.js` | Enhanced citation modal for PDF/image/video/audio viewing | | `static/js/chat/chat-documents.js` | `getDocumentMetadata()` for metadata citations; dropdown width updates | +| `functions_citation_tracking.py` | Exact document/web citation matching and compatibility-aware reference selection | +| `route_backend_conversation_export.py` | Citation-accurate conversation and per-message reference sections | +| `static/js/chat/chat-conversation-details.js` | Full source inventory and exact Cited annotations | +| `static/js/chat/chat-conversation-contents.js` | Exact Used documents selection with historical fallback | ## Commits @@ -81,3 +99,5 @@ The document selection dropdown in the chat interface was widened to accommodate - **Timestamp seeking**: Verify video and audio citations navigate to the correct timestamp - **Metadata citations**: Verify extracted metadata (OCR, vision, objects) displays correctly in expandable sections - **Error handling**: Verify malformed citation strings are handled gracefully without breaking the chat interface +- **Source/reference distinction**: Verify retrieved-only sources remain under Sources but are excluded from new Used documents and export references +- **Compatibility**: Verify historical messages and conversations retain their legacy fallback without read-time history parsing diff --git a/docs/explanation/fixes/SOURCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md b/docs/explanation/fixes/SOURCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md new file mode 100644 index 000000000..2860e34a6 --- /dev/null +++ b/docs/explanation/fixes/SOURCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md @@ -0,0 +1,95 @@ +# Retrieved Sources and Cited References Distinction Fix + +## Header information + +**Issue:** Retrieved document and web search results were presented as citations even when the final assistant response did not reference them. + +**Root cause:** SimpleChat stored complete retrieval result arrays in `hybrid_citations` and `web_search_citations`, then reused those arrays for the per-message Sources disclosure, the Used documents drawer, conversation details, and export reference sections. The data model did not preserve a separate exact-citation subset. + +Fixed/Implemented in version: **0.250.215** + +Related config.py update: `VERSION = "0.250.215"` + +Associated issue: `microsoft/simplechat#1249` + +## Technical details + +### Persisted contract + +New assistant responses preserve both source and reference data: + +| Field | Meaning | +|---|---| +| `hybrid_citations` | Every returned workspace document source | +| `web_search_citations` | Every returned web source | +| `agent_citations` | Tool and agent executions that occurred | +| `citation_tracking_version` | Indicates exact citation subsets are authoritative | +| `cited_hybrid_citations` | Returned document records matched by exact citation IDs, with strict filename/location matching for explicit legacy-style source references | +| `cited_web_search_citations` | Returned web records whose normalized URLs appear in the final response | + +Conversation records persist `used_documents` for exact active-response usage and `legacy_used_documents` for pre-tracking compatibility. Normal metadata reads use these compact fields and do not reparse message history. + +### Files modified + +- `application/single_app/functions_citation_tracking.py` +- `application/single_app/route_backend_chats.py` +- `application/single_app/route_backend_conversations.py` +- `application/single_app/functions_simplechat_operations.py` +- `application/single_app/functions_collaboration.py` +- `application/single_app/collaboration_models.py` +- `application/single_app/route_backend_collaboration.py` +- `application/single_app/functions_workflow_runner.py` +- `application/single_app/route_backend_conversation_export.py` +- `application/single_app/static/js/chat/chat-citations.js` +- `application/single_app/static/js/chat/chat-collaboration.js` +- `application/single_app/static/js/chat/chat-conversation-details.js` +- `application/single_app/static/js/chat/chat-conversation-contents.js` +- `application/single_app/static/js/chat/chat-streaming.js` + +### Code changes + +- Exact `[#citation-id]` tokens are matched to returned document records after final response content is settled. Explicit `(Source: filename, Page/Sheet/Location: value)` references provide a strict fallback for response paths that omit the hidden ID. +- Final-response HTTP(S) URLs are normalized and matched to returned web records. +- Full source arrays remain unchanged for the per-message **Sources** disclosure and JSON audit output. +- The **Used documents** drawer reads exact active-response usage plus an explicit historical fallback. +- Conversation details continue to show all source documents and add a **Cited** badge only to exact tracked documents. +- Retry, edit, attempt switching, deletion, collaboration mirroring, workflow messages, and conversation forks maintain the exact used-document aggregate. +- Canceled/interrupted persisted responses update source metadata and notify active clients; collaboration stream bridges mirror persisted partial responses before returning the terminal error. +- Conversation Markdown/PDF and per-message Word/PowerPoint/email reference sections use cited document/web subsets plus executed tools. Foundry web-source proxy entries are represented once as web references rather than duplicated as tool references. +- Historical messages without a tracking version retain their previous broad export behavior. + +### Testing approach + +- `functional_tests/test_chat_cited_source_tracking.py` validates exact document IDs, normalized web URLs, immutable source arrays, aggregation, lifecycle rebuilds, collaboration propagation, UI contracts, and export selection. +- `functional_tests/test_conversation_export.py` validates that tracked reference appendices exclude retrieved-only document and web sources while JSON retains them. +- `functional_tests/test_per_message_export.py` and `functional_tests/test_per_message_powerpoint_export.py` validate the shared Word, email, and PowerPoint reference path. +- `ui_tests/test_chat_conversation_contents_drawer.py` validates strict Used documents selection and historical fallback. +- `ui_tests/test_chat_scope_lock_and_conversation_details_escaping.py` validates the full source inventory, exact Cited badge, and safe rendering. + +### Impact analysis + +- No Cosmos migration is required. +- Existing source links and enhanced citation behavior remain unchanged. +- New responses gain exact reference semantics. +- Historical conversations remain readable and keep their previous drawer/export behavior when exact data is unavailable. +- Ordinary metadata reads remain constant-time with respect to message history; message scans occur only after uncommon lifecycle mutations. + +## Validation + +### Before + +- Every retrieved document could appear in **Used documents**. +- Conversation details could not distinguish cited documents from retrieved-only documents. +- Export reference sections could list every returned document and web result. + +### After + +- **Sources** remains the complete retrieval/execution disclosure. +- **Used documents** reflects cited documents in active responses, with an explicit legacy fallback. +- Conversation details list all source documents and mark exact citations. +- Reference-producing exports contain exact cited document/web records and executed tools. +- JSON exports preserve both complete source arrays and exact cited subsets. + +### User experience improvement + +Users can now tell what the model reviewed from what it actually cited, and exported references accurately represent the evidence visible in the generated response. diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 7ad9569c3..4ba293ab5 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,16 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](/explanation/features/) and [Fixes by Version](/explanation/fixes/). +### **(v0.250.215)** + +#### Bug Fixes + +* **Retrieved Sources and Cited References** + * Separated complete document/web retrieval results from the exact references used in final assistant responses, while preserving all returned results under **Sources**. + * Used documents now follows active cited responses, conversation details marks cited items within the full source inventory, and conversation/message export references exclude retrieved-only sources. + * Historical conversations retain their previous fallback without a migration or ordinary read-time history parsing. + * (Ref: #1249, `functions_citation_tracking.py`, `route_backend_chats.py`, `route_backend_conversation_export.py`, `SOURCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md`) + ### **(v0.250.214)** #### New Features diff --git a/functional_tests/test_chat_cited_source_tracking.py b/functional_tests/test_chat_cited_source_tracking.py new file mode 100644 index 000000000..87db78764 --- /dev/null +++ b/functional_tests/test_chat_cited_source_tracking.py @@ -0,0 +1,630 @@ +# test_chat_cited_source_tracking.py +#!/usr/bin/env python3 +""" +Functional test for cited-source tracking. +Version: 0.250.215 +Implemented in: 0.250.215 + +This test ensures returned sources remain complete while exact document and +web references drive used-document aggregates and export reference buckets. +""" + +from copy import deepcopy +from pathlib import Path +import sys + +from test_support.versioning import assert_app_version_at_least + + +REPO_ROOT = Path(__file__).resolve().parents[1] +APP_ROOT = REPO_ROOT / "application" / "single_app" +CHAT_ROUTE = APP_ROOT / "route_backend_chats.py" +CONVERSATIONS_ROUTE = APP_ROOT / "route_backend_conversations.py" +COLLABORATION_FUNCTIONS = APP_ROOT / "functions_collaboration.py" +COLLABORATION_ROUTE = APP_ROOT / "route_backend_collaboration.py" +CONVERSATION_EXPORT_ROUTE = APP_ROOT / "route_backend_conversation_export.py" +CONVERSATION_DETAILS_JS = ( + APP_ROOT / "static" / "js" / "chat" / "chat-conversation-details.js" +) +CONVERSATION_CONTENTS_JS = ( + APP_ROOT / "static" / "js" / "chat" / "chat-conversation-contents.js" +) +CHAT_STREAMING_JS = APP_ROOT / "static" / "js" / "chat" / "chat-streaming.js" +CHAT_COLLABORATION_JS = ( + APP_ROOT / "static" / "js" / "chat" / "chat-collaboration.js" +) +CHAT_MESSAGES_JS = APP_ROOT / "static" / "js" / "chat" / "chat-messages.js" +CHAT_RETRY_JS = APP_ROOT / "static" / "js" / "chat" / "chat-retry.js" +CHAT_EDIT_JS = APP_ROOT / "static" / "js" / "chat" / "chat-edit.js" +SIMPLECHAT_OPERATIONS = APP_ROOT / "functions_simplechat_operations.py" +WORKFLOW_RUNNER = APP_ROOT / "functions_workflow_runner.py" +if str(APP_ROOT) not in sys.path: + sys.path.insert(0, str(APP_ROOT)) + +from functions_citation_tracking import ( # noqa: E402 + CITATION_TRACKING_VERSION, + USED_DOCUMENTS_TRACKING_VERSION, + build_cited_source_subsets, + build_used_documents, + get_message_reference_citation_buckets, + initialize_conversation_used_document_tracking, + merge_cited_documents_into_conversation, + rebuild_conversation_used_documents, + resolve_citation_location, +) +from collaboration_models import ( # noqa: E402 + build_collaboration_message_doc_from_legacy, +) + + +def _hybrid_sources(): + return [ + { + "file_name": "Policy.pdf", + "document_id": "doc-policy", + "citation_id": "doc-policy_1", + "chunk_id": "1", + "page_number": 1, + "classification": "Internal", + }, + { + "file_name": "Policy.pdf", + "document_id": "doc-policy", + "citation_id": "doc-policy_2", + "chunk_id": "2", + "page_number": 2, + "classification": "Internal", + }, + { + "file_name": "Unused.pdf", + "document_id": "doc-unused", + "citation_id": "doc-unused_7", + "chunk_id": "7", + "page_number": 7, + }, + ] + + +def _web_sources(): + return [ + { + "title": "Cited web page", + "url": "https://Example.com:443/guidance?view=current#overview", + }, + { + "title": "Unused web page", + "url": "https://example.com/unused", + }, + ] + + +def test_exact_document_and_web_references_preserve_sources(): + """Match grouped document IDs and normalized URLs without mutating sources.""" + hybrid_sources = _hybrid_sources() + web_sources = _web_sources() + original_hybrid_sources = deepcopy(hybrid_sources) + original_web_sources = deepcopy(web_sources) + content = ( + "The policy requires review " + "(Source: Policy.pdf, Pages: 1, 2) [#doc-policy_1; #doc-policy_2]. " + "See [current guidance](https://example.com/guidance?view=current). " + "An unknown token [#doc-missing_9] is ignored." + ) + + tracked = build_cited_source_subsets( + content, + hybrid_citations=hybrid_sources, + web_search_citations=web_sources, + ) + + assert tracked["citation_tracking_version"] == CITATION_TRACKING_VERSION + assert [ + citation["citation_id"] + for citation in tracked["cited_hybrid_citations"] + ] == ["doc-policy_1", "doc-policy_2"] + assert [ + citation["title"] + for citation in tracked["cited_web_search_citations"] + ] == ["Cited web page"] + assert hybrid_sources == original_hybrid_sources + assert web_sources == original_web_sources + + tracked["cited_hybrid_citations"][0]["file_name"] = "Changed.pdf" + assert hybrid_sources[0]["file_name"] == "Policy.pdf" + + +def test_explicit_source_location_matches_without_hidden_id(): + """Match the visible citation format required by the model prompt.""" + tracked = build_cited_source_subsets( + "Does the second page control (Source: Policy.pdf, Page: 2)?", + hybrid_citations=_hybrid_sources(), + web_search_citations=[], + ) + + assert [ + citation["citation_id"] + for citation in tracked["cited_hybrid_citations"] + ] == ["doc-policy_2"] + + +def test_hidden_id_prevents_same_page_source_overmatching(): + """An exact ID should win over other returned chunks from the same page.""" + same_page_source = { + **_hybrid_sources()[0], + "citation_id": "doc-policy_1_other", + "chunk_id": "1-other", + } + tracked = build_cited_source_subsets( + ( + "The first page controls " + "(Source: Policy.pdf, Page: 1) [#doc-policy_1]" + ), + hybrid_citations=[_hybrid_sources()[0], same_page_source], + web_search_citations=[], + ) + + assert [ + citation["citation_id"] + for citation in tracked["cited_hybrid_citations"] + ] == ["doc-policy_1"] + + +def test_visible_source_matching_handles_sheet_and_filename_commas(): + """Visible references should support sheet locations and original filenames.""" + tracked = build_cited_source_subsets( + ( + "The Q1 sheet controls. " + "(Source: Budget, 2026.xlsx, Sheet: Q1, Final (Approved))" + ), + hybrid_citations=[{ + "file_name": "Budget, 2026.xlsx", + "document_id": "budget-doc", + "citation_id": "budget-doc_Q1", + "page_number": "Q1, Final (Approved)", + "sheet_name": "Q1, Final (Approved)", + "location_label": "Sheet", + "location_value": "Q1, Final (Approved)", + }], + web_search_citations=[], + ) + + assert [ + citation["citation_id"] + for citation in tracked["cited_hybrid_citations"] + ] == ["budget-doc_Q1"] + + +def test_schema_aware_citation_location_is_shared(): + """Workflow and chat citations should label workbook schemas consistently.""" + assert resolve_citation_location( + page_number=1, + chunk_text="Tabular workbook: Budget.xlsx", + is_tabular=True, + ) == ("Location", "Workbook Schema") + assert resolve_citation_location( + page_number=4, + chunk_text="Narrative page", + is_tabular=False, + ) == ("Page", "4") + + +def test_used_documents_collapse_cited_chunks_only(): + """Collapse multiple cited chunks while excluding retrieved-only documents.""" + source_tags = [ + { + "category": "document", + "document_id": "doc-policy", + "title": "Policy Handbook", + "file_name": "Policy.pdf", + "classification": "Internal", + "chunk_ids": ["doc-policy_1", "doc-policy_2", "doc-policy_3"], + "scope": { + "type": "group", + "id": "group-1", + "name": "Policy Team", + }, + }, + ] + + used_documents = build_used_documents( + _hybrid_sources()[:2], + source_document_tags=source_tags, + ) + + assert len(used_documents) == 1 + used_document = used_documents[0] + assert used_document["document_id"] == "doc-policy" + assert used_document["title"] == "Policy Handbook" + assert used_document["chunk_ids"] == ["doc-policy_1", "doc-policy_2"] + assert used_document["citation_ids"] == ["doc-policy_1", "doc-policy_2"] + assert used_document["page_numbers"] == [1, 2] + assert used_document["scope"]["name"] == "Policy Team" + + +def test_reference_buckets_use_strict_empty_values_and_legacy_fallback(): + """Tracked messages must not fall back when exact cited subsets are empty.""" + source_message = { + "citations": [{"title": "Legacy"}], + "hybrid_citations": _hybrid_sources(), + "web_search_citations": _web_sources(), + "agent_citations": [ + {"tool_name": "Calculator", "function_name": "calculate"}, + { + "tool_name": "Used web reference", + "function_name": "azure_ai_foundry_web_search", + }, + ], + } + + legacy_buckets = get_message_reference_citation_buckets(source_message) + assert len(legacy_buckets["hybrid"]) == 3 + assert len(legacy_buckets["web"]) == 2 + assert len(legacy_buckets["agent"]) == 2 + + tracked_message = { + **source_message, + "citation_tracking_version": CITATION_TRACKING_VERSION, + "cited_hybrid_citations": [], + "cited_web_search_citations": [], + } + tracked_buckets = get_message_reference_citation_buckets(tracked_message) + assert tracked_buckets["hybrid"] == [] + assert tracked_buckets["web"] == [] + assert tracked_buckets["agent"] == [{ + "tool_name": "Calculator", + "function_name": "calculate", + }] + assert tracked_buckets["legacy"] == [{"title": "Legacy"}] + + +def test_conversation_tracking_preserves_legacy_and_merges_exact_usage(): + """Snapshot pre-tracking tags and keep new exact documents separate.""" + conversation = { + "tags": [ + { + "category": "document", + "document_id": "doc-legacy", + "title": "Historical Source", + "chunk_ids": ["doc-legacy_1"], + }, + { + "category": "participant", + "user_id": "user-1", + }, + ], + } + + initialized = initialize_conversation_used_document_tracking(conversation) + assert initialized is True + assert ( + conversation["used_documents_tracking_version"] + == USED_DOCUMENTS_TRACKING_VERSION + ) + assert [ + document["document_id"] + for document in conversation["legacy_used_documents"] + ] == ["doc-legacy"] + assert conversation["used_documents"] == [] + + conversation["tags"].append({ + "category": "document", + "document_id": "doc-policy", + "title": "Policy Handbook", + "file_name": "Policy.pdf", + "chunk_ids": ["doc-policy_1", "doc-policy_2", "doc-policy_7"], + "scope": { + "type": "group", + "id": "group-1", + "name": "Policy Team", + }, + }) + merged = merge_cited_documents_into_conversation( + conversation, + _hybrid_sources()[:2], + ) + + assert [document["document_id"] for document in merged] == ["doc-policy"] + assert merged[0]["chunk_ids"] == ["doc-policy_1", "doc-policy_2"] + assert [ + document["document_id"] + for document in conversation["legacy_used_documents"] + ] == ["doc-legacy"] + + +def test_rebuild_uses_active_non_deleted_tracked_assistant_messages(): + """Retry and deletion rebuilds must follow the active visible response set.""" + conversation = { + "tags": [ + { + "category": "document", + "document_id": "doc-policy", + "title": "Policy Handbook", + "chunk_ids": ["doc-policy_1", "doc-policy_2"], + }, + { + "category": "document", + "document_id": "doc-unused", + "title": "Unused Source", + "chunk_ids": ["doc-unused_7"], + }, + ], + "used_documents_tracking_version": USED_DOCUMENTS_TRACKING_VERSION, + "legacy_used_documents": [], + "used_documents": [], + } + messages = [ + { + "role": "assistant", + "citation_tracking_version": CITATION_TRACKING_VERSION, + "cited_hybrid_citations": [_hybrid_sources()[0]], + "metadata": { + "thread_info": { + "active_thread": True, + }, + }, + }, + { + "role": "assistant", + "citation_tracking_version": CITATION_TRACKING_VERSION, + "cited_hybrid_citations": [_hybrid_sources()[1]], + "metadata": { + "thread_info": { + "active_thread": False, + }, + }, + }, + { + "role": "assistant", + "citation_tracking_version": CITATION_TRACKING_VERSION, + "cited_hybrid_citations": [_hybrid_sources()[2]], + "metadata": { + "is_deleted": True, + "thread_info": { + "active_thread": True, + }, + }, + }, + { + "role": "assistant", + "hybrid_citations": [_hybrid_sources()[2]], + "metadata": {}, + }, + ] + + rebuilt = rebuild_conversation_used_documents(conversation, messages) + + assert [document["document_id"] for document in rebuilt] == ["doc-policy"] + assert rebuilt[0]["chunk_ids"] == ["doc-policy_1"] + + +def test_fork_rebuilds_legacy_fallback_from_copied_messages(): + """Fork legacy fallback must exclude documents introduced after the fork point.""" + conversation = { + "tags": [ + { + "category": "document", + "document_id": "doc-policy", + "title": "Policy Handbook", + "chunk_ids": ["doc-policy_1"], + }, + { + "category": "document", + "document_id": "doc-unused", + "title": "Later Source", + "chunk_ids": ["doc-unused_7"], + }, + ], + } + copied_messages = [ + { + "role": "assistant", + "hybrid_citations": [_hybrid_sources()[0]], + "metadata": { + "thread_info": { + "active_thread": True, + }, + }, + }, + ] + + rebuild_conversation_used_documents( + conversation, + copied_messages, + rebuild_legacy=True, + ) + + assert conversation["used_documents"] == [] + assert [ + document["document_id"] + for document in conversation["legacy_used_documents"] + ] == ["doc-policy"] + + +def test_chat_publication_paths_persist_tracking_and_aggregates(): + """Verify each chat publication path stores exact subsets and aggregates.""" + route_source = CHAT_ROUTE.read_text(encoding="utf-8") + + required_snippets = [ + "document_action_citation_tracking = build_cited_source_subsets(", + "citation_tracking = build_cited_source_subsets(", + "stream_citation_tracking = build_cited_source_subsets(", + "partial_citation_tracking = build_cited_source_subsets(", + "interrupted_citation_tracking = build_cited_source_subsets(", + "**document_action_citation_tracking,", + "**citation_tracking,", + "**stream_citation_tracking,", + "initialize_conversation_used_document_tracking(conversation_item)", + "merge_cited_documents_into_conversation(", + "'hybrid_citations': hybrid_citations_list", + "'web_search_citations': web_search_citations_list", + ] + missing = [ + snippet + for snippet in required_snippets + if snippet not in route_source + ] + assert not missing, f"Missing chat citation tracking wiring: {missing}" + assert route_source.count("merge_cited_documents_into_conversation(") >= 4 + assert "Copy the exact bracketed citation ID shown after the supporting excerpt." in route_source + assert 'reason="chat_stream_interrupted"' in route_source + assert route_source.count("collect_stream_response_conversation_metadata()") >= 4 + assert '"sheet_name": source_doc.get("sheet_name")' in route_source + assert '"sheet_name": sheet_name' in route_source + + +def test_lifecycle_mutations_and_forks_rebuild_exact_usage(): + """Verify retry/edit/switch/delete/fork paths invoke the shared rebuild.""" + route_source = CONVERSATIONS_ROUTE.read_text(encoding="utf-8") + operations_source = SIMPLECHAT_OPERATIONS.read_text(encoding="utf-8") + + assert ( + route_source.count( + "_rebuild_authorized_personal_conversation_used_documents(" + ) + >= 5 + ) + for reason in ( + "message_deleted", + "message_retry_created", + "message_edit_created", + "message_attempt_switched", + ): + assert reason in route_source + assert '"used_documents_tracking_version"' in operations_source + assert '"legacy_used_documents"' in operations_source + assert "rebuild_legacy=True" in operations_source + assert '"used_documents_tracking_version": conversation_item.get(' in route_source + assert '"legacy_used_documents": conversation_item.get(' in route_source + assert '"used_documents": conversation_item.get(' in route_source + assert "except Exception as rebuild_error:" in route_source + assert "used_documents_rebuild_required" in route_source + + +def test_collaboration_and_workflow_propagate_tracking_contract(): + """Verify shared and workflow messages retain exact cited subsets.""" + legacy_message = { + "id": "assistant-source-1", + "role": "assistant", + "content": "Tracked response", + "timestamp": "2026-08-14T12:00:00Z", + "citation_tracking_version": CITATION_TRACKING_VERSION, + "cited_hybrid_citations": [_hybrid_sources()[0]], + "cited_web_search_citations": [_web_sources()[0]], + } + converted = build_collaboration_message_doc_from_legacy( + conversation_id="collaboration-1", + legacy_message=legacy_message, + default_sender_user={ + "user_id": "user-1", + "display_name": "Test User", + "email": "test@example.com", + }, + ) + assert converted["citation_tracking_version"] == CITATION_TRACKING_VERSION + assert converted["cited_hybrid_citations"][0]["citation_id"] == "doc-policy_1" + assert converted["cited_web_search_citations"][0]["title"] == "Cited web page" + + collaboration_source = COLLABORATION_FUNCTIONS.read_text(encoding="utf-8") + collaboration_route_source = COLLABORATION_ROUTE.read_text(encoding="utf-8") + workflow_source = WORKFLOW_RUNNER.read_text(encoding="utf-8") + + for field_name in ( + "citation_tracking_version", + "cited_hybrid_citations", + "cited_web_search_citations", + "used_documents_tracking_version", + "legacy_used_documents", + "used_documents", + ): + assert field_name in collaboration_source + assert "rebuild_conversation_used_documents(conversation_doc, remaining_messages)" in collaboration_source + assert "'cited_hybrid_citations': serialized_assistant_message.get(" in collaboration_route_source + assert "stream_payload['done'] = True" in collaboration_route_source + assert "'conversation_kind': serialized_final_conversation.get(" in collaboration_route_source + assert "citation_tracking = build_cited_source_subsets(" in workflow_source + assert "merge_cited_documents_into_conversation(" in workflow_source + assert "(Source: {file_name}, {location_label}: {location_value}) [#{citation_id}]" in workflow_source + assert "'sheet_name': sheet_name" in workflow_source + assert "resolve_citation_location(" in workflow_source + + +def test_ui_and_exports_select_cited_subsets_without_losing_sources(): + """Verify downstream consumers separate source inventory from references.""" + details_source = CONVERSATION_DETAILS_JS.read_text(encoding="utf-8") + contents_source = CONVERSATION_CONTENTS_JS.read_text(encoding="utf-8") + export_source = CONVERSATION_EXPORT_ROUTE.read_text(encoding="utf-8") + + assert "export function getConversationDocumentTags(metadata = {})" in details_source + assert "export function getConversationUsedDocuments(metadata = {})" in details_source + assert "getConversationExactUsedDocuments(metadata)" in details_source + assert "getConversationLegacyUsedDocuments(metadata)" in details_source + assert "conversation-document-cited-badge" in details_source + assert "Source documents" in details_source + assert "getConversationUsedDocuments," in contents_source + assert "const documents = getConversationUsedDocuments(metadata);" in contents_source + assert "Documents cited in active responses" in contents_source + assert "exactDocuments.length > 0" in contents_source + + streaming_source = CHAT_STREAMING_JS.read_text(encoding="utf-8") + assert "function hasCitedWorkspaceDocuments(payload)" in streaming_source + assert "hasCitedWorkspaceDocuments(finalData)" in streaming_source + assert ( + "notifyConversationDocumentsMayHaveChanged(" + in streaming_source + ) + assert "data.message_persisted" in streaming_source + assert "&& data.message_id" in streaming_source + assert "&& data.conversation_id" in streaming_source + assert "void loadMessages(data.conversation_id).catch" in streaming_source + assert "data.conversation_kind === 'collaborative'" in streaming_source + assert ".loadConversationMessages(data.conversation_id)" in streaming_source + + collaboration_ui_source = CHAT_COLLABORATION_JS.read_text(encoding="utf-8") + assert "'chat:conversation-documents-refresh'" in collaboration_ui_source + assert "String(payload.message.role || '').trim().toLowerCase() === 'assistant'" in collaboration_ui_source + assert "conversationKind: 'collaborative'" in collaboration_ui_source + assert "loadConversationMessages," in collaboration_ui_source + + for mutation_source_path in ( + CHAT_MESSAGES_JS, + CHAT_RETRY_JS, + CHAT_EDIT_JS, + ): + mutation_source = mutation_source_path.read_text(encoding="utf-8") + assert "'chat:conversation-documents-refresh'" in mutation_source + + assert "return get_message_reference_citation_buckets(message)" in export_source + assert "return get_message_source_citation_buckets(message)" in export_source + assert "source_citation_buckets = _collect_source_citation_buckets(message)" in export_source + assert "'cited_hybrid_citations': (" in export_source + assert "'cited_web_search_citations': (" in export_source + assert "Document References" in export_source + assert "Web References" in export_source + + +def test_version_is_available(): + """Verify the application includes the cited-source tracking version.""" + assert_app_version_at_least("0.250.215") + + +if __name__ == "__main__": + tests = [ + test_exact_document_and_web_references_preserve_sources, + test_explicit_source_location_matches_without_hidden_id, + test_hidden_id_prevents_same_page_source_overmatching, + test_visible_source_matching_handles_sheet_and_filename_commas, + test_schema_aware_citation_location_is_shared, + test_used_documents_collapse_cited_chunks_only, + test_reference_buckets_use_strict_empty_values_and_legacy_fallback, + test_conversation_tracking_preserves_legacy_and_merges_exact_usage, + test_rebuild_uses_active_non_deleted_tracked_assistant_messages, + test_fork_rebuilds_legacy_fallback_from_copied_messages, + test_chat_publication_paths_persist_tracking_and_aggregates, + test_lifecycle_mutations_and_forks_rebuild_exact_usage, + test_collaboration_and_workflow_propagate_tracking_contract, + test_ui_and_exports_select_cited_subsets_without_losing_sources, + test_version_is_available, + ] + for test in tests: + test() + print(f"Cited-source tracking tests passed: {len(tests)}/{len(tests)}") diff --git a/functional_tests/test_conversation_export.py b/functional_tests/test_conversation_export.py index 654059b13..a6daf5277 100644 --- a/functional_tests/test_conversation_export.py +++ b/functional_tests/test_conversation_export.py @@ -2,8 +2,8 @@ # test_conversation_export.py """ Functional test for conversation export enhancements. -Version: 0.239.029 -Implemented in: 0.239.022 (base), 0.239.023 (PDF export), 0.239.025 (tag formatting fix), 0.239.028 (summary token budget fix) +Version: 0.250.215 +Implemented in: 0.239.022 (base); Updated in: 0.250.215 This test ensures conversation export now includes normalized and raw citations, processing thoughts, transcript-style Markdown appendices, deleted-message filtering, @@ -20,9 +20,11 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'application', 'single_app')) from route_backend_conversation_export import ( # noqa: E402 + _build_message_citation_labels, _build_citation_counts, _build_pdf_html_body, _collect_raw_citation_buckets, + _collect_source_citation_buckets, _conversation_to_markdown, _conversation_to_pdf_bytes, _filter_messages_for_export, @@ -130,6 +132,130 @@ def test_sanitize_message_with_citations_and_thoughts(): return True +def test_tracked_export_references_exclude_uncited_sources(): + """Tracked exports should reference exact subsets while retaining source audit data.""" + print("Testing tracked export reference filtering...") + + cited_document = { + 'file_name': 'used-policy.pdf', + 'document_id': 'doc-used', + 'citation_id': 'doc-used_2', + 'page_number': 2, + } + uncited_document = { + 'file_name': 'retrieved-only.pdf', + 'document_id': 'doc-unused', + 'citation_id': 'doc-unused_8', + 'page_number': 8, + } + cited_web = { + 'title': 'Used web reference', + 'url': 'https://example.com/used', + } + uncited_web = { + 'title': 'Retrieved web result', + 'url': 'https://example.com/unused', + } + message = { + 'id': 'assistant-tracked', + 'role': 'assistant', + 'content': ( + 'The cited policy applies (Source: used-policy.pdf, Page: 2). ' + 'See https://example.com/used.' + ), + 'timestamp': '2026-08-14T12:00:00Z', + 'citation_tracking_version': 1, + 'hybrid_citations': [cited_document, uncited_document], + 'web_search_citations': [cited_web, uncited_web], + 'cited_hybrid_citations': [cited_document], + 'cited_web_search_citations': [cited_web], + 'agent_citations': [ + { + 'tool_name': 'Calculator', + 'function_name': 'calculate', + }, + { + 'tool_name': 'Retrieved web result', + 'function_name': 'azure_ai_foundry_web_search', + 'plugin_name': 'azure_ai_foundry', + }, + ], + } + + references = _collect_raw_citation_buckets(message) + sources = _collect_source_citation_buckets(message) + sanitized = _sanitize_message( + message, + sequence_index=1, + transcript_index=1, + thoughts=[], + ) + labels = _build_message_citation_labels(message) + + assert [item['file_name'] for item in references['hybrid']] == ['used-policy.pdf'] + assert [item['title'] for item in references['web']] == ['Used web reference'] + assert len(sources['hybrid']) == 2 + assert len(sources['web']) == 2 + assert sanitized['citation_counts'] == { + 'document': 1, + 'web': 1, + 'agent_tool': 1, + 'legacy': 0, + 'total': 3, + } + assert len(sanitized['hybrid_citations']) == 2 + assert len(sanitized['web_search_citations']) == 2 + assert len(sanitized['agent_citations']) == 2 + assert sanitized['cited_hybrid_citations'] == [cited_document] + assert sanitized['cited_web_search_citations'] == [cited_web] + assert 'used-policy.pdf — Page 2' in labels + assert 'Used web reference' in labels + assert 'Calculator' in labels + assert all('retrieved-only.pdf' not in label for label in labels) + assert all('Retrieved web result' not in label for label in labels) + + entry = { + 'conversation': { + 'id': 'conversation-tracked', + 'title': 'Tracked export', + 'message_count': 1, + 'citation_counts': sanitized['citation_counts'], + }, + 'summary_intro': {'enabled': False, 'generated': False}, + 'messages': [sanitized], + } + markdown = _conversation_to_markdown(entry) + pdf_html = _build_pdf_html_body(entry) + + assert 'Document References' in markdown + assert 'Web References' in markdown + assert 'retrieved-only.pdf' not in markdown + assert 'Retrieved web result' not in markdown + assert 'Document References' in pdf_html + assert 'Web References' in pdf_html + assert 'retrieved-only.pdf' not in pdf_html + assert 'Retrieved web result' not in pdf_html + + sheet_reference = { + 'file_name': 'Budget, 2026.xlsx', + 'document_id': 'budget-doc', + 'citation_id': 'budget-doc_Q1', + 'page_number': 'Q1', + 'sheet_name': 'Q1', + 'location_label': 'Sheet', + 'location_value': 'Q1', + } + sheet_labels = _build_message_citation_labels({ + 'citation_tracking_version': 1, + 'hybrid_citations': [sheet_reference], + 'cited_hybrid_citations': [sheet_reference], + }) + assert sheet_labels == ['Budget, 2026.xlsx — Sheet: Q1'] + + print("Tracked export reference filtering passed") + return True + + def test_conversation_metadata_and_json_shape(): """Test that conversation-level metadata captures counts for export JSON.""" print("🔍 Testing conversation metadata and JSON shape...") @@ -534,6 +660,7 @@ def test_tag_formatting(): tests = [ test_filter_messages_for_export, test_sanitize_message_with_citations_and_thoughts, + test_tracked_export_references_exclude_uncited_sources, test_conversation_metadata_and_json_shape, test_markdown_export_structure, test_safe_filename_and_zip_packaging, diff --git a/functional_tests/test_per_message_export.py b/functional_tests/test_per_message_export.py index 3b8e3ddf9..ef933df24 100644 --- a/functional_tests/test_per_message_export.py +++ b/functional_tests/test_per_message_export.py @@ -2,8 +2,8 @@ # test_per_message_export.py """ Functional tests for the per-message export feature and export route regressions. -Version: 0.241.146 -Implemented in: 0.241.019 +Version: 0.250.215 +Implemented in: 0.241.019; Updated in: 0.250.215 Covers: - Happy path: Word document built successfully from a valid message. @@ -874,6 +874,8 @@ def test_per_message_export_uses_content_override_contract(): assert 'def _apply_message_export_content_override(' in route_source assert route_source.count('message_content_override = _get_message_export_content_override(data)') >= 3 assert route_source.count('_apply_message_export_content_override(message, message_content_override)') >= 3 + assert 'get_message_reference_citation_buckets' in route_source + assert 'normalized_citations = _normalize_citations(_collect_raw_citation_buckets(message))' in route_source assert 'function buildMessageExportRequestBody(messageDiv, messageId, conversationId, role, extraFields = {})' in frontend_source assert 'requestBody.message_content_override = messageContentOverride;' in frontend_source diff --git a/functional_tests/test_per_message_powerpoint_export.py b/functional_tests/test_per_message_powerpoint_export.py index 4d6963fc6..7dc09bdc8 100644 --- a/functional_tests/test_per_message_powerpoint_export.py +++ b/functional_tests/test_per_message_powerpoint_export.py @@ -2,8 +2,8 @@ # test_per_message_powerpoint_export.py """ Functional test for per-message PowerPoint export. -Version: 0.241.146 -Implemented in: 0.241.033 +Version: 0.250.215 +Implemented in: 0.241.033; Updated in: 0.250.215 This test ensures the message export flow exposes a PowerPoint route, uses the frontend PowerPoint action hook, prefers the message model @@ -426,6 +426,7 @@ def test_powerpoint_frontend_hooks_present() -> bool: assert "fetch('/api/message/export-powerpoint'" in frontend_source, 'Expected frontend fetch for the PowerPoint export endpoint' assert 'exportMessageAsPowerPoint' in frontend_source, 'Expected frontend PowerPoint export helper' + assert 'get_message_reference_citation_buckets' in ROUTE_FILE.read_text(encoding='utf-8') assert 'message_content_override' in frontend_source, 'Expected PowerPoint export requests to support edited assistant markdown overrides' assert 'delete requestBody.message_content_override;' in frontend_source, 'Expected artifact PowerPoint export to ignore visible-message overrides' assert 'dropdown-export-ppt-btn' in menu_source, 'Expected chat message menu PowerPoint action' diff --git a/ui_tests/test_chat_conversation_contents_drawer.py b/ui_tests/test_chat_conversation_contents_drawer.py index e3dc96180..505669872 100644 --- a/ui_tests/test_chat_conversation_contents_drawer.py +++ b/ui_tests/test_chat_conversation_contents_drawer.py @@ -1,10 +1,11 @@ # test_chat_conversation_contents_drawer.py """ UI test for the conversation contents drawer. -Version: 0.250.171 +Version: 0.250.215 Implemented in: 0.250.074 Documents mode added in: 0.250.159 Compact overflow-safe layout added in: 0.250.171 +Source/cited distinction added in: 0.250.215 This test validates user-message filtering, cited-document mode, safe labels, navigation, live updates, conversation replacement, keyboard closing, and @@ -239,38 +240,53 @@ def test_chat_conversation_contents_drawer(playwright, viewport): "bank-treasury-operations-quarterly-reference-" "with-an-intentionally-long-filename.xlsx" ) + source_document_tags = [ + { + "category": "document", + "document_id": "doc-1", + "title": document_title, + "file_name": document_file_name, + "classification": "Confidential", + "chunk_ids": ["doc-1_1", "doc-1_2"], + "scope": { + "type": "group", + "id": "group-1", + "name": "Product Documentation Workspace With A Long Display Name", + }, + }, + { + "category": "document", + "document_id": "doc-2", + "title": "Release Plan", + "file_name": "release-plan.pdf", + "classification": "None", + "chunk_ids": ["doc-2_4"], + "scope": { + "type": "personal", + "id": "user-1", + "name": "Personal", + }, + }, + ] page.route( "**/api/conversations/conversation-documents-pane/metadata", lambda route: _fulfill_json(route, { "title": "Documents pane conversation", - "tags": [ - { - "category": "document", - "document_id": "doc-1", - "title": document_title, - "file_name": document_file_name, - "classification": "Confidential", - "chunk_ids": ["doc-1_1", "doc-1_2"], - "scope": { - "type": "group", - "id": "group-1", - "name": "Product Documentation Workspace With A Long Display Name", - }, - }, - { - "category": "document", - "document_id": "doc-2", - "title": "Release Plan", - "file_name": "release-plan.pdf", - "classification": "None", - "chunk_ids": ["doc-2_4"], - "scope": { - "type": "personal", - "id": "user-1", - "name": "Personal", - }, - }, - ], + "tags": source_document_tags, + "used_documents_tracking_version": 1, + "legacy_used_documents": [], + "used_documents": [{ + **source_document_tags[0], + "chunk_ids": ["doc-1_99"], + "citation_ids": ["doc-1_99"], + "page_numbers": [2], + "citation_locations": [{ + "citation_id": "doc-1_99", + "page_number": 2, + "location_label": "Page", + "location_value": 2, + }], + }], }), ) @@ -294,18 +310,20 @@ def test_chat_conversation_contents_drawer(playwright, viewport): expect(documents_toggle).to_be_visible() expect(page.locator("#conversation-contents-drawer")).to_be_visible() expect(page.locator("#conversation-contents-title")).to_have_text("Used documents") - expect(documents_entries).to_have_count(2) + expect(page.locator("#conversation-contents-subtitle")).to_have_text( + "Documents cited in active responses" + ) + expect(documents_entries).to_have_count(1) expect(documents_entries.nth(0)).to_contain_text(document_title) expect(documents_entries.nth(0)).to_contain_text(document_file_name) expect(documents_entries.nth(0)).to_contain_text("Confidential") - expect(documents_entries.nth(0)).to_contain_text("Pages: 1, 2") + expect(documents_entries.nth(0)).to_contain_text("Pages: 2") expect(documents_entries.nth(0)).to_contain_text( "group scope: Product Documentation Workspace With A Long Display Name" ) expect(documents_entries.nth(0)).not_to_contain_text("chunks") expect(documents_entries.nth(0)).not_to_contain_text("ID:") - expect(documents_entries.nth(1)).to_contain_text("Release Plan") - expect(page.locator("#conversation-documents-count")).to_have_text("2") + expect(page.locator("#conversation-documents-count")).to_have_text("1") expect(page.locator("#conversation-documents-panel img[src='x']")).to_have_count(0) assert page.evaluate("() => window.__documentsPaneXss") is False _assert_no_horizontal_overflow( @@ -314,6 +332,39 @@ def test_chat_conversation_contents_drawer(playwright, viewport): ) _assert_no_horizontal_overflow(documents_entries.nth(0), "document entry") + documents_toggle.click() + expect(page.locator("#conversation-contents-drawer")).to_be_hidden() + + page.route( + "**/api/conversations/conversation-documents-legacy/metadata", + lambda route: _fulfill_json(route, { + "title": "Legacy documents pane conversation", + "tags": source_document_tags, + }), + ) + page.evaluate( + """ + () => { + window.currentConversationId = 'conversation-documents-legacy'; + window.dispatchEvent(new CustomEvent('chat:conversation-documents-refresh', { + detail: { + conversationId: 'conversation-documents-legacy', + autoOpen: true + } + })); + } + """ + ) + expect(documents_entries).to_have_count(2) + expect(documents_entries.nth(1)).to_contain_text("Release Plan") + expect(page.locator("#conversation-documents-count")).to_have_text("2") + expect(page.locator("#conversation-contents-drawer")).to_be_hidden() + expect(page.locator("#conversation-contents-subtitle")).to_have_text( + "Documents associated with this historical conversation" + ) + + documents_toggle.click() + expect(page.locator("#conversation-contents-drawer")).to_be_visible() page.locator("#conversation-contents-mode-contents").click() expect(page.locator("#conversation-contents-title")).to_have_text("Conversation contents") expect(page.locator("#conversation-contents-panel")).to_be_visible() diff --git a/ui_tests/test_chat_markdown_citation_lookup_payload.py b/ui_tests/test_chat_markdown_citation_lookup_payload.py index 4516667b8..b2a5cf5ba 100644 --- a/ui_tests/test_chat_markdown_citation_lookup_payload.py +++ b/ui_tests/test_chat_markdown_citation_lookup_payload.py @@ -2,8 +2,8 @@ """ UI test for markdown citation lookup payloads. -Version: 0.241.021 -Implemented in: 0.241.021 +Version: 0.250.215 +Implemented in: 0.241.021; Updated in: 0.250.215 This test ensures markdown citation buttons send document, page, and chunk context to `/api/get_citation` so text citation lookup can recover when the @@ -108,11 +108,92 @@ def handle_get_citation(route): }, true ); + messagesModule.appendMessage( + 'AI', + 'Workbook answer (Source: Budget, 2026.xlsx, Sheet: Q1, Final (Approved)) [#budget-doc_Q1]', + null, + 'assistant-sheet-1', + true, + [ + { + file_name: 'Budget, 2026.xlsx', + document_id: 'budget-doc', + citation_id: 'budget-doc_Q1', + chunk_id: 'Q1', + page_number: 'Q1, Final (Approved)', + sheet_name: 'Q1, Final (Approved)', + location_label: 'Sheet', + location_value: 'Q1, Final (Approved)', + }, + ], + [], + [], + null, + null, + { + id: 'assistant-sheet-1', + role: 'assistant', + content: 'Workbook answer (Source: Budget, 2026.xlsx, Sheet: Q1, Final (Approved)) [#budget-doc_Q1]', + }, + true + ); + messagesModule.appendMessage( + 'AI', + 'A (Source: A.pdf, Page: 1). B (Source: B.pdf, Page: 2) [#b-doc_2]', + null, + 'assistant-adjacent-1', + true, + [ + { + file_name: 'B.pdf', + document_id: 'b-doc', + citation_id: 'b-doc_2', + chunk_id: '2', + page_number: 2, + location_label: 'Page', + location_value: 2, + }, + ], + [], + [], + null, + null, + { + id: 'assistant-adjacent-1', + role: 'assistant', + content: 'A (Source: A.pdf, Page: 1). B (Source: B.pdf, Page: 2) [#b-doc_2]', + }, + true + ); } """ ) message = page.locator('.message[data-message-id="assistant-md-1"]') + sheet_message = page.locator( + '.message[data-message-id="assistant-sheet-1"]' + ) + sheet_link = sheet_message.locator( + 'a.citation-link[data-citation-id="budget-doc_Q1"]' + ) + expect(sheet_link).to_be_visible() + expect(sheet_link).to_have_text("Q1, Final (Approved)") + expect(sheet_link).to_have_attribute( + "data-sheet-name", + "Q1, Final (Approved)" + ) + adjacent_message = page.locator( + '.message[data-message-id="assistant-adjacent-1"]' + ) + expect(adjacent_message.locator(".message-text")).to_contain_text( + "(Source: A.pdf, Page: 1)" + ) + adjacent_link = adjacent_message.locator( + 'a.citation-link[data-citation-id="b-doc_2"]' + ) + expect(adjacent_link).to_be_visible() + expect(adjacent_link).to_have_text("2") + citation_toggle = message.locator('.citation-toggle-btn') expect(citation_toggle).to_be_visible() citation_toggle.click() diff --git a/ui_tests/test_chat_scope_lock_and_conversation_details_escaping.py b/ui_tests/test_chat_scope_lock_and_conversation_details_escaping.py index e622b94dc..f4cf6e6dc 100644 --- a/ui_tests/test_chat_scope_lock_and_conversation_details_escaping.py +++ b/ui_tests/test_chat_scope_lock_and_conversation_details_escaping.py @@ -1,8 +1,8 @@ # test_chat_scope_lock_and_conversation_details_escaping.py """ UI test for chat scope-lock and conversation-details escaping. -Version: 0.241.019 -Implemented in: 0.241.019 +Version: 0.250.215 +Implemented in: 0.241.019; Updated in: 0.250.215 This test ensures malicious workspace names and conversation metadata render as inert text in the chat scope-lock modal and conversation-details modal. @@ -92,6 +92,18 @@ def test_chat_scope_lock_and_conversation_details_escape_malicious_metadata(play "name": document_scope_name, }, }, + { + "category": "document", + "document_id": "doc-2", + "title": "Retrieved but not cited", + "classification": "Public", + "chunk_ids": ["chunk_2_p1"], + "scope": { + "type": "personal", + "id": "user-1", + "name": "Personal", + }, + }, { "category": "semantic", "value": semantic_tag, @@ -109,6 +121,22 @@ def test_chat_scope_lock_and_conversation_details_escape_malicious_metadata(play "value": web_source, }, ], + "used_documents_tracking_version": 1, + "legacy_used_documents": [], + "used_documents": [ + { + "category": "document", + "document_id": "doc-1", + "title": document_title, + "classification": classification_label, + "chunk_ids": ["chunk_1_p2"], + "scope": { + "type": "group", + "id": "group-1", + "name": document_scope_name, + }, + }, + ], "strict": False, "classification": [classification_label], "last_updated": "2026-05-05T12:00:00Z", @@ -202,7 +230,19 @@ def fulfill_empty_docs_or_tags(route): expect(details_content).to_contain_text(participant_name) expect(details_content).to_contain_text(participant_email) expect(details_content).to_contain_text(document_title) + expect(details_content).to_contain_text("Retrieved but not cited") expect(details_content).to_contain_text(document_scope_name) + expect(details_content).to_contain_text("Source documents") + expect( + page.locator( + "#conversation-details-modal .conversation-document-cited-badge" + ) + ).to_have_count(1) + expect( + page.locator( + "#conversation-details-modal .conversation-document-cited-badge" + ) + ).to_have_text("Cited") expect(details_content).to_contain_text(classification_label) expect(details_content).to_contain_text(semantic_tag) expect(details_content).to_contain_text(model_tag) From b1ca1b812fcc39d9ca811728b1a7740993922e78 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Mon, 17 Aug 2026 23:03:37 -0400 Subject: [PATCH 2/2] Bound citation matching patterns against ReDoS CodeQL flagged three polynomial regular expressions in the new citation tracking module. The inline citation group, visible source reference, and page range patterns all combined unbounded whitespace or lazy quantifiers with variable-length character classes, so hostile or malformed model output could force quadratic backtracking while parsing references. Replace the unbounded quantifiers with bounded repetition, precompile the page range pattern, and add a regression test that asserts adversarial inputs are parsed well within a fixed time budget. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../single_app/functions_citation_tracking.py | 13 ++++++--- ...RCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md | 1 + .../test_chat_cited_source_tracking.py | 29 +++++++++++++++++++ 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/application/single_app/functions_citation_tracking.py b/application/single_app/functions_citation_tracking.py index e1d90aa04..b2da0ab74 100644 --- a/application/single_app/functions_citation_tracking.py +++ b/application/single_app/functions_citation_tracking.py @@ -12,14 +12,19 @@ USED_DOCUMENTS_TRACKING_VERSION = 1 _INLINE_DOCUMENT_CITATION_GROUP_PATTERN = re.compile( - r"\[\s*#([^\]\r\n]+)\]", + r"\[\s{0,8}#([^\]\r\n]{1,1024})\]", re.IGNORECASE, ) _INLINE_DOCUMENT_SOURCE_PATTERN = re.compile( - r"\(Source:\s*(.+?),\s*(Page(?:s)?|Sheet(?:s)?|Location):\s*(.*?)\)" - r"(?=\s*(?:\[#|$|[.;,!? \n]))", + r"\(Source:((?:(?!\(Source:).){1,256}?)," + r"\s{0,8}(Page(?:s)?|Sheet(?:s)?|Location):" + r"((?:(?!\(Source:).){0,256}?)\)" + r"(?=\s{0,8}(?:\[#|$|[.;,!? \n]))", re.IGNORECASE, ) +_PAGE_RANGE_PATTERN = re.compile( + r"(\d{1,9})\s{0,8}[-\u2013\u2014]\s{0,8}(\d{1,9})" +) _HTTP_URL_PATTERN = re.compile(r"https?://[^\s<>\"']+", re.IGNORECASE) _TRAILING_URL_PUNCTUATION = ".,;:!?" _MERGED_DOCUMENT_LIST_FIELDS = ( @@ -124,7 +129,7 @@ def _location_reference_matches( return False for token in explicit_tokens: - range_match = re.fullmatch(r"(\d+)\s*[-\u2013\u2014]\s*(\d+)", token) + range_match = _PAGE_RANGE_PATTERN.fullmatch(token) if not range_match: continue range_start, range_end = ( diff --git a/docs/explanation/fixes/SOURCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md b/docs/explanation/fixes/SOURCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md index 2860e34a6..e09b91e12 100644 --- a/docs/explanation/fixes/SOURCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md +++ b/docs/explanation/fixes/SOURCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md @@ -49,6 +49,7 @@ Conversation records persist `used_documents` for exact active-response usage an ### Code changes - Exact `[#citation-id]` tokens are matched to returned document records after final response content is settled. Explicit `(Source: filename, Page/Sheet/Location: value)` references provide a strict fallback for response paths that omit the hidden ID. +- Citation matching patterns use bounded quantifiers so hostile or malformed model output cannot trigger polynomial backtracking (ReDoS) while parsing references. - Final-response HTTP(S) URLs are normalized and matched to returned web records. - Full source arrays remain unchanged for the per-message **Sources** disclosure and JSON audit output. - The **Used documents** drawer reads exact active-response usage plus an explicit historical fallback. diff --git a/functional_tests/test_chat_cited_source_tracking.py b/functional_tests/test_chat_cited_source_tracking.py index 87db78764..3b8ba79c7 100644 --- a/functional_tests/test_chat_cited_source_tracking.py +++ b/functional_tests/test_chat_cited_source_tracking.py @@ -12,6 +12,7 @@ from copy import deepcopy from pathlib import Path import sys +import time from test_support.versioning import assert_app_version_at_least @@ -208,6 +209,33 @@ def test_schema_aware_citation_location_is_shared(): ) == ("Page", "4") +def test_citation_patterns_resist_adversarial_input(): + """Bounded citation patterns must not degrade on hostile model output.""" + adversarial_inputs = ( + "[#" + ("[#\\" * 20000), + "(Source:" + (" " * 40000), + "(Source: a, Page:" + (" " * 40000), + "(Source: a, Page: " + ("0" * 40000) + "-1)", + "[#" + ("0" * 40000), + ) + + hybrid_citations = _hybrid_sources() + started_at = time.monotonic() + for adversarial_input in adversarial_inputs: + tracked = build_cited_source_subsets( + adversarial_input, + hybrid_citations=hybrid_citations, + web_search_citations=_web_sources(), + ) + assert tracked["cited_hybrid_citations"] == [] + assert tracked["cited_web_search_citations"] == [] + elapsed_seconds = time.monotonic() - started_at + + assert elapsed_seconds < 5, ( + f"Citation matching took {elapsed_seconds:.2f}s on adversarial input" + ) + + def test_used_documents_collapse_cited_chunks_only(): """Collapse multiple cited chunks while excluding retrieved-only documents.""" source_tags = [ @@ -614,6 +642,7 @@ def test_version_is_available(): test_hidden_id_prevents_same_page_source_overmatching, test_visible_source_matching_handles_sheet_and_filename_commas, test_schema_aware_citation_location_is_shared, + test_citation_patterns_resist_adversarial_input, test_used_documents_collapse_cited_chunks_only, test_reference_buckets_use_strict_empty_values_and_legacy_fallback, test_conversation_tracking_preserves_legacy_and_merges_exact_usage,