diff --git a/application/single_app/collaboration_models.py b/application/single_app/collaboration_models.py index 0c98dd940..e2f322ea2 100644 --- a/application/single_app/collaboration_models.py +++ b/application/single_app/collaboration_models.py @@ -678,6 +678,9 @@ def build_collaboration_message_doc_from_legacy( 'augmented', 'hybrid_citations', 'web_search_citations', + 'citation_tracking_version', + 'cited_hybrid_citations', + 'cited_web_search_citations', 'agent_citations', 'agent_display_name', 'agent_name', diff --git a/application/single_app/config.py b/application/single_app/config.py index 534b9a812..3661cf2b0 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -96,7 +96,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.214" +VERSION = "0.250.215" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_citation_tracking.py b/application/single_app/functions_citation_tracking.py new file mode 100644 index 000000000..b2da0ab74 --- /dev/null +++ b/application/single_app/functions_citation_tracking.py @@ -0,0 +1,691 @@ +# functions_citation_tracking.py +"""Track retrieved sources separately from explicitly cited references.""" + +import html +import re +from copy import deepcopy +from typing import Any, Dict, Iterable, List, Mapping, Optional +from urllib.parse import urlsplit, urlunsplit + + +CITATION_TRACKING_VERSION = 1 +USED_DOCUMENTS_TRACKING_VERSION = 1 + +_INLINE_DOCUMENT_CITATION_GROUP_PATTERN = re.compile( + r"\[\s{0,8}#([^\]\r\n]{1,1024})\]", + re.IGNORECASE, +) +_INLINE_DOCUMENT_SOURCE_PATTERN = re.compile( + r"\(Source:((?:(?!\(Source:).){1,256}?)," + r"\s{0,8}(Page(?:s)?|Sheet(?:s)?|Location):" + r"((?:(?!\(Source:).){0,256}?)\)" + r"(?=\s{0,8}(?:\[#|$|[.;,!? \n]))", + re.IGNORECASE, +) +_PAGE_RANGE_PATTERN = re.compile( + r"(\d{1,9})\s{0,8}[-\u2013\u2014]\s{0,8}(\d{1,9})" +) +_HTTP_URL_PATTERN = re.compile(r"https?://[^\s<>\"']+", re.IGNORECASE) +_TRAILING_URL_PUNCTUATION = ".,;:!?" +_MERGED_DOCUMENT_LIST_FIELDS = ( + "chunk_ids", + "citation_ids", + "page_numbers", + "sheet_names", +) + + +def _as_dict_list(value: Any) -> List[Dict[str, Any]]: + if not isinstance(value, list): + return [] + return [deepcopy(item) for item in value if isinstance(item, dict)] + + +def _append_unique(values: List[Any], value: Any) -> None: + if value in (None, "") or value in values: + return + values.append(deepcopy(value)) + + +def _append_unique_mapping(values: List[Dict[str, Any]], value: Dict[str, Any]) -> None: + if not value or value in values: + return + values.append(deepcopy(value)) + + +def _normalize_reference_id(value: Any) -> str: + return str(value or "").strip().lstrip("#").strip() + + +def extract_explicit_document_citation_ids(content: Any) -> List[str]: + """Return ordered, unique citation IDs from explicit ``[#id]`` groups.""" + citation_ids: List[str] = [] + for citation_group in _INLINE_DOCUMENT_CITATION_GROUP_PATTERN.findall( + str(content or "") + ): + for raw_citation_id in re.split(r"[;,]", citation_group): + citation_id = _normalize_reference_id(raw_citation_id) + if citation_id and citation_id not in citation_ids: + citation_ids.append(citation_id) + return citation_ids + + +def _normalize_source_text(value: Any) -> str: + return re.sub(r"\s+", " ", html.unescape(str(value or ""))).strip().lower() + + +def _extract_explicit_document_source_references( + content: Any, +) -> List[Dict[str, str]]: + references = [] + for file_name, location_label, location_value in ( + _INLINE_DOCUMENT_SOURCE_PATTERN.findall(str(content or "")) + ): + normalized_file_name = _normalize_source_text(file_name) + normalized_location_label = ( + _normalize_source_text(location_label).rstrip("s") + ) + normalized_location_value = _normalize_source_text(location_value) + if ( + normalized_file_name + and normalized_location_label + and normalized_location_value + ): + references.append({ + "file_name": normalized_file_name, + "location_label": normalized_location_label, + "location_value": normalized_location_value, + }) + return references + + +def _parse_numeric_location(value: Any) -> Optional[int]: + normalized_value = _normalize_source_text(value) + if not normalized_value or not normalized_value.isdigit(): + return None + return int(normalized_value) + + +def _location_reference_matches( + explicit_location: str, + citation_location: Any, +) -> bool: + normalized_citation_location = _normalize_source_text(citation_location) + if not normalized_citation_location: + return False + if normalized_citation_location == explicit_location: + return True + + explicit_tokens = [ + token.strip() + for token in re.split(r"[,;]", explicit_location) + if token.strip() + ] + if normalized_citation_location in explicit_tokens: + return True + + citation_number = _parse_numeric_location(normalized_citation_location) + if citation_number is None: + return False + + for token in explicit_tokens: + range_match = _PAGE_RANGE_PATTERN.fullmatch(token) + if not range_match: + continue + range_start, range_end = ( + int(range_match.group(1)), + int(range_match.group(2)), + ) + lower_bound = min(range_start, range_end) + upper_bound = max(range_start, range_end) + if lower_bound <= citation_number <= upper_bound: + return True + return False + + +def resolve_citation_location( + page_number: Any = None, + chunk_text: Any = None, + sheet_name: Any = None, + is_tabular: bool = False, +) -> tuple[str, str]: + """Return a schema-, sheet-, or page-aware citation location.""" + if sheet_name not in (None, ""): + return "Sheet", str(sheet_name) + + normalized_chunk_text = str(chunk_text or "").strip() + if is_tabular and ( + normalized_chunk_text.startswith("Tabular workbook:") + or normalized_chunk_text.startswith("Tabular data file:") + ): + return "Location", "Workbook Schema" + + return "Page", str(page_number or 1) + + +def _source_reference_matches_citation( + source_reference: Mapping[str, str], + citation: Mapping[str, Any], +) -> bool: + citation_file_name = _normalize_source_text( + citation.get("file_name") + or citation.get("title") + ) + if ( + not citation_file_name + or citation_file_name != source_reference.get("file_name") + ): + return False + + explicit_label = source_reference.get("location_label") + citation_label = _normalize_source_text(citation.get("location_label")) + if not citation_label: + citation_label = "sheet" if citation.get("sheet_name") else "page" + citation_label = citation_label.rstrip("s") + if explicit_label != "location" and citation_label != explicit_label: + return False + + location_candidates = ( + citation.get("location_value"), + citation.get("sheet_name"), + citation.get("page_number"), + citation.get("chunk_sequence"), + ) + return any( + _location_reference_matches( + source_reference.get("location_value", ""), + candidate, + ) + for candidate in location_candidates + ) + + +def _trim_url_candidate(value: Any) -> str: + candidate = html.unescape(str(value or "")).strip() + while candidate and candidate[-1] in _TRAILING_URL_PUNCTUATION: + candidate = candidate[:-1] + while candidate.endswith(")") and candidate.count(")") > candidate.count("("): + candidate = candidate[:-1] + while candidate.endswith("]") and candidate.count("]") > candidate.count("["): + candidate = candidate[:-1] + return candidate + + +def normalize_http_url(value: Any) -> str: + """Normalize an HTTP(S) URL for exact response-to-source matching.""" + candidate = _trim_url_candidate(value) + if not candidate: + return "" + + try: + parsed = urlsplit(candidate) + except ValueError: + return "" + + scheme = parsed.scheme.lower() + if scheme not in {"http", "https"} or not parsed.hostname: + return "" + + hostname = parsed.hostname.lower() + try: + port = parsed.port + except ValueError: + return "" + if port and not ( + (scheme == "http" and port == 80) + or (scheme == "https" and port == 443) + ): + hostname = f"{hostname}:{port}" + + path = parsed.path or "" + if path == "/": + path = "" + + return urlunsplit((scheme, hostname, path, parsed.query, "")) + + +def extract_explicit_web_urls(content: Any) -> List[str]: + """Return ordered, unique normalized HTTP(S) URLs from final response text.""" + normalized_urls: List[str] = [] + for raw_url in _HTTP_URL_PATTERN.findall(str(content or "")): + normalized_url = normalize_http_url(raw_url) + if normalized_url and normalized_url not in normalized_urls: + normalized_urls.append(normalized_url) + return normalized_urls + + +def _get_document_citation_id(citation: Mapping[str, Any]) -> str: + return _normalize_reference_id( + citation.get("citation_id") + or citation.get("id") + ) + + +def _get_web_citation_url(citation: Mapping[str, Any]) -> str: + return normalize_http_url( + citation.get("url") + or citation.get("href") + or citation.get("link") + ) + + +def build_cited_source_subsets( + content: Any, + hybrid_citations: Any = None, + web_search_citations: Any = None, +) -> Dict[str, Any]: + """Match explicit final-response references to returned source records.""" + document_citation_ids = set(extract_explicit_document_citation_ids(content)) + document_source_references = _extract_explicit_document_source_references( + content + ) + explicit_web_urls = set(extract_explicit_web_urls(content)) + + hybrid_source_records = _as_dict_list(hybrid_citations) + exact_document_citations = [ + citation + for citation in hybrid_source_records + if _get_document_citation_id(citation) in document_citation_ids + ] + unmatched_source_references = [ + source_reference + for source_reference in document_source_references + if not any( + _source_reference_matches_citation( + source_reference, + exact_citation, + ) + for exact_citation in exact_document_citations + ) + ] + + cited_hybrid_citations: List[Dict[str, Any]] = [] + seen_document_citations = set() + for citation in hybrid_source_records: + citation_id = _get_document_citation_id(citation) + matched_by_id = bool( + citation_id + and citation_id in document_citation_ids + ) + matched_by_source = any( + _source_reference_matches_citation(source_reference, citation) + for source_reference in unmatched_source_references + ) + if not matched_by_id and not matched_by_source: + continue + if matched_by_id: + identity = ( + "citation_id", + citation_id, + str(citation.get("document_id") or ""), + str(citation.get("chunk_id") or ""), + ) + else: + identity = ( + "source_location", + _extract_document_id(citation), + _normalize_source_text( + citation.get("file_name") + or citation.get("title") + ), + _normalize_source_text( + citation.get("location_value") + or citation.get("sheet_name") + or citation.get("page_number") + or citation.get("chunk_sequence") + ), + ) + if identity in seen_document_citations: + continue + seen_document_citations.add(identity) + cited_hybrid_citations.append(citation) + + cited_web_search_citations: List[Dict[str, Any]] = [] + seen_web_urls = set() + for citation in _as_dict_list(web_search_citations): + normalized_url = _get_web_citation_url(citation) + if ( + not normalized_url + or normalized_url not in explicit_web_urls + or normalized_url in seen_web_urls + ): + continue + seen_web_urls.add(normalized_url) + cited_web_search_citations.append(citation) + + return { + "citation_tracking_version": CITATION_TRACKING_VERSION, + "cited_hybrid_citations": cited_hybrid_citations, + "cited_web_search_citations": cited_web_search_citations, + } + + +def _message_has_citation_tracking(message: Mapping[str, Any]) -> bool: + tracking_version = message.get("citation_tracking_version") + if isinstance(tracking_version, int) and tracking_version >= 1: + return True + return ( + "cited_hybrid_citations" in message + or "cited_web_search_citations" in message + ) + + +def get_message_source_citation_buckets( + message: Optional[Mapping[str, Any]], +) -> Dict[str, List[Any]]: + """Return complete stored source/execution buckets for audit and Sources UI.""" + safe_message = message if isinstance(message, Mapping) else {} + + def ensure_list(value: Any) -> List[Any]: + if not value: + return [] + return deepcopy(value) if isinstance(value, list) else [deepcopy(value)] + + return { + "legacy": ensure_list(safe_message.get("citations")), + "hybrid": ensure_list(safe_message.get("hybrid_citations")), + "web": ensure_list(safe_message.get("web_search_citations")), + "agent": ensure_list(safe_message.get("agent_citations")), + } + + +def get_message_reference_citation_buckets( + message: Optional[Mapping[str, Any]], +) -> Dict[str, List[Any]]: + """Return exact references, with broad source fallback for legacy messages.""" + safe_message = message if isinstance(message, Mapping) else {} + source_buckets = get_message_source_citation_buckets(safe_message) + if not _message_has_citation_tracking(safe_message): + return source_buckets + + referenced_agent_citations = [] + for citation in source_buckets["agent"]: + if ( + isinstance(citation, Mapping) + and str(citation.get("function_name") or "").strip().lower() + == "azure_ai_foundry_web_search" + ): + continue + referenced_agent_citations.append(deepcopy(citation)) + + return { + "legacy": source_buckets["legacy"], + "hybrid": _as_dict_list(safe_message.get("cited_hybrid_citations")), + "web": _as_dict_list(safe_message.get("cited_web_search_citations")), + "agent": referenced_agent_citations, + } + + +def _extract_document_id(citation: Mapping[str, Any]) -> str: + document_id = str(citation.get("document_id") or "").strip() + if document_id: + return document_id + + citation_id = _get_document_citation_id(citation) + if not citation_id: + return "" + if "_" in citation_id: + return citation_id.rsplit("_", 1)[0] + return citation_id + + +def _get_document_tags(conversation: Mapping[str, Any]) -> List[Dict[str, Any]]: + tags = conversation.get("tags") + if not isinstance(tags, list): + return [] + return [ + deepcopy(tag) + for tag in tags + if isinstance(tag, dict) + and tag.get("category") == "document" + and str(tag.get("document_id") or "").strip() + ] + + +def _scope_from_citation(citation: Mapping[str, Any]) -> Dict[str, Any]: + raw_scope = citation.get("scope") + if isinstance(raw_scope, dict): + scope_type = str( + raw_scope.get("type") + or raw_scope.get("scope") + or "" + ).strip() + scope_id = str( + raw_scope.get("id") + or raw_scope.get("scope_id") + or "" + ).strip() + if scope_type and scope_id: + scope = { + "type": scope_type, + "id": scope_id, + } + scope_name = str(raw_scope.get("name") or "").strip() + if scope_name: + scope["name"] = scope_name + return scope + + public_workspace_id = str( + citation.get("public_workspace_id") or "" + ).strip() + if public_workspace_id: + return {"type": "public", "id": public_workspace_id} + + group_id = str(citation.get("group_id") or "").strip() + if group_id: + return {"type": "group", "id": group_id} + + user_id = str(citation.get("user_id") or "").strip() + if user_id: + return {"type": "personal", "id": user_id} + return {} + + +def _merge_document_record( + target: Dict[str, Any], + incoming: Mapping[str, Any], +) -> Dict[str, Any]: + for field_name in _MERGED_DOCUMENT_LIST_FIELDS: + target_values = target.setdefault(field_name, []) + incoming_values = incoming.get(field_name) + if not isinstance(incoming_values, list): + continue + for value in incoming_values: + _append_unique(target_values, value) + + target_locations = target.setdefault("citation_locations", []) + incoming_locations = incoming.get("citation_locations") + if isinstance(incoming_locations, list): + for location in incoming_locations: + if isinstance(location, dict): + _append_unique_mapping(target_locations, location) + + for field_name in ( + "title", + "file_name", + "classification", + "scope", + ): + current_value = target.get(field_name) + incoming_value = incoming.get(field_name) + if current_value in (None, "", {}, "Unknown Document") and incoming_value not in ( + None, + "", + {}, + ): + target[field_name] = deepcopy(incoming_value) + return target + + +def build_used_documents( + cited_hybrid_citations: Any, + source_document_tags: Any = None, +) -> List[Dict[str, Any]]: + """Collapse exact cited chunks into document-level used-document records.""" + tag_by_document_id = { + str(tag.get("document_id") or "").strip(): tag + for tag in _as_dict_list(source_document_tags) + if str(tag.get("document_id") or "").strip() + } + documents_by_id: Dict[str, Dict[str, Any]] = {} + + for citation in _as_dict_list(cited_hybrid_citations): + document_id = _extract_document_id(citation) + if not document_id: + continue + + document = documents_by_id.get(document_id) + if document is None: + source_tag = tag_by_document_id.get(document_id) + document = deepcopy(source_tag) if source_tag else { + "category": "document", + "document_id": document_id, + } + document["category"] = "document" + document["document_id"] = document_id + document["chunk_ids"] = [] + document["citation_ids"] = [] + document["page_numbers"] = [] + document["sheet_names"] = [] + document["citation_locations"] = [] + documents_by_id[document_id] = document + + citation_id = _get_document_citation_id(citation) + _append_unique(document["citation_ids"], citation_id) + _append_unique( + document["chunk_ids"], + citation_id or citation.get("chunk_id"), + ) + _append_unique(document["page_numbers"], citation.get("page_number")) + _append_unique(document["sheet_names"], citation.get("sheet_name")) + + location = { + key: deepcopy(citation.get(key)) + for key in ( + "citation_id", + "chunk_id", + "page_number", + "sheet_name", + "location_label", + "location_value", + ) + if citation.get(key) not in (None, "") + } + _append_unique_mapping(document["citation_locations"], location) + + incoming_metadata = { + "title": citation.get("title") or citation.get("file_name"), + "file_name": citation.get("file_name") or citation.get("title"), + "classification": ( + citation.get("classification") + or citation.get("document_classification") + ), + "scope": _scope_from_citation(citation), + } + _merge_document_record(document, incoming_metadata) + + return list(documents_by_id.values()) + + +def merge_used_documents( + existing_documents: Any, + incoming_documents: Any, +) -> List[Dict[str, Any]]: + """Merge exact used-document aggregates without losing cited locations.""" + merged_by_id: Dict[str, Dict[str, Any]] = {} + for document in [ + *_as_dict_list(existing_documents), + *_as_dict_list(incoming_documents), + ]: + document_id = str(document.get("document_id") or "").strip() + if not document_id: + continue + if document_id not in merged_by_id: + merged_by_id[document_id] = deepcopy(document) + continue + _merge_document_record(merged_by_id[document_id], document) + return list(merged_by_id.values()) + + +def initialize_conversation_used_document_tracking( + conversation: Dict[str, Any], +) -> bool: + """Snapshot legacy document tags before the first strictly tracked turn.""" + if not isinstance(conversation, dict): + return False + tracking_version = conversation.get("used_documents_tracking_version") + if isinstance(tracking_version, int) and tracking_version >= 1: + return False + + conversation["used_documents_tracking_version"] = ( + USED_DOCUMENTS_TRACKING_VERSION + ) + conversation["legacy_used_documents"] = _get_document_tags(conversation) + conversation["used_documents"] = [] + return True + + +def merge_cited_documents_into_conversation( + conversation: Dict[str, Any], + cited_hybrid_citations: Any, +) -> List[Dict[str, Any]]: + """Merge one finalized assistant response into exact conversation usage.""" + initialize_conversation_used_document_tracking(conversation) + source_document_tags = _get_document_tags(conversation) + incoming_documents = build_used_documents( + cited_hybrid_citations, + source_document_tags=source_document_tags, + ) + conversation["used_documents"] = merge_used_documents( + conversation.get("used_documents"), + incoming_documents, + ) + return deepcopy(conversation["used_documents"]) + + +def _message_is_active_assistant(message: Mapping[str, Any]) -> bool: + if message.get("role") != "assistant": + return False + metadata = message.get("metadata") + metadata = metadata if isinstance(metadata, Mapping) else {} + if metadata.get("is_deleted") is True: + return False + if metadata.get("is_generated_chat_artifact") is True: + return False + thread_info = metadata.get("thread_info") + thread_info = thread_info if isinstance(thread_info, Mapping) else {} + return thread_info.get("active_thread") is not False + + +def rebuild_conversation_used_documents( + conversation: Dict[str, Any], + messages: Iterable[Mapping[str, Any]], + rebuild_legacy: bool = False, +) -> List[Dict[str, Any]]: + """Rebuild exact usage after a retry, switch, deletion, or fork mutation.""" + initialize_conversation_used_document_tracking(conversation) + cited_hybrid_citations: List[Dict[str, Any]] = [] + legacy_hybrid_citations: List[Dict[str, Any]] = [] + for message in messages or []: + if not isinstance(message, Mapping) or not _message_is_active_assistant(message): + continue + if _message_has_citation_tracking(message): + cited_hybrid_citations.extend( + _as_dict_list(message.get("cited_hybrid_citations")) + ) + elif rebuild_legacy: + legacy_hybrid_citations.extend( + _as_dict_list(message.get("hybrid_citations")) + ) + + source_document_tags = _get_document_tags(conversation) + conversation["used_documents"] = build_used_documents( + cited_hybrid_citations, + source_document_tags=source_document_tags, + ) + if rebuild_legacy: + conversation["legacy_used_documents"] = build_used_documents( + legacy_hybrid_citations, + source_document_tags=source_document_tags, + ) + return deepcopy(conversation["used_documents"]) diff --git a/application/single_app/functions_collaboration.py b/application/single_app/functions_collaboration.py index d434cb62a..75e9e78c0 100644 --- a/application/single_app/functions_collaboration.py +++ b/application/single_app/functions_collaboration.py @@ -40,6 +40,11 @@ log_conversation_deletion, ) from functions_appinsights import log_event +from functions_citation_tracking import ( + initialize_conversation_used_document_tracking, + merge_cited_documents_into_conversation, + rebuild_conversation_used_documents, +) from functions_conversation_cache import bump_conversation_cache_version, invalidate_conversation_cache_for_item from functions_documents import sync_chat_upload_workspace_document_sharing_for_collaboration from functions_group import ( @@ -64,6 +69,21 @@ COLLABORATION_EVENT_PUBLISHERS = [] +CITATION_TRACKING_CONVERSATION_FIELDS = ( + 'used_documents_tracking_version', + 'legacy_used_documents', + 'used_documents', +) + + +def _copy_citation_tracking_conversation_fields(target, source): + if not isinstance(target, dict) or not isinstance(source, dict): + return target + for field_name in CITATION_TRACKING_CONVERSATION_FIELDS: + if field_name in source: + target[field_name] = deepcopy(source.get(field_name)) + return target + def register_collaboration_event_publisher(event_publisher): if callable(event_publisher) and event_publisher not in COLLABORATION_EVENT_PUBLISHERS: @@ -355,7 +375,7 @@ def serialize_collaboration_message(message_doc): message_doc.get('id'), ) or serialized_content - return { + payload = { 'id': message_doc.get('id'), 'conversation_id': message_doc.get('conversation_id'), 'role': serialized_role, @@ -382,6 +402,14 @@ def serialize_collaboration_message(message_doc): 'extracted_text': message_doc.get('extracted_text'), 'vision_analysis': message_doc.get('vision_analysis'), } + for field_name in ( + 'citation_tracking_version', + 'cited_hybrid_citations', + 'cited_web_search_citations', + ): + if field_name in message_doc: + payload[field_name] = deepcopy(message_doc.get(field_name)) + return payload def _get_collaboration_source_message(message_doc): @@ -729,6 +757,15 @@ def build_collaboration_message_metadata_payload(message_doc, conversation_doc): 'extracted_text': message_doc.get('extracted_text') or payload.get('extracted_text'), 'vision_analysis': message_doc.get('vision_analysis') or payload.get('vision_analysis'), }) + for field_name in ( + 'citation_tracking_version', + 'cited_hybrid_citations', + 'cited_web_search_citations', + ): + if field_name in message_doc: + payload[field_name] = deepcopy(message_doc.get(field_name)) + elif field_name in (source_message_doc or {}): + payload[field_name] = deepcopy(source_message_doc.get(field_name)) payload['metadata'] = merged_metadata return payload @@ -820,6 +857,9 @@ def serialize_collaboration_conversation(conversation_doc, current_user_id, user 'is_hidden': bool((user_state or {}).get('is_hidden', False)), 'classification': list(conversation_doc.get('classification', []) or []), 'tags': list(conversation_doc.get('tags', []) or []), + 'used_documents_tracking_version': conversation_doc.get('used_documents_tracking_version'), + 'legacy_used_documents': deepcopy(list(conversation_doc.get('legacy_used_documents', []) or [])), + 'used_documents': deepcopy(list(conversation_doc.get('used_documents', []) or [])), 'strict': bool(conversation_doc.get('strict', False)), 'summary': conversation_doc.get('summary'), 'has_unread_assistant_response': False, @@ -959,6 +999,10 @@ def ensure_personal_collaboration_for_legacy_conversation(source_conversation_id collaboration_conversation_doc['source_conversation_id'] = source_conversation_id collaboration_conversation_doc['classification'] = list(source_conversation_doc.get('classification', []) or []) collaboration_conversation_doc['tags'] = list(source_conversation_doc.get('tags', []) or []) + _copy_citation_tracking_conversation_fields( + collaboration_conversation_doc, + source_conversation_doc, + ) collaboration_conversation_doc['strict'] = bool(source_conversation_doc.get('strict', False)) collaboration_conversation_doc['summary'] = source_conversation_doc.get('summary') @@ -1145,6 +1189,10 @@ def ensure_group_collaboration_for_legacy_conversation(source_conversation_id, o collaboration_conversation_doc['classification'] = list(source_conversation_doc.get('classification', []) or []) collaboration_conversation_doc['tags'] = list(source_conversation_doc.get('tags', []) or []) + _copy_citation_tracking_conversation_fields( + collaboration_conversation_doc, + source_conversation_doc, + ) collaboration_conversation_doc['strict'] = bool(source_conversation_doc.get('strict', False)) collaboration_conversation_doc['summary'] = source_conversation_doc.get('summary') collaboration_conversation_doc['legacy_source_conversation_id'] = source_conversation_id @@ -1749,6 +1797,19 @@ def _save_collaboration_message_doc(conversation_doc, message_doc): cosmos_collaboration_messages_container.upsert_item(message_doc) + if ( + str(message_doc.get('role') or '').strip().lower() == 'assistant' + and ( + message_doc.get('citation_tracking_version') is not None + or 'cited_hybrid_citations' in message_doc + ) + ): + initialize_conversation_used_document_tracking(conversation_doc) + merge_cited_documents_into_conversation( + conversation_doc, + message_doc.get('cited_hybrid_citations'), + ) + conversation_doc['last_message_at'] = message_doc.get('timestamp') conversation_doc['last_message_preview'] = ( message_doc.get('metadata', {}).get('last_message_preview', '') @@ -1814,6 +1875,11 @@ def sync_collaboration_conversation_metadata_from_source(conversation_doc, sourc 'classification': deepcopy(list(source_conversation_doc.get('classification', []) or [])), 'summary': deepcopy(source_conversation_doc.get('summary')), } + for field_name in CITATION_TRACKING_CONVERSATION_FIELDS: + if field_name in source_conversation_doc: + metadata_fields[field_name] = deepcopy( + source_conversation_doc.get(field_name) + ) updated = False for field_name, field_value in metadata_fields.items(): @@ -1868,6 +1934,10 @@ def ensure_collaboration_source_conversation(conversation_doc, current_user): 'collaboration_conversation_id': (conversation_doc or {}).get('id'), 'is_hidden': True, } + _copy_citation_tracking_conversation_fields( + source_conversation_doc, + conversation_doc, + ) source_updated = True else: synchronized_values = { @@ -1884,6 +1954,11 @@ def ensure_collaboration_source_conversation(conversation_doc, current_user): 'collaboration_conversation_id': (conversation_doc or {}).get('id'), 'is_hidden': True, } + for field_name in CITATION_TRACKING_CONVERSATION_FIELDS: + if field_name in (conversation_doc or {}): + synchronized_values[field_name] = deepcopy( + conversation_doc.get(field_name) + ) for field_name, field_value in synchronized_values.items(): if source_conversation_doc.get(field_name) != field_value: source_conversation_doc[field_name] = field_value @@ -1952,6 +2027,7 @@ def _refresh_collaboration_conversation_message_summary(conversation_doc): raise ValueError('conversation_id is required') remaining_messages = list_collaboration_messages(conversation_id) + rebuild_conversation_used_documents(conversation_doc, remaining_messages) conversation_doc['message_count'] = len(remaining_messages) if remaining_messages: diff --git a/application/single_app/functions_simplechat_operations.py b/application/single_app/functions_simplechat_operations.py index 03da3a6e5..2aff415c3 100644 --- a/application/single_app/functions_simplechat_operations.py +++ b/application/single_app/functions_simplechat_operations.py @@ -35,6 +35,7 @@ log_workflow_creation, ) from functions_appinsights import log_event +from functions_citation_tracking import rebuild_conversation_used_documents from functions_authentication import ( get_current_user_info, get_graph_endpoint, @@ -93,6 +94,8 @@ PERSONAL_FORK_CONVERSATION_FIELDS = ( "context", "tags", + "used_documents_tracking_version", + "legacy_used_documents", "strict", "classification", "scope_locked", @@ -808,6 +811,11 @@ def fork_personal_conversation_for_user( normalized_user_id, destination_chat_type, ) + rebuild_conversation_used_documents( + fork_conversation, + fork_documents, + rebuild_legacy=True, + ) written_message_ids = [] created_blob_targets: List[Tuple[str, str]] = [] diff --git a/application/single_app/functions_workflow_runner.py b/application/single_app/functions_workflow_runner.py index 83ccfd1ad..2c36b99f0 100644 --- a/application/single_app/functions_workflow_runner.py +++ b/application/single_app/functions_workflow_runner.py @@ -39,6 +39,7 @@ ) from config import ( SECRET_KEY, + TABULAR_EXTENSIONS, VERSION, cognitive_services_scope, cosmos_conversations_container, @@ -54,6 +55,12 @@ build_conversation_context_system_message, serialize_conversation_context_snapshot, ) +from functions_citation_tracking import ( + build_cited_source_subsets, + initialize_conversation_used_document_tracking, + merge_cited_documents_into_conversation, + resolve_citation_location, +) from functions_activity_logging import log_conversation_creation, log_token_usage, log_workflow_run from functions_appinsights import log_event from functions_assistant_table_exports import ( @@ -4470,8 +4477,26 @@ def _mirror_assistant_message_to_personal_conversation( previous_thread_id, ), } + for field_name in ( + 'citation_tracking_version', + 'cited_hybrid_citations', + 'cited_web_search_citations', + ): + if field_name in source_assistant_doc: + if field_name == 'citation_tracking_version': + mirrored_assistant_doc[field_name] = source_assistant_doc.get(field_name) + else: + mirrored_assistant_doc[field_name] = list( + source_assistant_doc.get(field_name) or [] + ) cosmos_messages_container.upsert_item(mirrored_assistant_doc) + if 'citation_tracking_version' in mirrored_assistant_doc: + initialize_conversation_used_document_tracking(conversation_doc) + merge_cited_documents_into_conversation( + conversation_doc, + mirrored_assistant_doc.get('cited_hybrid_citations'), + ) conversation_doc['last_updated'] = timestamp conversation_doc['has_unread_assistant_response'] = True conversation_doc['last_unread_assistant_message_id'] = mirrored_message_id @@ -5833,6 +5858,12 @@ def _create_assistant_message(conversation, workflow, result, trigger_source, ru if generated_file_output.get('output_format') == 'csv': generated_tabular_outputs.append(generated_file_output) web_search_citations = list(result.get('web_search_citations') or []) + hybrid_citations = list(result.get('hybrid_citations') or []) + citation_tracking = build_cited_source_subsets( + result.get('reply', ''), + hybrid_citations=hybrid_citations, + web_search_citations=web_search_citations, + ) source_review_metadata = result.get('source_review') if isinstance(result.get('source_review'), dict) else {} url_access_metadata = result.get('url_access') if isinstance(result.get('url_access'), dict) else {} prepared_agent_citations = _persist_agent_citation_artifacts( @@ -5851,10 +5882,11 @@ def _create_assistant_message(conversation, workflow, result, trigger_source, ru 'content': result.get('reply', ''), 'timestamp': timestamp, 'model_deployment_name': result.get('model_deployment_name'), - 'augmented': bool(result.get('augmented') or result.get('hybrid_citations')), - 'hybrid_citations': list(result.get('hybrid_citations') or []), + 'augmented': bool(result.get('augmented') or hybrid_citations), + 'hybrid_citations': hybrid_citations, 'agent_citations': prepared_agent_citations, 'web_search_citations': web_search_citations, + **citation_tracking, 'agent_display_name': result.get('agent_display_name'), 'agent_name': result.get('agent_name'), 'workspace_type': workspace_type, @@ -5927,6 +5959,11 @@ def _create_assistant_message(conversation, workflow, result, trigger_source, ru conversation['has_unread_assistant_response'] = True conversation['last_unread_assistant_message_id'] = assistant_message_id conversation['last_unread_assistant_at'] = timestamp + initialize_conversation_used_document_tracking(conversation) + merge_cited_documents_into_conversation( + conversation, + citation_tracking['cited_hybrid_citations'], + ) cosmos_conversations_container.upsert_item(conversation) return assistant_doc @@ -6418,12 +6455,25 @@ def _build_workflow_search_citation(result): document_id = str(result.get('document_id') or '').strip() if not document_id: document_id = '_'.join(str(citation_id).split('_')[:-1]) if '_' in str(citation_id) else str(citation_id) + sheet_name = result.get('sheet_name') + page_number = result.get('page_number') or result.get('chunk_sequence') or 1 + file_name = result.get('file_name') or result.get('title') or 'Unknown document' + _, extension = os.path.splitext(str(file_name).strip().lower()) + location_label, location_value = resolve_citation_location( + page_number=page_number, + chunk_text=result.get('chunk_text'), + sheet_name=sheet_name, + is_tabular=extension.lstrip('.') in TABULAR_EXTENSIONS, + ) return { - 'file_name': result.get('file_name') or result.get('title') or 'Unknown document', + 'file_name': file_name, 'document_id': document_id, 'citation_id': citation_id, - 'page_number': result.get('page_number'), + 'page_number': page_number, + 'sheet_name': sheet_name, + 'location_label': location_label, + 'location_value': location_value, 'chunk_id': result.get('chunk_id'), 'chunk_sequence': result.get('chunk_sequence'), 'score': result.get('score'), @@ -6445,9 +6495,18 @@ def _format_workflow_search_results(results): file_name = str(result.get('file_name') or result.get('title') or 'Unknown document').strip() or 'Unknown document' page_number = result.get('page_number') or result.get('chunk_sequence') or 1 + sheet_name = result.get('sheet_name') + _, extension = os.path.splitext(file_name.lower()) + location_label, location_value = resolve_citation_location( + page_number=page_number, + chunk_text=chunk_text, + sheet_name=sheet_name, + is_tabular=extension.lstrip('.') in TABULAR_EXTENSIONS, + ) citation_id = result.get('id') or result.get('chunk_id') or f'workflow-search-{index}' result_lines.append( - f'[{index}] {file_name}, page {page_number}, citation #{citation_id}\n{chunk_text}' + f'[{index}] {chunk_text}\n' + f'(Source: {file_name}, {location_label}: {location_value}) [#{citation_id}]' ) citations.append(_build_workflow_search_citation(result)) @@ -6468,7 +6527,8 @@ def _build_workflow_search_prompt(task_prompt, search_context): prompt_sections = [ '[Workflow document search context]\n' 'Use the bounded native-engine evidence below as grounding for the workflow task. ' - 'When the evidence is insufficient or source coverage is partial, say what is missing instead of guessing.' + 'When the evidence is insufficient or source coverage is partial, say what is missing instead of guessing. ' + 'Preserve the exact (Source: ...) [#citation-id] reference for every excerpt used in the response.' ] if retrieved_content: prompt_sections.append( diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index d675d7292..bad4c20dc 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -179,6 +179,12 @@ inject_conversation_context_message, serialize_conversation_context_snapshot, ) +from functions_citation_tracking import ( + build_cited_source_subsets, + initialize_conversation_used_document_tracking, + merge_cited_documents_into_conversation, + resolve_citation_location, +) from functions_conversation_metadata import collect_conversation_metadata, update_conversation_with_metadata from functions_conversation_unread import mark_conversation_unread from functions_image_messages import build_image_message_documents, decode_image_content @@ -4298,7 +4304,7 @@ def build_tabular_fallback_system_message(tabular_filenames_str, execution_mode= def build_search_augmentation_system_prompt(retrieved_content): """Build the retrieval augmentation prompt without blocking later tool-backed results.""" - return f"""You are an AI assistant. Use the following retrieved document excerpts to answer the user's question. Cite sources using the format (Source: filename, Page: page number). + return f"""You are an AI assistant. Use the following retrieved document excerpts to answer the user's question. Cite sources using the format (Source: filename, Page: page number) [#citation-id]. Copy the exact bracketed citation ID shown after the supporting excerpt. Retrieved Excerpts: {retrieved_content} @@ -4308,7 +4314,7 @@ def build_search_augmentation_system_prompt(retrieved_content): Example User: What is the policy on double dipping? - Assistant: The policy prohibits entities from using federal funds received through one program to apply for additional funds through another program, commonly known as 'double dipping' (Source: PolicyDocument.pdf, Page: 12) + Assistant: The policy prohibits entities from using federal funds received through one program to apply for additional funds through another program, commonly known as 'double dipping' (Source: PolicyDocument.pdf, Page: 12) [#example-citation-id] """ @@ -13046,17 +13052,12 @@ def is_tabular_filename(filename): def get_citation_location(file_name, page_number=None, chunk_text=None, sheet_name=None): """Return a display label/value pair for a citation location.""" - if sheet_name: - return 'Sheet', str(sheet_name) - - normalized_chunk_text = (chunk_text or '').strip() - if is_tabular_filename(file_name) and ( - normalized_chunk_text.startswith('Tabular workbook:') - or normalized_chunk_text.startswith('Tabular data file:') - ): - return 'Location', 'Workbook Schema' - - return 'Page', str(page_number or 1) + return resolve_citation_location( + page_number=page_number, + chunk_text=chunk_text, + sheet_name=sheet_name, + is_tabular=is_tabular_filename(file_name), + ) def get_document_container_for_scope(document_scope): @@ -14220,6 +14221,9 @@ def normalize_terminal_chat_payload(payload): 'augmented': payload.get('augmented', False), 'hybrid_citations': payload.get('hybrid_citations', []), 'web_search_citations': payload.get('web_search_citations', []), + 'citation_tracking_version': payload.get('citation_tracking_version'), + 'cited_hybrid_citations': payload.get('cited_hybrid_citations', []), + 'cited_web_search_citations': payload.get('cited_web_search_citations', []), 'agent_citations': payload.get('agent_citations', []), 'agent_display_name': payload.get('agent_display_name'), 'agent_name': payload.get('agent_name'), @@ -15400,6 +15404,11 @@ def execute_document_action_chat_request( generated_analysis_artifacts=document_generated_analysis_artifacts, generated_tabular_outputs=document_generated_tabular_outputs, ) + document_action_citation_tracking = build_cited_source_subsets( + document_action_reply_content, + hybrid_citations=hybrid_citations_list, + web_search_citations=[], + ) document_action_capability_usage = _build_capability_usage_metadata( workspace_search_used=True, workspace_search_result_count=len(hybrid_citations_list or []), @@ -15419,6 +15428,7 @@ def execute_document_action_chat_request( 'augmented': False, 'hybrid_citations': hybrid_citations_list, 'web_search_citations': [], + **document_action_citation_tracking, 'hybridsearch_query': None, 'agent_citations': prepared_agent_citations, 'model_deployment_name': execution_result.get('model_deployment_name'), @@ -15521,6 +15531,7 @@ def execute_document_action_chat_request( conversation_item['last_updated'] = datetime.utcnow().isoformat() conversation_item['chat_type'] = data.get('chat_type') or conversation_item.get('chat_type') or 'new' + initialize_conversation_used_document_tracking(conversation_item) try: conversation_item = collect_conversation_metadata( @@ -15548,6 +15559,10 @@ def execute_document_action_chat_request( except Exception as exc: debug_print(f'[CHAT_DOCUMENT_ANALYSIS] Conversation metadata update failed: {exc}') + merge_cited_documents_into_conversation( + conversation_item, + document_action_citation_tracking['cited_hybrid_citations'], + ) cosmos_conversations_container.upsert_item(conversation_item) invalidate_conversation_cache_for_item(conversation_item, reason="document_action_chat_completed") debug_print( @@ -15580,6 +15595,7 @@ def execute_document_action_chat_request( 'augmented': False, 'hybrid_citations': hybrid_citations_list, 'web_search_citations': [], + **document_action_citation_tracking, 'agent_citations': prepared_agent_citations, 'reload_messages': False, 'kernel_fallback_notice': None, @@ -17610,6 +17626,9 @@ def result_requires_message_reload(result: Any) -> bool: "document_id": source_doc.get("document_id"), "citation_id": source_doc.get("citation_id"), # Seems like a useful identifier "page_number": source_doc.get("page_number"), + "sheet_name": source_doc.get("sheet_name"), + "location_label": source_doc.get("location_label"), + "location_value": source_doc.get("location_value"), "chunk_id": source_doc.get("chunk_id"), # Specific chunk identifier "chunk_sequence": source_doc.get("chunk_sequence"), # Order within document/group "score": source_doc.get("score"), # Relevance score from search @@ -19807,6 +19826,11 @@ def gpt_error(e): deep_research_query_count=_deep_research_query_count(deep_research_query_plan, deep_research_web_search_runs), ) agent_runtime_metadata = _build_foundry_runtime_metadata(selected_agent) if selected_agent else {} + citation_tracking = build_cited_source_subsets( + ai_message, + hybrid_citations=hybrid_citations_list, + web_search_citations=web_search_citations_list, + ) assistant_doc = make_json_serializable({ 'id': assistant_message_id, @@ -19817,6 +19841,7 @@ def gpt_error(e): 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, # <--- SIMPLIFIED: Directly use the list 'web_search_citations': web_search_citations_list, + **citation_tracking, 'hybridsearch_query': search_query if search_results else None, # Log query when any bounded document retrieval produced results 'agent_citations': prepared_agent_citations, 'model_deployment_name': actual_model_used, @@ -19928,6 +19953,7 @@ def gpt_error(e): # Update conversation's last_updated timestamp one last time conversation_item['last_updated'] = datetime.utcnow().isoformat() + initialize_conversation_used_document_tracking(conversation_item) # Collect comprehensive conversation metadata try: @@ -19971,6 +19997,10 @@ def gpt_error(e): debug_print(f"Error collecting conversation metadata: {e}") # Continue even if metadata collection fails + merge_cited_documents_into_conversation( + conversation_item, + citation_tracking['cited_hybrid_citations'], + ) # Add any other final updates to conversation_item if needed (like classifications if not done earlier) cosmos_conversations_container.upsert_item(conversation_item) invalidate_conversation_cache_for_item(conversation_item, reason="chat_completed") @@ -19999,6 +20029,7 @@ def gpt_error(e): 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, 'web_search_citations': web_search_citations_list, + **citation_tracking, 'source_review': compact_source_review_result_for_metadata(source_review_result), 'deep_research': deep_research_result, 'agent_citations': prepared_agent_citations, @@ -20134,6 +20165,9 @@ def normalize_legacy_chat_payload(payload): 'augmented': payload.get('augmented', False), 'hybrid_citations': payload.get('hybrid_citations', []), 'web_search_citations': payload.get('web_search_citations', []), + 'citation_tracking_version': payload.get('citation_tracking_version'), + 'cited_hybrid_citations': payload.get('cited_hybrid_citations', []), + 'cited_web_search_citations': payload.get('cited_web_search_citations', []), 'agent_citations': payload.get('agent_citations', []), 'agent_display_name': payload.get('agent_display_name'), 'agent_name': payload.get('agent_name'), @@ -20634,6 +20668,39 @@ def build_streaming_capability_usage(): deep_research_query_count=_deep_research_query_count(deep_research_query_plan, deep_research_web_search_runs), ) + def collect_stream_response_conversation_metadata(): + nonlocal conversation_item + source_continuity_refs = None + if ( + is_mixed_source_conversation_continuity_enabled(settings) + and mixed_source_manifest + ): + source_continuity_refs = _build_mixed_source_continuity_refs( + mixed_source_manifest, + mixed_source_evidence_envelopes, + effective_mixed_source_selection_mode, + ) + conversation_item = collect_conversation_metadata( + user_message=user_message, + conversation_id=conversation_id, + user_id=user_id, + active_group_id=effective_active_group_id, + active_group_ids=effective_active_group_ids, + document_scope=effective_document_scope, + selected_document_id=effective_selected_document_id, + model_deployment=final_model_used if use_agent_streaming else gpt_model, + hybrid_search_enabled=mixed_source_document_context_active, + image_gen_enabled=False, + selected_documents=combined_documents if combined_documents else None, + selected_agent=agent_name_used if use_agent_streaming else None, + selected_agent_details=selected_agent_metadata if use_agent_streaming else None, + search_results=search_results if search_results else None, + conversation_item=conversation_item, + active_public_workspace_id=effective_active_public_workspace_id, + active_public_workspace_ids=effective_active_public_workspace_ids, + source_continuity_refs=source_continuity_refs, + ) + # Initialize GPT client (simplified version) gpt_model = "" gpt_client = None @@ -21876,6 +21943,9 @@ def record_and_publish_streaming_thought(thought_payload): "document_id": document_id, "citation_id": citation_id, "page_number": page_number, + "sheet_name": sheet_name, + "location_label": location_label, + "location_value": location_value, "chunk_id": chunk_id, "chunk_sequence": chunk_sequence, "score": score, @@ -22934,6 +23004,7 @@ def finalize_cancelled_stream_response(): cancel_reason = stream_session.get_cancel_reason() if stream_session else 'user_requested' partial_content = accumulated_content.strip() message_persisted = False + partial_citation_tracking = {} cancel_metadata = { 'incomplete': True, 'canceled': True, @@ -22960,6 +23031,11 @@ def finalize_cancelled_stream_response(): if partial_content: assistant_timestamp = datetime.utcnow().isoformat() + partial_citation_tracking = build_cited_source_subsets( + partial_content, + hybrid_citations=hybrid_citations_list, + web_search_citations=web_search_citations_list, + ) prepared_agent_citations = persist_agent_citation_artifacts( conversation_id=conversation_id, assistant_message_id=assistant_message_id, @@ -22980,6 +23056,7 @@ def finalize_cancelled_stream_response(): 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, 'web_search_citations': web_search_citations_list, + **partial_citation_tracking, 'hybridsearch_query': search_query if hybrid_search_enabled and search_results else None, 'agent_citations': prepared_agent_citations, 'model_deployment_name': final_model_used if use_agent_streaming else gpt_model, @@ -23016,6 +23093,24 @@ def finalize_cancelled_stream_response(): }) cosmos_messages_container.upsert_item(assistant_doc) conversation_item['last_updated'] = datetime.utcnow().isoformat() + initialize_conversation_used_document_tracking(conversation_item) + try: + collect_stream_response_conversation_metadata() + except Exception as metadata_error: + log_event( + '[STREAMING] Failed to collect canceled response metadata', + extra={ + 'conversation_id': conversation_id, + 'message_id': assistant_message_id, + 'error_type': type(metadata_error).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + merge_cited_documents_into_conversation( + conversation_item, + partial_citation_tracking['cited_hybrid_citations'], + ) cosmos_conversations_container.upsert_item(conversation_item) invalidate_conversation_cache_for_item(conversation_item, reason="chat_stream_stopped") message_persisted = True @@ -23043,6 +23138,7 @@ def finalize_cancelled_stream_response(): 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, 'web_search_citations': web_search_citations_list, + **partial_citation_tracking, 'agent_citations': agent_citations_list, 'model_deployment_name': final_model_used if use_agent_streaming else gpt_model, 'model_icon': gpt_model_icon, @@ -23666,6 +23762,11 @@ def finalize_cancelled_agent_stream_response(): generated_tabular_outputs=generated_tabular_outputs_list, ) agent_runtime_metadata = _build_foundry_runtime_metadata(selected_agent) if use_agent_streaming else {} + stream_citation_tracking = build_cited_source_subsets( + accumulated_content, + hybrid_citations=hybrid_citations_list, + web_search_citations=web_search_citations_list, + ) assistant_doc = make_json_serializable({ 'id': assistant_message_id, @@ -23676,6 +23777,7 @@ def finalize_cancelled_agent_stream_response(): 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, 'web_search_citations': web_search_citations_list, + **stream_citation_tracking, 'hybridsearch_query': search_query if search_results else None, 'agent_citations': prepared_agent_citations, 'model_deployment_name': final_model_used if use_agent_streaming else gpt_model, @@ -23778,6 +23880,7 @@ def finalize_cancelled_agent_stream_response(): # Update conversation conversation_item['last_updated'] = datetime.utcnow().isoformat() + initialize_conversation_used_document_tracking(conversation_item) try: user_message_doc = cosmos_messages_container.read_item( @@ -23795,39 +23898,14 @@ def finalize_cancelled_agent_stream_response(): debug_print(f"Warning: Could not update streaming user message metadata: {e}") try: - source_continuity_refs = None - if ( - is_mixed_source_conversation_continuity_enabled(settings) - and mixed_source_manifest - ): - source_continuity_refs = _build_mixed_source_continuity_refs( - mixed_source_manifest, - mixed_source_evidence_envelopes, - effective_mixed_source_selection_mode, - ) - conversation_item = collect_conversation_metadata( - user_message=user_message, - conversation_id=conversation_id, - user_id=user_id, - active_group_id=effective_active_group_id, - active_group_ids=effective_active_group_ids, - document_scope=effective_document_scope, - selected_document_id=effective_selected_document_id, - model_deployment=final_model_used if use_agent_streaming else gpt_model, - hybrid_search_enabled=mixed_source_document_context_active, - image_gen_enabled=False, - selected_documents=combined_documents if combined_documents else None, - selected_agent=agent_name_used if use_agent_streaming else None, - selected_agent_details=selected_agent_metadata if use_agent_streaming else None, - search_results=search_results if search_results else None, - conversation_item=conversation_item, - active_public_workspace_id=effective_active_public_workspace_id, - active_public_workspace_ids=effective_active_public_workspace_ids, - source_continuity_refs=source_continuity_refs, - ) + collect_stream_response_conversation_metadata() except Exception as e: debug_print(f"Error collecting conversation metadata: {e}") + merge_cited_documents_into_conversation( + conversation_item, + stream_citation_tracking['cited_hybrid_citations'], + ) if is_personal_chat_conversation(conversation_item): conversation_item = mark_conversation_unread( conversation_item, @@ -23871,6 +23949,7 @@ def finalize_cancelled_agent_stream_response(): 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, 'web_search_citations': web_search_citations_list, + **stream_citation_tracking, 'source_review': compact_source_review_result_for_metadata(source_review_result), 'deep_research': deep_research_result, 'agent_citations': prepared_agent_citations, @@ -23942,10 +24021,18 @@ def finalize_cancelled_agent_stream_response(): debug_print(f"Error during streaming: {error_msg}") # Save partial response if we have content + interrupted_message_persisted = False + interrupted_citation_tracking = {} + interrupted_agent_citations = [] if accumulated_content: current_assistant_thread_id = str(uuid.uuid4()) assistant_timestamp = datetime.utcnow().isoformat() - prepared_agent_citations = persist_agent_citation_artifacts( + interrupted_citation_tracking = build_cited_source_subsets( + accumulated_content, + hybrid_citations=hybrid_citations_list, + web_search_citations=web_search_citations_list, + ) + interrupted_agent_citations = persist_agent_citation_artifacts( conversation_id=conversation_id, assistant_message_id=assistant_message_id, agent_citations=agent_citations_list, @@ -23966,8 +24053,9 @@ def finalize_cancelled_agent_stream_response(): 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, 'web_search_citations': web_search_citations_list, + **interrupted_citation_tracking, 'hybridsearch_query': search_query if hybrid_search_enabled and search_results else None, - 'agent_citations': prepared_agent_citations, + 'agent_citations': interrupted_agent_citations, 'model_deployment_name': final_model_used if use_agent_streaming else gpt_model, 'agent_display_name': agent_display_name_used if use_agent_streaming else None, 'agent_name': agent_name_used if use_agent_streaming else None, @@ -23991,12 +24079,68 @@ def finalize_cancelled_agent_stream_response(): }) try: cosmos_messages_container.upsert_item(assistant_doc) - except Exception as ex: - pass + interrupted_message_persisted = True + conversation_item['last_updated'] = assistant_timestamp + initialize_conversation_used_document_tracking( + conversation_item + ) + try: + collect_stream_response_conversation_metadata() + except Exception as metadata_error: + log_event( + '[STREAMING] Failed to collect interrupted response metadata', + extra={ + 'conversation_id': conversation_id, + 'message_id': assistant_message_id, + 'error_type': type( + metadata_error + ).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + merge_cited_documents_into_conversation( + conversation_item, + interrupted_citation_tracking[ + 'cited_hybrid_citations' + ], + ) + cosmos_conversations_container.upsert_item( + conversation_item + ) + invalidate_conversation_cache_for_item( + conversation_item, + reason="chat_stream_interrupted", + ) + except Exception as persistence_error: + log_event( + '[STREAMING] Failed to persist interrupted response metadata', + extra={ + 'conversation_id': conversation_id, + 'message_id': assistant_message_id, + 'error_type': type( + persistence_error + ).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) yield build_stream_error_event( CLIENT_SAFE_STREAM_ERROR_MESSAGE, partial_content=accumulated_content, + conversation_id=conversation_id, + user_message_id=user_message_id, + message_id=( + assistant_message_id + if interrupted_message_persisted + else None + ), + message_persisted=interrupted_message_persisted, + hybrid_citations=hybrid_citations_list, + web_search_citations=web_search_citations_list, + agent_citations=interrupted_agent_citations, + **interrupted_citation_tracking, ) except Exception as e: diff --git a/application/single_app/route_backend_collaboration.py b/application/single_app/route_backend_collaboration.py index bf1f20af3..6b45af1bc 100644 --- a/application/single_app/route_backend_collaboration.py +++ b/application/single_app/route_backend_collaboration.py @@ -1507,7 +1507,13 @@ def transform_event_block(event_block): except json.JSONDecodeError: return normalized_event_block + '\n\n' - if stream_payload.get('error'): + if ( + stream_payload.get('error') + and not ( + stream_payload.get('message_persisted') + and stream_payload.get('message_id') + ) + ): return _serialize_stream_error( stream_payload.get('error'), partial_content=stream_payload.get('partial_content'), @@ -1515,6 +1521,8 @@ def transform_event_block(event_block): message_persisted=True, conversation_id=conversation_id, ) + if stream_payload.get('error'): + stream_payload['done'] = True if stream_payload.get('type') == USER_MESSAGE_PERSISTED_EVENT_TYPE: return None @@ -1627,6 +1635,7 @@ def transform_event_block(event_block): **stream_payload, 'conversation_id': conversation_id, 'conversation_title': serialized_final_conversation.get('title'), + 'conversation_kind': serialized_final_conversation.get('conversation_kind'), 'chat_type': serialized_final_conversation.get('chat_type'), 'classification': serialized_final_conversation.get('classification', []), 'context': serialized_final_conversation.get('context', []), @@ -1638,12 +1647,15 @@ def transform_event_block(event_block): 'augmented': serialized_assistant_message.get('augmented', False), 'hybrid_citations': serialized_assistant_message.get('hybrid_citations', []), 'web_search_citations': serialized_assistant_message.get('web_search_citations', []), + 'citation_tracking_version': serialized_assistant_message.get('citation_tracking_version'), + 'cited_hybrid_citations': serialized_assistant_message.get('cited_hybrid_citations', []), + 'cited_web_search_citations': serialized_assistant_message.get('cited_web_search_citations', []), 'agent_citations': serialized_assistant_message.get('agent_citations', []), 'agent_display_name': serialized_assistant_message.get('agent_display_name'), 'agent_name': serialized_assistant_message.get('agent_name'), 'full_content': serialized_assistant_message.get('content') if serialized_assistant_message.get('role') != 'image' else stream_payload.get('full_content', ''), 'image_url': serialized_assistant_message.get('content') if serialized_assistant_message.get('role') == 'image' else stream_payload.get('image_url'), - 'reload_messages': False, + 'reload_messages': bool(stream_payload.get('error')), } return f'data: {json.dumps(make_json_serializable(transformed_payload))}\n\n' diff --git a/application/single_app/route_backend_conversation_export.py b/application/single_app/route_backend_conversation_export.py index 68b2b855d..56af5c671 100644 --- a/application/single_app/route_backend_conversation_export.py +++ b/application/single_app/route_backend_conversation_export.py @@ -30,6 +30,10 @@ is_collaboration_conversation, list_collaboration_messages, ) +from functions_citation_tracking import ( + get_message_reference_citation_buckets, + get_message_source_citation_buckets, +) from functions_conversation_metadata import update_conversation_with_metadata from functions_debug import debug_print from functions_group import get_group_model_endpoints, get_user_groups @@ -565,8 +569,9 @@ def _sanitize_message( ) -> Dict[str, Any]: role = message.get('role', '') content = message.get('content', '') - raw_citation_buckets = _collect_raw_citation_buckets(message) - normalized_citations = _normalize_citations(raw_citation_buckets) + reference_citation_buckets = _collect_raw_citation_buckets(message) + source_citation_buckets = _collect_source_citation_buckets(message) + normalized_citations = _normalize_citations(reference_citation_buckets) citation_counts = _build_citation_counts(normalized_citations) details = _curate_message_details(message, citation_counts, len(thoughts)) @@ -585,10 +590,21 @@ def _sanitize_message( 'citations': normalized_citations, 'citation_counts': citation_counts, 'thoughts': thoughts, - 'legacy_citations': raw_citation_buckets['legacy'], - 'hybrid_citations': raw_citation_buckets['hybrid'], - 'web_search_citations': raw_citation_buckets['web'], - 'agent_citations': raw_citation_buckets['agent'] + 'citation_tracking_version': message.get('citation_tracking_version'), + 'cited_hybrid_citations': ( + list(message.get('cited_hybrid_citations')) + if isinstance(message.get('cited_hybrid_citations'), list) + else [] + ), + 'cited_web_search_citations': ( + list(message.get('cited_web_search_citations')) + if isinstance(message.get('cited_web_search_citations'), list) + else [] + ), + 'legacy_citations': source_citation_buckets['legacy'], + 'hybrid_citations': source_citation_buckets['hybrid'], + 'web_search_citations': source_citation_buckets['web'], + 'agent_citations': source_citation_buckets['agent'] } @@ -604,17 +620,13 @@ def _sanitize_thought(thought: Dict[str, Any]) -> Dict[str, Any]: def _collect_raw_citation_buckets(message: Dict[str, Any]) -> Dict[str, List[Any]]: - def ensure_list(value: Any) -> List[Any]: - if not value: - return [] - return value if isinstance(value, list) else [value] + """Return citation records that are valid references for this message.""" + return get_message_reference_citation_buckets(message) - return { - 'legacy': ensure_list(message.get('citations')), - 'hybrid': ensure_list(message.get('hybrid_citations')), - 'web': ensure_list(message.get('web_search_citations')), - 'agent': ensure_list(message.get('agent_citations')) - } + +def _collect_source_citation_buckets(message: Dict[str, Any]) -> Dict[str, List[Any]]: + """Return complete source and execution records for JSON audit output.""" + return get_message_source_citation_buckets(message) def _normalize_citations(raw_citation_buckets: Dict[str, List[Any]]) -> List[Dict[str, Any]]: @@ -627,7 +639,11 @@ def _normalize_citations(raw_citation_buckets: Dict[str, List[Any]]) -> List[Dic 'label': _build_document_citation_label(citation), 'file_name': citation.get('file_name'), 'title': citation.get('title') or citation.get('file_name'), + 'document_id': citation.get('document_id'), 'page_number': citation.get('page_number'), + 'sheet_name': citation.get('sheet_name'), + 'location_label': citation.get('location_label'), + 'location_value': citation.get('location_value'), 'citation_id': citation.get('citation_id'), 'chunk_id': citation.get('chunk_id'), 'metadata_type': citation.get('metadata_type'), @@ -668,6 +684,9 @@ def _normalize_citations(raw_citation_buckets: Dict[str, List[Any]]) -> List[Dic 'tool_name': citation.get('tool_name'), 'function_name': citation.get('function_name'), 'plugin_name': citation.get('plugin_name'), + 'function_arguments': citation.get('function_arguments'), + 'function_result': citation.get('function_result'), + 'error_message': citation.get('error_message'), 'success': citation.get('success'), 'timestamp': citation.get('timestamp') }) @@ -701,12 +720,20 @@ def _normalize_citations(raw_citation_buckets: Dict[str, List[Any]]) -> List[Dic def _build_document_citation_label(citation: Dict[str, Any]) -> str: file_name = citation.get('file_name') or citation.get('title') or 'Document source' metadata_type = citation.get('metadata_type') - page_number = citation.get('page_number') + location_label = ( + citation.get('location_label') + or ('Sheet' if citation.get('sheet_name') else 'Page') + ) + location_value = ( + citation.get('location_value') + or citation.get('sheet_name') + or citation.get('page_number') + ) if metadata_type: return f"{file_name} — {metadata_type.replace('_', ' ').title()}" - if page_number not in (None, ''): - return f"{file_name} — Page {page_number}" + if location_value not in (None, ''): + return f"{file_name} — {location_label}: {location_value}" return file_name @@ -1624,7 +1651,7 @@ def _conversation_to_markdown(entry: Dict[str, Any]) -> str: def _append_citations_markdown(lines: List[str], message: Dict[str, Any]): document_citations = [citation for citation in message.get('citations', []) if citation.get('citation_type') == 'document'] web_citations = [citation for citation in message.get('citations', []) if citation.get('citation_type') == 'web'] - agent_citations = message.get('agent_citations', []) or [] + agent_citations = [citation for citation in message.get('citations', []) if citation.get('citation_type') == 'agent_tool'] legacy_citations = [citation for citation in message.get('citations', []) if citation.get('citation_type') == 'legacy'] if not any([document_citations, web_citations, agent_citations, legacy_citations]): @@ -1632,13 +1659,16 @@ def _append_citations_markdown(lines: List[str], message: Dict[str, Any]): return if document_citations: - lines.append('#### Document Sources') + lines.append('#### Document References') lines.append('') for index, citation in enumerate(document_citations, start=1): lines.append(f"{index}. **{citation.get('label', 'Document source')}**") detail_mapping = _remove_empty_values({ 'citation_id': citation.get('citation_id'), 'page_number': citation.get('page_number'), + 'sheet_name': citation.get('sheet_name'), + 'location_label': citation.get('location_label'), + 'location_value': citation.get('location_value'), 'classification': citation.get('classification'), 'score': citation.get('score'), 'metadata_type': citation.get('metadata_type') @@ -1650,7 +1680,7 @@ def _append_citations_markdown(lines: List[str], message: Dict[str, Any]): lines.append('') if web_citations: - lines.append('#### Web Sources') + lines.append('#### Web References') lines.append('') for index, citation in enumerate(web_citations, start=1): title = citation.get('title') or citation.get('label') or 'Web source' @@ -5565,7 +5595,7 @@ def _append_html_citations(parts: List[str], message: Dict[str, Any]): legacy_citations = [c for c in citations if c.get('citation_type') == 'legacy'] if doc_citations: - parts.append('

Document Sources

') + parts.append('

Document References

') parts.append('
    ') for citation in doc_citations: parts.append( @@ -5574,6 +5604,9 @@ def _append_html_citations(parts: List[str], message: Dict[str, Any]): detail_items = _remove_empty_values({ 'citation_id': citation.get('citation_id'), 'page_number': citation.get('page_number'), + 'sheet_name': citation.get('sheet_name'), + 'location_label': citation.get('location_label'), + 'location_value': citation.get('location_value'), 'classification': citation.get('classification'), 'score': citation.get('score'), 'metadata_type': citation.get('metadata_type') @@ -5591,7 +5624,7 @@ def _append_html_citations(parts: List[str], message: Dict[str, Any]): parts.append('
') if web_citations: - parts.append('

Web Sources

') + parts.append('

Web References

') parts.append('
    ') for citation in web_citations: title = _escape_html( diff --git a/application/single_app/route_backend_conversations.py b/application/single_app/route_backend_conversations.py index b286afc04..d00b78534 100644 --- a/application/single_app/route_backend_conversations.py +++ b/application/single_app/route_backend_conversations.py @@ -31,6 +31,7 @@ tag_conversation_feed_source, ) from functions_conversation_metadata import get_conversation_metadata, update_conversation_with_metadata +from functions_citation_tracking import rebuild_conversation_used_documents from functions_conversation_unread import clear_conversation_unread, normalize_conversation_unread_state from functions_conversation_cache import ( build_conversation_cache_key, @@ -822,6 +823,63 @@ def _authorize_personal_conversation_read(user_id, conversation_id): return conversation_item +def _rebuild_authorized_personal_conversation_used_documents( + user_id, + conversation_id, +): + """Persist exact used documents after an authorized message mutation.""" + try: + conversation_item = _authorize_personal_conversation_read( + user_id, + conversation_id, + ) + messages = list(cosmos_messages_container.query_items( + query=( + "SELECT * FROM c " + "WHERE c.conversation_id = @conversation_id" + ), + parameters=[ + { + "name": "@conversation_id", + "value": conversation_id, + }, + ], + partition_key=conversation_id, + )) + rebuild_conversation_used_documents(conversation_item, messages) + conversation_item.pop('used_documents_rebuild_required', None) + conversation_item['last_updated'] = datetime.utcnow().isoformat() + cosmos_conversations_container.upsert_item(conversation_item) + return conversation_item + except Exception as rebuild_error: + if 'conversation_item' in locals() and isinstance(conversation_item, dict): + conversation_item['used_documents_rebuild_required'] = True + try: + cosmos_conversations_container.upsert_item(conversation_item) + except Exception as marker_error: + log_event( + "[CONVERSATION_METADATA] Failed to mark used documents for rebuild", + extra={ + "conversation_id": conversation_id, + "user_id": user_id, + "error_type": type(marker_error).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + log_event( + "[CONVERSATION_METADATA] Failed to rebuild used documents after message mutation", + extra={ + "conversation_id": conversation_id, + "user_id": user_id, + "error_type": type(rebuild_error).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + return None + + def _invalidate_conversation_cache_after_message_mutation(conversation_id, user_id, reason): """Invalidate conversation caches after message-level changes without failing the caller.""" try: @@ -1800,6 +1858,16 @@ def get_conversation_metadata_api(conversation_id): # Ensure that the conversation belongs to the current user if conversation_item.get('user_id') != user_id: return jsonify({'error': 'Forbidden'}), 403 + + if conversation_item.get('used_documents_rebuild_required') is True: + rebuilt_conversation = ( + _rebuild_authorized_personal_conversation_used_documents( + user_id, + conversation_id, + ) + ) + if rebuilt_conversation: + conversation_item = rebuilt_conversation _, updated = normalize_chat_type(conversation_item) if updated: @@ -1828,6 +1896,9 @@ def get_conversation_metadata_api(conversation_id): "classification": conversation_item.get('classification', []), "context": conversation_item.get('context', []), "tags": conversation_item.get('tags', []), + "used_documents_tracking_version": conversation_item.get('used_documents_tracking_version'), + "legacy_used_documents": conversation_item.get('legacy_used_documents', []), + "used_documents": conversation_item.get('used_documents', []), "strict": conversation_item.get('strict', False), "is_pinned": conversation_item.get('is_pinned', False), "is_hidden": conversation_item.get('is_hidden', False), @@ -2609,6 +2680,10 @@ def delete_message(message_id): deleted_message_ids.append(msg_id) + _rebuild_authorized_personal_conversation_used_documents( + user_id, + conversation_id, + ) _invalidate_conversation_cache_after_message_mutation( conversation_id, user_id, @@ -2787,6 +2862,10 @@ def retry_message(message_id): } cosmos_messages_container.upsert_item(new_user_message) + _rebuild_authorized_personal_conversation_used_documents( + user_id, + conversation_id, + ) _invalidate_conversation_cache_after_message_mutation( conversation_id, user_id, @@ -3006,6 +3085,10 @@ def edit_message(message_id): } cosmos_messages_container.upsert_item(new_user_message) + _rebuild_authorized_personal_conversation_used_documents( + user_id, + conversation_id, + ) _invalidate_conversation_cache_after_message_mutation( conversation_id, user_id, @@ -3170,6 +3253,10 @@ def switch_attempt(message_id): msg['metadata']['thread_info']['active_thread'] = (msg_attempt == target_attempt) cosmos_messages_container.upsert_item(msg) + _rebuild_authorized_personal_conversation_used_documents( + user_id, + conversation_id, + ) _invalidate_conversation_cache_after_message_mutation( conversation_id, user_id, diff --git a/application/single_app/static/js/chat/chat-citations.js b/application/single_app/static/js/chat/chat-citations.js index c51927233..edd6ac293 100644 --- a/application/single_app/static/js/chat/chat-citations.js +++ b/application/single_app/static/js/chat/chat-citations.js @@ -42,7 +42,7 @@ export function parseDocIdAndPage(citationId) { export function parseCitations(message) { // ... (keep existing implementation) - const citationRegex = /\(Source:\s*([^,]+),\s*(Page(?:s)?|Sheet(?:s)?|Location):\s*([^)]+)\)\s*((?:\[#.*?\]\s*)+)/gi; + const citationRegex = /\(Source:\s*((?:(?!\(Source:).)+?),\s*(Page(?:s)?|Sheet(?:s)?|Location):\s*((?:(?!\(Source:).)+?)\)\s*((?:\[#.*?\]\s*)+)/gi; let result = message.replace(citationRegex, (whole, filename, locationLabel, locations, bracketSection) => { const trimmedFilename = filename.trim(); @@ -80,7 +80,12 @@ export function parseCitations(message) { } const normalizedLocationLabel = locationLabel.toLowerCase(); - const locationTokens = locations.split(/,/).map(tok => tok.trim()); + const locationTokens = ( + normalizedLocationLabel === 'sheet' + || normalizedLocationLabel === 'location' + ) + ? [locations.trim()] + : locations.split(/,/).map(tok => tok.trim()); const linkedTokens = locationTokens.map((token, index) => { if (!normalizedLocationLabel.startsWith('page')) { const ref = orderedRefs[index] || orderedRefs[0]; diff --git a/application/single_app/static/js/chat/chat-collaboration.js b/application/single_app/static/js/chat/chat-collaboration.js index 1dff3c9f1..d2ae11ac5 100644 --- a/application/single_app/static/js/chat/chat-collaboration.js +++ b/application/single_app/static/js/chat/chat-collaboration.js @@ -1074,6 +1074,24 @@ function handleConversationEvent(eventEnvelope = {}) { const decoratedMessage = decorateReplyMessage(payload.message); cacheCollaborationMessage(payload.message); + if (String(payload.message.role || '').trim().toLowerCase() === 'assistant') { + const conversationId = ( + eventEnvelope.conversation_id + || payload.message.conversation_id + || payload.conversation?.id + || '' + ); + window.dispatchEvent(new CustomEvent( + 'chat:conversation-documents-refresh', + { + detail: { + conversationId, + autoOpen: false, + conversationKind: 'collaborative', + }, + } + )); + } if (reconcilePendingCollaborativeUserMessage(payload.message)) { const messageKind = String( payload.message.message_kind || payload.message.metadata?.message_kind || '' @@ -1112,6 +1130,16 @@ function handleConversationEvent(eventEnvelope = {}) { if (eventEnvelope.event_type === 'collaboration.message.deleted' && payload.message_id) { removeCollaborationMessage(payload.message_id); + window.dispatchEvent(new CustomEvent( + 'chat:conversation-documents-refresh', + { + detail: { + conversationId: eventEnvelope.conversation_id || '', + autoOpen: false, + conversationKind: 'collaborative', + }, + } + )); if (payload.deleted_by_user_id && payload.deleted_by_user_id !== getCurrentUserId()) { showToast('A shared message was deleted.', 'info'); } @@ -1937,6 +1965,7 @@ window.chatCollaboration = { deactivateConversation, fetchCollaborationConversationList, fetchConversationMetadata, + loadConversationMessages, getPendingMessageContext, handleComposerBlur, handleComposerInput, diff --git a/application/single_app/static/js/chat/chat-conversation-contents.js b/application/single_app/static/js/chat/chat-conversation-contents.js index 97d1a9016..3034d119e 100644 --- a/application/single_app/static/js/chat/chat-conversation-contents.js +++ b/application/single_app/static/js/chat/chat-conversation-contents.js @@ -3,7 +3,10 @@ import { extractPageNumbers, fetchConversationMetadata, - getConversationDocumentTags, + getConversationExactUsedDocuments, + getConversationLegacyUsedDocuments, + getConversationUsedDocuments, + hasConversationUsedDocumentTracking, } from "./chat-conversation-details.js"; import { isColorLight } from "./chat-utils.js"; @@ -47,6 +50,9 @@ let restoreFocusAfterClose = true; let metadataRequestToken = 0; let documentLoadState = "idle"; let activeDocumentConversationId = ""; +let documentUsageTrackingAvailable = false; +let exactDocumentUsageAvailable = false; +let legacyDocumentFallbackAvailable = false; const autoOpenedDocumentConversationIds = new Set(); export function normalizeConversationContentsLabel(value, fallbackLabel = "User message") { @@ -178,9 +184,15 @@ function syncDrawerModeControls() { drawerTitle.textContent = contentsActive ? "Conversation contents" : "Used documents"; } if (drawerSubtitle) { + let documentsSubtitle = "Documents associated with this historical conversation"; + if (documentUsageTrackingAvailable && exactDocumentUsageAvailable) { + documentsSubtitle = legacyDocumentFallbackAvailable + ? "Cited documents and historical associations" + : "Documents cited in active responses"; + } drawerSubtitle.textContent = contentsActive ? "Jump to a user message" - : "Documents cited in this conversation"; + : documentsSubtitle; } contentsPanel?.classList.toggle("d-none", !contentsActive); @@ -428,9 +440,62 @@ function createDocumentMetaLine(iconName, text) { return line; } +function getDocumentCitationLocationLines(doc) { + const groupedLocations = new Map(); + const appendLocation = (label, value) => { + const normalizedLabel = String(label || "Location").trim() || "Location"; + const normalizedValue = String(value ?? "").trim(); + if (!normalizedValue) { + return; + } + const values = groupedLocations.get(normalizedLabel) || []; + if (!values.includes(normalizedValue)) { + values.push(normalizedValue); + groupedLocations.set(normalizedLabel, values); + } + }; + + const citationLocations = Array.isArray(doc?.citation_locations) + ? doc.citation_locations + : []; + citationLocations.forEach(location => { + if (!location || typeof location !== "object") { + return; + } + const label = location.location_label + || (location.sheet_name ? "Sheet" : "Page"); + const value = location.location_value + || location.sheet_name + || location.page_number; + appendLocation(label, value); + }); + + if (groupedLocations.size === 0) { + (Array.isArray(doc?.sheet_names) ? doc.sheet_names : []) + .forEach(value => appendLocation("Sheet", value)); + (Array.isArray(doc?.page_numbers) ? doc.page_numbers : []) + .forEach(value => appendLocation("Page", value)); + } + + if (groupedLocations.size === 0) { + const chunkIds = Array.isArray(doc?.chunk_ids) ? doc.chunk_ids : []; + extractPageNumbers(chunkIds).forEach(value => appendLocation("Page", value)); + } + + return Array.from(groupedLocations.entries()).map(([label, values]) => { + const normalizedLabel = label.toLowerCase(); + const displayLabel = normalizedLabel === "page" + ? "Pages" + : normalizedLabel === "sheet" + ? "Sheets" + : normalizedLabel === "location" + ? "Locations" + : label; + return `${displayLabel}: ${values.join(", ")}`; + }); +} + function createDocumentEntry(doc) { - const chunkIds = Array.isArray(doc?.chunk_ids) ? doc.chunk_ids : []; - const chunkPages = extractPageNumbers(chunkIds); const documentId = String(doc?.document_id || "Unknown Document"); const documentTitle = String(doc?.title || doc?.file_name || documentId); const documentFileName = String(doc?.file_name || ""); @@ -460,9 +525,9 @@ function createDocumentEntry(doc) { ) { entry.appendChild(createDocumentMetaLine("file-earmark", documentFileName)); } - if (chunkPages.length > 0) { - entry.appendChild(createDocumentMetaLine("file-earmark-text", `Pages: ${chunkPages.join(", ")}`)); - } + getDocumentCitationLocationLines(doc).forEach(locationLine => { + entry.appendChild(createDocumentMetaLine("file-earmark-text", locationLine)); + }); entry.appendChild(createDocumentMetaLine( getScopeIcon(scopeType), `${scopeType} scope: ${scopeName}` @@ -503,6 +568,7 @@ function renderConversationDocuments(documents) { async function refreshConversationDocuments(options = {}) { const conversationId = String(options.conversationId || getCurrentConversationId()).trim(); + const conversationKind = String(options.conversationKind || "").trim(); const requestToken = ++metadataRequestToken; const autoOpen = Boolean(options.autoOpen); const conversationChanged = conversationId !== activeDocumentConversationId; @@ -510,6 +576,9 @@ async function refreshConversationDocuments(options = {}) { activeDocumentConversationId = conversationId; if (!conversationId) { documentLoadState = "idle"; + documentUsageTrackingAvailable = false; + exactDocumentUsageAvailable = false; + legacyDocumentFallbackAvailable = false; renderConversationDocuments([]); return; } @@ -527,18 +596,26 @@ async function refreshConversationDocuments(options = {}) { } try { - const metadata = await fetchConversationMetadata(conversationId); + const metadata = await fetchConversationMetadata( + conversationId, + { conversationKind } + ); if (requestToken !== metadataRequestToken || conversationId !== activeDocumentConversationId) { return; } documentLoadState = "ready"; - const documents = getConversationDocumentTags(metadata); + documentUsageTrackingAvailable = hasConversationUsedDocumentTracking(metadata); + const exactDocuments = getConversationExactUsedDocuments(metadata); + const legacyDocuments = getConversationLegacyUsedDocuments(metadata); + exactDocumentUsageAvailable = exactDocuments.length > 0; + legacyDocumentFallbackAvailable = legacyDocuments.length > 0; + const documents = getConversationUsedDocuments(metadata); renderConversationDocuments(documents); if ( autoOpen - && documents.length > 0 + && exactDocuments.length > 0 && conversationId === getCurrentConversationId() && !autoOpenedDocumentConversationIds.has(conversationId) ) { @@ -551,6 +628,9 @@ async function refreshConversationDocuments(options = {}) { } documentLoadState = "error"; + documentUsageTrackingAvailable = false; + exactDocumentUsageAvailable = false; + legacyDocumentFallbackAvailable = false; documentEntries = []; documentsList?.replaceChildren(); documentsEmptyState?.classList.add("d-none"); @@ -658,6 +738,7 @@ function initializeConversationContents() { void refreshConversationDocuments({ conversationId: event.detail?.conversationId || "", autoOpen: false, + conversationKind: event.detail?.conversationKind || "", }); }); window.addEventListener("chat:conversation-documents-refresh", event => { @@ -668,6 +749,7 @@ function initializeConversationContents() { void refreshConversationDocuments({ conversationId, autoOpen: Boolean(event.detail?.autoOpen), + conversationKind: event.detail?.conversationKind || "", }); }); diff --git a/application/single_app/static/js/chat/chat-conversation-details.js b/application/single_app/static/js/chat/chat-conversation-details.js index da3f195b3..3101e0791 100644 --- a/application/single_app/static/js/chat/chat-conversation-details.js +++ b/application/single_app/static/js/chat/chat-conversation-details.js @@ -41,14 +41,17 @@ function getConversationMetadataDomItem(conversationId) { || document.querySelector(`.sidebar-conversation-item[data-conversation-id="${escapedConversationId}"]`); } -export async function fetchConversationMetadata(conversationId) { +export async function fetchConversationMetadata(conversationId, options = {}) { const normalizedConversationId = String(conversationId || '').trim(); if (!normalizedConversationId) { throw new Error('Conversation ID is required'); } const conversationItem = getConversationMetadataDomItem(normalizedConversationId); - const isCollaborativeConversation = conversationItem?.dataset?.conversationKind === 'collaborative'; + const isCollaborativeConversation = ( + options.conversationKind === 'collaborative' + || conversationItem?.dataset?.conversationKind === 'collaborative' + ); if (isCollaborativeConversation && window.chatCollaboration?.fetchConversationMetadata) { return window.chatCollaboration.fetchConversationMetadata(normalizedConversationId); @@ -68,6 +71,57 @@ export function getConversationDocumentTags(metadata = {}) { return tags.filter(tag => tag?.category === 'document'); } +export function hasConversationUsedDocumentTracking(metadata = {}) { + return Number(metadata.used_documents_tracking_version || 0) >= 1; +} + +export function getConversationExactUsedDocuments(metadata = {}) { + if (!hasConversationUsedDocumentTracking(metadata)) { + return []; + } + return Array.isArray(metadata.used_documents) ? metadata.used_documents : []; +} + +export function getConversationLegacyUsedDocuments(metadata = {}) { + if (!hasConversationUsedDocumentTracking(metadata)) { + return getConversationDocumentTags(metadata); + } + return Array.isArray(metadata.legacy_used_documents) + ? metadata.legacy_used_documents + : []; +} + +function mergeConversationDocumentCollections(...collections) { + const documentsById = new Map(); + collections.forEach(collection => { + if (!Array.isArray(collection)) { + return; + } + collection.forEach(doc => { + const documentId = String(doc?.document_id || '').trim(); + if (!documentId) { + return; + } + const existingDocument = documentsById.get(documentId) || {}; + documentsById.set(documentId, { + ...existingDocument, + ...doc, + }); + }); + }); + return Array.from(documentsById.values()); +} + +export function getConversationUsedDocuments(metadata = {}) { + if (!hasConversationUsedDocumentTracking(metadata)) { + return getConversationDocumentTags(metadata); + } + return mergeConversationDocumentCollections( + getConversationLegacyUsedDocuments(metadata), + getConversationExactUsedDocuments(metadata), + ); +} + export function hideConversationDetails() { const { modal } = getConversationDetailsModalElements(); const modalInstance = getConversationDetailsModalInstance(); @@ -355,6 +409,14 @@ function renderConversationMetadata(metadata, conversationId) { } const documentTags = getConversationDocumentTags(metadata); + const exactUsedDocumentIds = new Set( + getConversationExactUsedDocuments(metadata) + .map(doc => String(doc?.document_id || '').trim()) + .filter(Boolean) + ); + const documentUsageNote = hasConversationUsedDocumentTracking(metadata) + ? 'All returned source documents are listed. Cited marks documents referenced by active responses.' + : 'All associated source documents are listed. Exact citation use was not tracked for this historical conversation.'; // Documents Section if (documentTags.length > 0) { @@ -362,10 +424,11 @@ function renderConversationMetadata(metadata, conversationId) {
    -
    Documents
    +
    Source documents
    - ${renderDocumentsSection(documentTags)} +

    ${documentUsageNote}

    + ${renderDocumentsSection(documentTags, exactUsedDocumentIds)}
    @@ -748,7 +811,7 @@ function renderModelsAndAgentsSection(models, agents) { /** * Render documents section */ -function renderDocumentsSection(documents) { +function renderDocumentsSection(documents, citedDocumentIds = new Set()) { let html = ''; documents.forEach(doc => { @@ -761,6 +824,10 @@ function renderDocumentsSection(documents) { const safeDocumentTitle = escapeHtml(documentTitle); const safeScopeName = escapeHtml(scopeName); const safeScopeType = escapeHtml(doc.scope?.type || 'Unknown'); + const isCited = citedDocumentIds.has(String(doc.document_id || '').trim()); + const citedBadgeHtml = isCited + ? 'Cited' + : ''; // Format document classification with custom colors const allCategories = window.classification_categories || []; @@ -778,7 +845,10 @@ function renderDocumentsSection(documents) {
    ${safeDocumentTitle}
    - ${classificationHtml} +
    + ${citedBadgeHtml} + ${classificationHtml} +
    diff --git a/application/single_app/static/js/chat/chat-edit.js b/application/single_app/static/js/chat/chat-edit.js index f8d109a77..f4173cc6f 100644 --- a/application/single_app/static/js/chat/chat-edit.js +++ b/application/single_app/static/js/chat/chat-edit.js @@ -147,6 +147,15 @@ window.executeMessageEdit = function() { console.log(' retry_thread_id:', data.chat_request.retry_thread_id); console.log(' retry_thread_attempt:', data.chat_request.retry_thread_attempt); console.log(' Full chat_request:', data.chat_request); + window.dispatchEvent(new CustomEvent( + 'chat:conversation-documents-refresh', + { + detail: { + conversationId: data.chat_request.conversation_id, + autoOpen: false, + }, + } + )); sendMessageWithStreaming( data.chat_request, diff --git a/application/single_app/static/js/chat/chat-messages.js b/application/single_app/static/js/chat/chat-messages.js index ae90c097e..da6310144 100644 --- a/application/single_app/static/js/chat/chat-messages.js +++ b/application/single_app/static/js/chat/chat-messages.js @@ -9571,6 +9571,17 @@ function executeMessageDeletion(deleteThread = false) { if (typeof loadConversations === 'function') { loadConversations(); } + if (window.currentConversationId) { + window.dispatchEvent(new CustomEvent( + 'chat:conversation-documents-refresh', + { + detail: { + conversationId: window.currentConversationId, + autoOpen: false, + }, + } + )); + } } else { showToast('Failed to delete message', 'error'); } diff --git a/application/single_app/static/js/chat/chat-retry.js b/application/single_app/static/js/chat/chat-retry.js index 6dcdebd9e..99d3cfdf3 100644 --- a/application/single_app/static/js/chat/chat-retry.js +++ b/application/single_app/static/js/chat/chat-retry.js @@ -320,6 +320,15 @@ window.executeMessageRetry = function() { console.log(' retry_thread_id:', data.chat_request.retry_thread_id); console.log(' retry_thread_attempt:', data.chat_request.retry_thread_attempt); console.log(' Full chat_request:', data.chat_request); + window.dispatchEvent(new CustomEvent( + 'chat:conversation-documents-refresh', + { + detail: { + conversationId: data.chat_request.conversation_id, + autoOpen: false, + }, + } + )); sendMessageWithStreaming( data.chat_request, @@ -393,6 +402,15 @@ export function handleCarouselNavigation(messageDiv, messageId, direction) { }) .then(data => { console.log(`✅ Switched to attempt ${data.new_active_attempt}:`, data); + window.dispatchEvent(new CustomEvent( + 'chat:conversation-documents-refresh', + { + detail: { + conversationId: window.currentConversationId || '', + autoOpen: false, + }, + } + )); // Reload messages to show new active attempt if (window.currentConversationId) { diff --git a/application/single_app/static/js/chat/chat-streaming.js b/application/single_app/static/js/chat/chat-streaming.js index 15ab2aeff..a2a09b493 100644 --- a/application/single_app/static/js/chat/chat-streaming.js +++ b/application/single_app/static/js/chat/chat-streaming.js @@ -1,6 +1,7 @@ // chat-streaming.js import { appendMessage, + loadMessages, markUserMessageMetadataFinalizationUnconfirmed, markUserMessageMetadataUnconfirmed, refreshUserMessageMetadata, @@ -374,6 +375,11 @@ function notifyConversationDocumentsMayHaveChanged(conversationId, autoOpen = fa })); } +function hasCitedWorkspaceDocuments(payload) { + return Array.isArray(payload?.cited_hybrid_citations) + && payload.cited_hybrid_citations.length > 0; +} + async function requestStreamCancellation(streamContext = currentStreamContext) { if (!streamContext || streamContext.cancellationRequested) { return false; @@ -697,6 +703,36 @@ function consumeStreamingResponse(requestFactory, tempAiMessageId, tempUserMessa error_message: data.error, }); handleStreamError(tempAiMessageId, data.partial_content || accumulatedContent, data.error, data); + if ( + data.message_persisted + && data.message_id + && data.conversation_id + ) { + notifyConversationDocumentsMayHaveChanged( + data.conversation_id, + false + ); + if ( + data.conversation_kind === 'collaborative' + && typeof window.chatCollaboration?.loadConversationMessages === 'function' + ) { + void window.chatCollaboration + .loadConversationMessages(data.conversation_id) + .catch(error => { + console.warn( + 'Failed to reload persisted collaborative response:', + error + ); + }); + } else { + void loadMessages(data.conversation_id).catch(error => { + console.warn( + 'Failed to reload persisted response:', + error + ); + }); + } + } clearCurrentStreamController(abortController); if (typeof onError === 'function') { onError(data.error, data); @@ -1276,6 +1312,10 @@ function finalizeCancelledStreamingMessage(messageId, userMessageId, finalData, document.querySelector(`[data-message-id="${finalData.message_id}"]`), Boolean(String(partialContent || '').trim()) ); + notifyConversationDocumentsMayHaveChanged( + finalData.conversation_id, + false + ); } else if (messageElement) { renderStoppedContent(messageElement, partialContent); } @@ -1377,7 +1417,10 @@ function finalizeStreamingMessage(messageId, userMessageId, finalData, fallbackA if (existingFinalMessage) { markStreamingConversationReadIfActive(finalData.conversation_id, 'live streaming completion'); - notifyConversationDocumentsMayHaveChanged(finalData.conversation_id, Boolean(finalData.augmented)); + notifyConversationDocumentsMayHaveChanged( + finalData.conversation_id, + hasCitedWorkspaceDocuments(finalData) + ); return; } @@ -1406,7 +1449,10 @@ function finalizeStreamingMessage(messageId, userMessageId, finalData, fallbackA if (finalData.reload_messages && finalData.conversation_id && typeof window.chatMessages?.loadMessages === 'function') { window.chatMessages.loadMessages(finalData.conversation_id); } - notifyConversationDocumentsMayHaveChanged(finalData.conversation_id, Boolean(finalData.augmented)); + notifyConversationDocumentsMayHaveChanged( + finalData.conversation_id, + hasCitedWorkspaceDocuments(finalData) + ); return; } @@ -1470,7 +1516,10 @@ function finalizeStreamingMessage(messageId, userMessageId, finalData, fallbackA window.chatMessages.loadMessages(finalData.conversation_id); } - notifyConversationDocumentsMayHaveChanged(finalData.conversation_id, Boolean(finalData.augmented)); + notifyConversationDocumentsMayHaveChanged( + finalData.conversation_id, + hasCitedWorkspaceDocuments(finalData) + ); markStreamingConversationReadIfActive(finalData.conversation_id, 'live streaming completion'); } diff --git a/docs/explanation/features/CONVERSATION_CONTENTS_DRAWER.md b/docs/explanation/features/CONVERSATION_CONTENTS_DRAWER.md index 6b8cbccbd..bd7b03b01 100644 --- a/docs/explanation/features/CONVERSATION_CONTENTS_DRAWER.md +++ b/docs/explanation/features/CONVERSATION_CONTENTS_DRAWER.md @@ -7,6 +7,7 @@ The conversation contents drawer provides a compact index of persisted user mess Implemented in version: **0.250.074** Used documents mode added in version: **0.250.159** Compact overflow-safe layout added in version: **0.250.171** +Source/cited distinction added in version: **0.250.215** ### Dependencies @@ -21,7 +22,9 @@ Compact overflow-safe layout added in version: **0.250.171** The contents mode is derived entirely from messages already loaded into the chat DOM. It does not add a conversation-index API or a second datastore. `chat-conversation-contents.js` observes persisted user-message elements, renders labels with `textContent`, and tracks the message nearest the current scroll position. -The documents mode uses the same conversation metadata source as the details modal. `functions_conversation_metadata.py` retains each document's user-facing title and source filename in the conversation tag. `chat-conversation-details.js` exposes shared helpers for fetching conversation metadata and filtering `tags` where `category === "document"`. `chat-conversation-contents.js` renders those document tags into a vertical row list, so selected-but-unused documents are excluded and the side pane stays aligned with the details modal document card. +The documents mode uses the same conversation metadata source as the details modal but applies a different selector. Document tags remain the complete source inventory. New assistant messages persist exact cited document records, and conversation metadata maintains a compact `used_documents` aggregate for active responses. `chat-conversation-contents.js` renders exact used documents plus any explicit legacy fallback, while `chat-conversation-details.js` keeps the complete source list and marks exact matches with a **Cited** badge. + +`functions_citation_tracking.py` matches final-response citation IDs without mutating full source arrays. Historical conversations without versioned tracking continue to use their existing document tags in the drawer; they are not falsely marked as exact citations in the details modal. The application setting `enable_conversation_contents_drawer` is the authoritative global gate and defaults to `true`. The user setting `conversationContentsDrawerEnabled` also defaults to `true`; it is only effective while the admin gate is enabled. @@ -38,6 +41,8 @@ The application setting `enable_conversation_contents_drawer` is the authoritati - Contents and Documents share the same right-side drawer. Opening either mode swaps the drawer content instead of creating side-by-side panes. - After a streamed response completes, the drawer refreshes full conversation metadata and auto-opens Documents once per conversation when cited documents first appear. - Document rows show title, classification, source filename when it differs from the title, cited page references, and workspace scope. +- The full conversation details modal lists every source document and marks only exact tracked use with a **Cited** badge. +- Retrieved-only documents remain available under each response's **Sources** disclosure but do not enter the strict Used documents list. - Chunk counts and backend document IDs are intentionally omitted from the user-facing drawer. - Both lists use compact typography and constrained grid/flex tracks so long labels and metadata truncate instead of creating horizontal scrolling. @@ -46,6 +51,8 @@ The application setting `enable_conversation_contents_drawer` is the authoritati - `application/single_app/functions_settings.py` - `application/single_app/functions_conversation_contents.py` - `application/single_app/functions_conversation_metadata.py` +- `application/single_app/functions_citation_tracking.py` +- `application/single_app/route_backend_conversations.py` - `application/single_app/route_frontend_admin_settings.py` - `application/single_app/route_backend_users.py` - `application/single_app/route_frontend_chats.py` @@ -69,12 +76,15 @@ The application setting `enable_conversation_contents_drawer` is the authoritati - `functional_tests/test_conversation_contents_drawer_settings.py` validates global and user defaults, persistence wiring, validation, and gate precedence. - `functional_tests/test_document_action_conversation_scope_metadata.py` validates that source filenames survive conversation document-tag creation and refresh. +- `functional_tests/test_chat_cited_source_tracking.py` validates exact citation matching, source preservation, legacy compatibility, and active-attempt aggregation. - `ui_tests/test_chat_conversation_contents_drawer.py` covers filtering, ordering, safe 30-character labels, fallback labels, compact cited-document rendering, documents-mode swapping, auto-open behavior, navigation, focus, live updates, conversation replacement, keyboard closing, and desktop/mobile horizontal-overflow measurements. +- `ui_tests/test_chat_scope_lock_and_conversation_details_escaping.py` verifies the full source inventory and safe exact Cited badge. - The contents list is generated locally from authorized messages and inserts user-authored labels only through safe text APIs. The documents list also renders metadata through DOM APIs and `textContent`. ### Known limitations - Contents mode indexes user messages only. -- Documents mode includes cited/used conversation metadata documents only; documents merely selected in the picker but not used for citations are intentionally excluded. +- New tracked responses include exact cited/used documents only; retrieved-only documents are intentionally excluded. +- Historical conversations retain the prior document list as a compatibility fallback because exact citation use was not persisted before version `0.250.215`. - Search, grouping, generated summaries, assistant-message entries, and document filtering are not included. - Only messages currently loaded into the active timeline can appear in the contents list. diff --git a/docs/explanation/features/v0.238.024/CITATION_IMPROVEMENTS.md b/docs/explanation/features/v0.238.024/CITATION_IMPROVEMENTS.md index dd2029c02..66ca169c9 100644 --- a/docs/explanation/features/v0.238.024/CITATION_IMPROVEMENTS.md +++ b/docs/explanation/features/v0.238.024/CITATION_IMPROVEMENTS.md @@ -2,6 +2,7 @@ **Feature**: Citation Improvements **Version**: v0.238.024 +**Updated**: v0.250.215 **Dependencies**: Enhanced Citations System, chat-citations.js ## Overview and Purpose @@ -15,6 +16,7 @@ This release includes several improvements to the citation system in the chat in - **Improved Error Handling**: Graceful JSON parsing with explicit error messages in citation processing - **Robust Citation Links**: Support for both inline citation links and hybrid citation buttons - **Metadata Citations**: New metadata citation type for viewing extracted metadata (OCR text, vision analysis, detected objects) +- **Source and Reference Separation**: Complete returned sources remain inspectable while exact document and web citations are persisted separately for Used documents, details badges, and exports ## Technical Specifications @@ -37,6 +39,18 @@ The citation system supports multiple citation types: | **Agent Citations** | Tool execution details from Semantic Kernel agents | Expandable section with function args and results | | **Metadata Citations** | Extracted metadata from document processing | Expandable section with OCR text, vision analysis, detected objects | +### Retrieved sources and cited references + +Starting in version `0.250.215`, SimpleChat distinguishes retrieval disclosure from final-response references: + +- `hybrid_citations` and `web_search_citations` retain all returned sources for the **Sources** control and JSON audit output. +- `cited_hybrid_citations` contains only returned document records whose citation IDs appear in the final response, with strict filename/location matching for explicit legacy-style source references that omit the ID. +- `cited_web_search_citations` contains only returned web records whose normalized URLs appear in the final response. +- Agent citations continue to represent tool executions that actually occurred; Foundry web-source proxy entries are deduplicated into web references for exports. +- Conversation details list all source documents and mark exact use with **Cited**. +- Used documents and reference-producing exports consume exact cited subsets for new responses. +- Historical records without versioned tracking use the previous broad fallback and require no migration. + ### Enhanced Citation Modal Citations that reference documents open specialized viewers based on file type: @@ -62,6 +76,10 @@ The document selection dropdown in the chat interface was widened to accommodate | `static/js/chat/chat-citations.js` | Core citation parsing, link generation, and rendering | | `static/js/chat/chat-enhanced-citations.js` | Enhanced citation modal for PDF/image/video/audio viewing | | `static/js/chat/chat-documents.js` | `getDocumentMetadata()` for metadata citations; dropdown width updates | +| `functions_citation_tracking.py` | Exact document/web citation matching and compatibility-aware reference selection | +| `route_backend_conversation_export.py` | Citation-accurate conversation and per-message reference sections | +| `static/js/chat/chat-conversation-details.js` | Full source inventory and exact Cited annotations | +| `static/js/chat/chat-conversation-contents.js` | Exact Used documents selection with historical fallback | ## Commits @@ -81,3 +99,5 @@ The document selection dropdown in the chat interface was widened to accommodate - **Timestamp seeking**: Verify video and audio citations navigate to the correct timestamp - **Metadata citations**: Verify extracted metadata (OCR, vision, objects) displays correctly in expandable sections - **Error handling**: Verify malformed citation strings are handled gracefully without breaking the chat interface +- **Source/reference distinction**: Verify retrieved-only sources remain under Sources but are excluded from new Used documents and export references +- **Compatibility**: Verify historical messages and conversations retain their legacy fallback without read-time history parsing diff --git a/docs/explanation/fixes/SOURCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md b/docs/explanation/fixes/SOURCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md new file mode 100644 index 000000000..e09b91e12 --- /dev/null +++ b/docs/explanation/fixes/SOURCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md @@ -0,0 +1,96 @@ +# Retrieved Sources and Cited References Distinction Fix + +## Header information + +**Issue:** Retrieved document and web search results were presented as citations even when the final assistant response did not reference them. + +**Root cause:** SimpleChat stored complete retrieval result arrays in `hybrid_citations` and `web_search_citations`, then reused those arrays for the per-message Sources disclosure, the Used documents drawer, conversation details, and export reference sections. The data model did not preserve a separate exact-citation subset. + +Fixed/Implemented in version: **0.250.215** + +Related config.py update: `VERSION = "0.250.215"` + +Associated issue: `microsoft/simplechat#1249` + +## Technical details + +### Persisted contract + +New assistant responses preserve both source and reference data: + +| Field | Meaning | +|---|---| +| `hybrid_citations` | Every returned workspace document source | +| `web_search_citations` | Every returned web source | +| `agent_citations` | Tool and agent executions that occurred | +| `citation_tracking_version` | Indicates exact citation subsets are authoritative | +| `cited_hybrid_citations` | Returned document records matched by exact citation IDs, with strict filename/location matching for explicit legacy-style source references | +| `cited_web_search_citations` | Returned web records whose normalized URLs appear in the final response | + +Conversation records persist `used_documents` for exact active-response usage and `legacy_used_documents` for pre-tracking compatibility. Normal metadata reads use these compact fields and do not reparse message history. + +### Files modified + +- `application/single_app/functions_citation_tracking.py` +- `application/single_app/route_backend_chats.py` +- `application/single_app/route_backend_conversations.py` +- `application/single_app/functions_simplechat_operations.py` +- `application/single_app/functions_collaboration.py` +- `application/single_app/collaboration_models.py` +- `application/single_app/route_backend_collaboration.py` +- `application/single_app/functions_workflow_runner.py` +- `application/single_app/route_backend_conversation_export.py` +- `application/single_app/static/js/chat/chat-citations.js` +- `application/single_app/static/js/chat/chat-collaboration.js` +- `application/single_app/static/js/chat/chat-conversation-details.js` +- `application/single_app/static/js/chat/chat-conversation-contents.js` +- `application/single_app/static/js/chat/chat-streaming.js` + +### Code changes + +- Exact `[#citation-id]` tokens are matched to returned document records after final response content is settled. Explicit `(Source: filename, Page/Sheet/Location: value)` references provide a strict fallback for response paths that omit the hidden ID. +- Citation matching patterns use bounded quantifiers so hostile or malformed model output cannot trigger polynomial backtracking (ReDoS) while parsing references. +- Final-response HTTP(S) URLs are normalized and matched to returned web records. +- Full source arrays remain unchanged for the per-message **Sources** disclosure and JSON audit output. +- The **Used documents** drawer reads exact active-response usage plus an explicit historical fallback. +- Conversation details continue to show all source documents and add a **Cited** badge only to exact tracked documents. +- Retry, edit, attempt switching, deletion, collaboration mirroring, workflow messages, and conversation forks maintain the exact used-document aggregate. +- Canceled/interrupted persisted responses update source metadata and notify active clients; collaboration stream bridges mirror persisted partial responses before returning the terminal error. +- Conversation Markdown/PDF and per-message Word/PowerPoint/email reference sections use cited document/web subsets plus executed tools. Foundry web-source proxy entries are represented once as web references rather than duplicated as tool references. +- Historical messages without a tracking version retain their previous broad export behavior. + +### Testing approach + +- `functional_tests/test_chat_cited_source_tracking.py` validates exact document IDs, normalized web URLs, immutable source arrays, aggregation, lifecycle rebuilds, collaboration propagation, UI contracts, and export selection. +- `functional_tests/test_conversation_export.py` validates that tracked reference appendices exclude retrieved-only document and web sources while JSON retains them. +- `functional_tests/test_per_message_export.py` and `functional_tests/test_per_message_powerpoint_export.py` validate the shared Word, email, and PowerPoint reference path. +- `ui_tests/test_chat_conversation_contents_drawer.py` validates strict Used documents selection and historical fallback. +- `ui_tests/test_chat_scope_lock_and_conversation_details_escaping.py` validates the full source inventory, exact Cited badge, and safe rendering. + +### Impact analysis + +- No Cosmos migration is required. +- Existing source links and enhanced citation behavior remain unchanged. +- New responses gain exact reference semantics. +- Historical conversations remain readable and keep their previous drawer/export behavior when exact data is unavailable. +- Ordinary metadata reads remain constant-time with respect to message history; message scans occur only after uncommon lifecycle mutations. + +## Validation + +### Before + +- Every retrieved document could appear in **Used documents**. +- Conversation details could not distinguish cited documents from retrieved-only documents. +- Export reference sections could list every returned document and web result. + +### After + +- **Sources** remains the complete retrieval/execution disclosure. +- **Used documents** reflects cited documents in active responses, with an explicit legacy fallback. +- Conversation details list all source documents and mark exact citations. +- Reference-producing exports contain exact cited document/web records and executed tools. +- JSON exports preserve both complete source arrays and exact cited subsets. + +### User experience improvement + +Users can now tell what the model reviewed from what it actually cited, and exported references accurately represent the evidence visible in the generated response. diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 7ad9569c3..4ba293ab5 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,16 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](/explanation/features/) and [Fixes by Version](/explanation/fixes/). +### **(v0.250.215)** + +#### Bug Fixes + +* **Retrieved Sources and Cited References** + * Separated complete document/web retrieval results from the exact references used in final assistant responses, while preserving all returned results under **Sources**. + * Used documents now follows active cited responses, conversation details marks cited items within the full source inventory, and conversation/message export references exclude retrieved-only sources. + * Historical conversations retain their previous fallback without a migration or ordinary read-time history parsing. + * (Ref: #1249, `functions_citation_tracking.py`, `route_backend_chats.py`, `route_backend_conversation_export.py`, `SOURCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md`) + ### **(v0.250.214)** #### New Features diff --git a/functional_tests/test_chat_cited_source_tracking.py b/functional_tests/test_chat_cited_source_tracking.py new file mode 100644 index 000000000..3b8ba79c7 --- /dev/null +++ b/functional_tests/test_chat_cited_source_tracking.py @@ -0,0 +1,659 @@ +# test_chat_cited_source_tracking.py +#!/usr/bin/env python3 +""" +Functional test for cited-source tracking. +Version: 0.250.215 +Implemented in: 0.250.215 + +This test ensures returned sources remain complete while exact document and +web references drive used-document aggregates and export reference buckets. +""" + +from copy import deepcopy +from pathlib import Path +import sys +import time + +from test_support.versioning import assert_app_version_at_least + + +REPO_ROOT = Path(__file__).resolve().parents[1] +APP_ROOT = REPO_ROOT / "application" / "single_app" +CHAT_ROUTE = APP_ROOT / "route_backend_chats.py" +CONVERSATIONS_ROUTE = APP_ROOT / "route_backend_conversations.py" +COLLABORATION_FUNCTIONS = APP_ROOT / "functions_collaboration.py" +COLLABORATION_ROUTE = APP_ROOT / "route_backend_collaboration.py" +CONVERSATION_EXPORT_ROUTE = APP_ROOT / "route_backend_conversation_export.py" +CONVERSATION_DETAILS_JS = ( + APP_ROOT / "static" / "js" / "chat" / "chat-conversation-details.js" +) +CONVERSATION_CONTENTS_JS = ( + APP_ROOT / "static" / "js" / "chat" / "chat-conversation-contents.js" +) +CHAT_STREAMING_JS = APP_ROOT / "static" / "js" / "chat" / "chat-streaming.js" +CHAT_COLLABORATION_JS = ( + APP_ROOT / "static" / "js" / "chat" / "chat-collaboration.js" +) +CHAT_MESSAGES_JS = APP_ROOT / "static" / "js" / "chat" / "chat-messages.js" +CHAT_RETRY_JS = APP_ROOT / "static" / "js" / "chat" / "chat-retry.js" +CHAT_EDIT_JS = APP_ROOT / "static" / "js" / "chat" / "chat-edit.js" +SIMPLECHAT_OPERATIONS = APP_ROOT / "functions_simplechat_operations.py" +WORKFLOW_RUNNER = APP_ROOT / "functions_workflow_runner.py" +if str(APP_ROOT) not in sys.path: + sys.path.insert(0, str(APP_ROOT)) + +from functions_citation_tracking import ( # noqa: E402 + CITATION_TRACKING_VERSION, + USED_DOCUMENTS_TRACKING_VERSION, + build_cited_source_subsets, + build_used_documents, + get_message_reference_citation_buckets, + initialize_conversation_used_document_tracking, + merge_cited_documents_into_conversation, + rebuild_conversation_used_documents, + resolve_citation_location, +) +from collaboration_models import ( # noqa: E402 + build_collaboration_message_doc_from_legacy, +) + + +def _hybrid_sources(): + return [ + { + "file_name": "Policy.pdf", + "document_id": "doc-policy", + "citation_id": "doc-policy_1", + "chunk_id": "1", + "page_number": 1, + "classification": "Internal", + }, + { + "file_name": "Policy.pdf", + "document_id": "doc-policy", + "citation_id": "doc-policy_2", + "chunk_id": "2", + "page_number": 2, + "classification": "Internal", + }, + { + "file_name": "Unused.pdf", + "document_id": "doc-unused", + "citation_id": "doc-unused_7", + "chunk_id": "7", + "page_number": 7, + }, + ] + + +def _web_sources(): + return [ + { + "title": "Cited web page", + "url": "https://Example.com:443/guidance?view=current#overview", + }, + { + "title": "Unused web page", + "url": "https://example.com/unused", + }, + ] + + +def test_exact_document_and_web_references_preserve_sources(): + """Match grouped document IDs and normalized URLs without mutating sources.""" + hybrid_sources = _hybrid_sources() + web_sources = _web_sources() + original_hybrid_sources = deepcopy(hybrid_sources) + original_web_sources = deepcopy(web_sources) + content = ( + "The policy requires review " + "(Source: Policy.pdf, Pages: 1, 2) [#doc-policy_1; #doc-policy_2]. " + "See [current guidance](https://example.com/guidance?view=current). " + "An unknown token [#doc-missing_9] is ignored." + ) + + tracked = build_cited_source_subsets( + content, + hybrid_citations=hybrid_sources, + web_search_citations=web_sources, + ) + + assert tracked["citation_tracking_version"] == CITATION_TRACKING_VERSION + assert [ + citation["citation_id"] + for citation in tracked["cited_hybrid_citations"] + ] == ["doc-policy_1", "doc-policy_2"] + assert [ + citation["title"] + for citation in tracked["cited_web_search_citations"] + ] == ["Cited web page"] + assert hybrid_sources == original_hybrid_sources + assert web_sources == original_web_sources + + tracked["cited_hybrid_citations"][0]["file_name"] = "Changed.pdf" + assert hybrid_sources[0]["file_name"] == "Policy.pdf" + + +def test_explicit_source_location_matches_without_hidden_id(): + """Match the visible citation format required by the model prompt.""" + tracked = build_cited_source_subsets( + "Does the second page control (Source: Policy.pdf, Page: 2)?", + hybrid_citations=_hybrid_sources(), + web_search_citations=[], + ) + + assert [ + citation["citation_id"] + for citation in tracked["cited_hybrid_citations"] + ] == ["doc-policy_2"] + + +def test_hidden_id_prevents_same_page_source_overmatching(): + """An exact ID should win over other returned chunks from the same page.""" + same_page_source = { + **_hybrid_sources()[0], + "citation_id": "doc-policy_1_other", + "chunk_id": "1-other", + } + tracked = build_cited_source_subsets( + ( + "The first page controls " + "(Source: Policy.pdf, Page: 1) [#doc-policy_1]" + ), + hybrid_citations=[_hybrid_sources()[0], same_page_source], + web_search_citations=[], + ) + + assert [ + citation["citation_id"] + for citation in tracked["cited_hybrid_citations"] + ] == ["doc-policy_1"] + + +def test_visible_source_matching_handles_sheet_and_filename_commas(): + """Visible references should support sheet locations and original filenames.""" + tracked = build_cited_source_subsets( + ( + "The Q1 sheet controls. " + "(Source: Budget, 2026.xlsx, Sheet: Q1, Final (Approved))" + ), + hybrid_citations=[{ + "file_name": "Budget, 2026.xlsx", + "document_id": "budget-doc", + "citation_id": "budget-doc_Q1", + "page_number": "Q1, Final (Approved)", + "sheet_name": "Q1, Final (Approved)", + "location_label": "Sheet", + "location_value": "Q1, Final (Approved)", + }], + web_search_citations=[], + ) + + assert [ + citation["citation_id"] + for citation in tracked["cited_hybrid_citations"] + ] == ["budget-doc_Q1"] + + +def test_schema_aware_citation_location_is_shared(): + """Workflow and chat citations should label workbook schemas consistently.""" + assert resolve_citation_location( + page_number=1, + chunk_text="Tabular workbook: Budget.xlsx", + is_tabular=True, + ) == ("Location", "Workbook Schema") + assert resolve_citation_location( + page_number=4, + chunk_text="Narrative page", + is_tabular=False, + ) == ("Page", "4") + + +def test_citation_patterns_resist_adversarial_input(): + """Bounded citation patterns must not degrade on hostile model output.""" + adversarial_inputs = ( + "[#" + ("[#\\" * 20000), + "(Source:" + (" " * 40000), + "(Source: a, Page:" + (" " * 40000), + "(Source: a, Page: " + ("0" * 40000) + "-1)", + "[#" + ("0" * 40000), + ) + + hybrid_citations = _hybrid_sources() + started_at = time.monotonic() + for adversarial_input in adversarial_inputs: + tracked = build_cited_source_subsets( + adversarial_input, + hybrid_citations=hybrid_citations, + web_search_citations=_web_sources(), + ) + assert tracked["cited_hybrid_citations"] == [] + assert tracked["cited_web_search_citations"] == [] + elapsed_seconds = time.monotonic() - started_at + + assert elapsed_seconds < 5, ( + f"Citation matching took {elapsed_seconds:.2f}s on adversarial input" + ) + + +def test_used_documents_collapse_cited_chunks_only(): + """Collapse multiple cited chunks while excluding retrieved-only documents.""" + source_tags = [ + { + "category": "document", + "document_id": "doc-policy", + "title": "Policy Handbook", + "file_name": "Policy.pdf", + "classification": "Internal", + "chunk_ids": ["doc-policy_1", "doc-policy_2", "doc-policy_3"], + "scope": { + "type": "group", + "id": "group-1", + "name": "Policy Team", + }, + }, + ] + + used_documents = build_used_documents( + _hybrid_sources()[:2], + source_document_tags=source_tags, + ) + + assert len(used_documents) == 1 + used_document = used_documents[0] + assert used_document["document_id"] == "doc-policy" + assert used_document["title"] == "Policy Handbook" + assert used_document["chunk_ids"] == ["doc-policy_1", "doc-policy_2"] + assert used_document["citation_ids"] == ["doc-policy_1", "doc-policy_2"] + assert used_document["page_numbers"] == [1, 2] + assert used_document["scope"]["name"] == "Policy Team" + + +def test_reference_buckets_use_strict_empty_values_and_legacy_fallback(): + """Tracked messages must not fall back when exact cited subsets are empty.""" + source_message = { + "citations": [{"title": "Legacy"}], + "hybrid_citations": _hybrid_sources(), + "web_search_citations": _web_sources(), + "agent_citations": [ + {"tool_name": "Calculator", "function_name": "calculate"}, + { + "tool_name": "Used web reference", + "function_name": "azure_ai_foundry_web_search", + }, + ], + } + + legacy_buckets = get_message_reference_citation_buckets(source_message) + assert len(legacy_buckets["hybrid"]) == 3 + assert len(legacy_buckets["web"]) == 2 + assert len(legacy_buckets["agent"]) == 2 + + tracked_message = { + **source_message, + "citation_tracking_version": CITATION_TRACKING_VERSION, + "cited_hybrid_citations": [], + "cited_web_search_citations": [], + } + tracked_buckets = get_message_reference_citation_buckets(tracked_message) + assert tracked_buckets["hybrid"] == [] + assert tracked_buckets["web"] == [] + assert tracked_buckets["agent"] == [{ + "tool_name": "Calculator", + "function_name": "calculate", + }] + assert tracked_buckets["legacy"] == [{"title": "Legacy"}] + + +def test_conversation_tracking_preserves_legacy_and_merges_exact_usage(): + """Snapshot pre-tracking tags and keep new exact documents separate.""" + conversation = { + "tags": [ + { + "category": "document", + "document_id": "doc-legacy", + "title": "Historical Source", + "chunk_ids": ["doc-legacy_1"], + }, + { + "category": "participant", + "user_id": "user-1", + }, + ], + } + + initialized = initialize_conversation_used_document_tracking(conversation) + assert initialized is True + assert ( + conversation["used_documents_tracking_version"] + == USED_DOCUMENTS_TRACKING_VERSION + ) + assert [ + document["document_id"] + for document in conversation["legacy_used_documents"] + ] == ["doc-legacy"] + assert conversation["used_documents"] == [] + + conversation["tags"].append({ + "category": "document", + "document_id": "doc-policy", + "title": "Policy Handbook", + "file_name": "Policy.pdf", + "chunk_ids": ["doc-policy_1", "doc-policy_2", "doc-policy_7"], + "scope": { + "type": "group", + "id": "group-1", + "name": "Policy Team", + }, + }) + merged = merge_cited_documents_into_conversation( + conversation, + _hybrid_sources()[:2], + ) + + assert [document["document_id"] for document in merged] == ["doc-policy"] + assert merged[0]["chunk_ids"] == ["doc-policy_1", "doc-policy_2"] + assert [ + document["document_id"] + for document in conversation["legacy_used_documents"] + ] == ["doc-legacy"] + + +def test_rebuild_uses_active_non_deleted_tracked_assistant_messages(): + """Retry and deletion rebuilds must follow the active visible response set.""" + conversation = { + "tags": [ + { + "category": "document", + "document_id": "doc-policy", + "title": "Policy Handbook", + "chunk_ids": ["doc-policy_1", "doc-policy_2"], + }, + { + "category": "document", + "document_id": "doc-unused", + "title": "Unused Source", + "chunk_ids": ["doc-unused_7"], + }, + ], + "used_documents_tracking_version": USED_DOCUMENTS_TRACKING_VERSION, + "legacy_used_documents": [], + "used_documents": [], + } + messages = [ + { + "role": "assistant", + "citation_tracking_version": CITATION_TRACKING_VERSION, + "cited_hybrid_citations": [_hybrid_sources()[0]], + "metadata": { + "thread_info": { + "active_thread": True, + }, + }, + }, + { + "role": "assistant", + "citation_tracking_version": CITATION_TRACKING_VERSION, + "cited_hybrid_citations": [_hybrid_sources()[1]], + "metadata": { + "thread_info": { + "active_thread": False, + }, + }, + }, + { + "role": "assistant", + "citation_tracking_version": CITATION_TRACKING_VERSION, + "cited_hybrid_citations": [_hybrid_sources()[2]], + "metadata": { + "is_deleted": True, + "thread_info": { + "active_thread": True, + }, + }, + }, + { + "role": "assistant", + "hybrid_citations": [_hybrid_sources()[2]], + "metadata": {}, + }, + ] + + rebuilt = rebuild_conversation_used_documents(conversation, messages) + + assert [document["document_id"] for document in rebuilt] == ["doc-policy"] + assert rebuilt[0]["chunk_ids"] == ["doc-policy_1"] + + +def test_fork_rebuilds_legacy_fallback_from_copied_messages(): + """Fork legacy fallback must exclude documents introduced after the fork point.""" + conversation = { + "tags": [ + { + "category": "document", + "document_id": "doc-policy", + "title": "Policy Handbook", + "chunk_ids": ["doc-policy_1"], + }, + { + "category": "document", + "document_id": "doc-unused", + "title": "Later Source", + "chunk_ids": ["doc-unused_7"], + }, + ], + } + copied_messages = [ + { + "role": "assistant", + "hybrid_citations": [_hybrid_sources()[0]], + "metadata": { + "thread_info": { + "active_thread": True, + }, + }, + }, + ] + + rebuild_conversation_used_documents( + conversation, + copied_messages, + rebuild_legacy=True, + ) + + assert conversation["used_documents"] == [] + assert [ + document["document_id"] + for document in conversation["legacy_used_documents"] + ] == ["doc-policy"] + + +def test_chat_publication_paths_persist_tracking_and_aggregates(): + """Verify each chat publication path stores exact subsets and aggregates.""" + route_source = CHAT_ROUTE.read_text(encoding="utf-8") + + required_snippets = [ + "document_action_citation_tracking = build_cited_source_subsets(", + "citation_tracking = build_cited_source_subsets(", + "stream_citation_tracking = build_cited_source_subsets(", + "partial_citation_tracking = build_cited_source_subsets(", + "interrupted_citation_tracking = build_cited_source_subsets(", + "**document_action_citation_tracking,", + "**citation_tracking,", + "**stream_citation_tracking,", + "initialize_conversation_used_document_tracking(conversation_item)", + "merge_cited_documents_into_conversation(", + "'hybrid_citations': hybrid_citations_list", + "'web_search_citations': web_search_citations_list", + ] + missing = [ + snippet + for snippet in required_snippets + if snippet not in route_source + ] + assert not missing, f"Missing chat citation tracking wiring: {missing}" + assert route_source.count("merge_cited_documents_into_conversation(") >= 4 + assert "Copy the exact bracketed citation ID shown after the supporting excerpt." in route_source + assert 'reason="chat_stream_interrupted"' in route_source + assert route_source.count("collect_stream_response_conversation_metadata()") >= 4 + assert '"sheet_name": source_doc.get("sheet_name")' in route_source + assert '"sheet_name": sheet_name' in route_source + + +def test_lifecycle_mutations_and_forks_rebuild_exact_usage(): + """Verify retry/edit/switch/delete/fork paths invoke the shared rebuild.""" + route_source = CONVERSATIONS_ROUTE.read_text(encoding="utf-8") + operations_source = SIMPLECHAT_OPERATIONS.read_text(encoding="utf-8") + + assert ( + route_source.count( + "_rebuild_authorized_personal_conversation_used_documents(" + ) + >= 5 + ) + for reason in ( + "message_deleted", + "message_retry_created", + "message_edit_created", + "message_attempt_switched", + ): + assert reason in route_source + assert '"used_documents_tracking_version"' in operations_source + assert '"legacy_used_documents"' in operations_source + assert "rebuild_legacy=True" in operations_source + assert '"used_documents_tracking_version": conversation_item.get(' in route_source + assert '"legacy_used_documents": conversation_item.get(' in route_source + assert '"used_documents": conversation_item.get(' in route_source + assert "except Exception as rebuild_error:" in route_source + assert "used_documents_rebuild_required" in route_source + + +def test_collaboration_and_workflow_propagate_tracking_contract(): + """Verify shared and workflow messages retain exact cited subsets.""" + legacy_message = { + "id": "assistant-source-1", + "role": "assistant", + "content": "Tracked response", + "timestamp": "2026-08-14T12:00:00Z", + "citation_tracking_version": CITATION_TRACKING_VERSION, + "cited_hybrid_citations": [_hybrid_sources()[0]], + "cited_web_search_citations": [_web_sources()[0]], + } + converted = build_collaboration_message_doc_from_legacy( + conversation_id="collaboration-1", + legacy_message=legacy_message, + default_sender_user={ + "user_id": "user-1", + "display_name": "Test User", + "email": "test@example.com", + }, + ) + assert converted["citation_tracking_version"] == CITATION_TRACKING_VERSION + assert converted["cited_hybrid_citations"][0]["citation_id"] == "doc-policy_1" + assert converted["cited_web_search_citations"][0]["title"] == "Cited web page" + + collaboration_source = COLLABORATION_FUNCTIONS.read_text(encoding="utf-8") + collaboration_route_source = COLLABORATION_ROUTE.read_text(encoding="utf-8") + workflow_source = WORKFLOW_RUNNER.read_text(encoding="utf-8") + + for field_name in ( + "citation_tracking_version", + "cited_hybrid_citations", + "cited_web_search_citations", + "used_documents_tracking_version", + "legacy_used_documents", + "used_documents", + ): + assert field_name in collaboration_source + assert "rebuild_conversation_used_documents(conversation_doc, remaining_messages)" in collaboration_source + assert "'cited_hybrid_citations': serialized_assistant_message.get(" in collaboration_route_source + assert "stream_payload['done'] = True" in collaboration_route_source + assert "'conversation_kind': serialized_final_conversation.get(" in collaboration_route_source + assert "citation_tracking = build_cited_source_subsets(" in workflow_source + assert "merge_cited_documents_into_conversation(" in workflow_source + assert "(Source: {file_name}, {location_label}: {location_value}) [#{citation_id}]" in workflow_source + assert "'sheet_name': sheet_name" in workflow_source + assert "resolve_citation_location(" in workflow_source + + +def test_ui_and_exports_select_cited_subsets_without_losing_sources(): + """Verify downstream consumers separate source inventory from references.""" + details_source = CONVERSATION_DETAILS_JS.read_text(encoding="utf-8") + contents_source = CONVERSATION_CONTENTS_JS.read_text(encoding="utf-8") + export_source = CONVERSATION_EXPORT_ROUTE.read_text(encoding="utf-8") + + assert "export function getConversationDocumentTags(metadata = {})" in details_source + assert "export function getConversationUsedDocuments(metadata = {})" in details_source + assert "getConversationExactUsedDocuments(metadata)" in details_source + assert "getConversationLegacyUsedDocuments(metadata)" in details_source + assert "conversation-document-cited-badge" in details_source + assert "Source documents" in details_source + assert "getConversationUsedDocuments," in contents_source + assert "const documents = getConversationUsedDocuments(metadata);" in contents_source + assert "Documents cited in active responses" in contents_source + assert "exactDocuments.length > 0" in contents_source + + streaming_source = CHAT_STREAMING_JS.read_text(encoding="utf-8") + assert "function hasCitedWorkspaceDocuments(payload)" in streaming_source + assert "hasCitedWorkspaceDocuments(finalData)" in streaming_source + assert ( + "notifyConversationDocumentsMayHaveChanged(" + in streaming_source + ) + assert "data.message_persisted" in streaming_source + assert "&& data.message_id" in streaming_source + assert "&& data.conversation_id" in streaming_source + assert "void loadMessages(data.conversation_id).catch" in streaming_source + assert "data.conversation_kind === 'collaborative'" in streaming_source + assert ".loadConversationMessages(data.conversation_id)" in streaming_source + + collaboration_ui_source = CHAT_COLLABORATION_JS.read_text(encoding="utf-8") + assert "'chat:conversation-documents-refresh'" in collaboration_ui_source + assert "String(payload.message.role || '').trim().toLowerCase() === 'assistant'" in collaboration_ui_source + assert "conversationKind: 'collaborative'" in collaboration_ui_source + assert "loadConversationMessages," in collaboration_ui_source + + for mutation_source_path in ( + CHAT_MESSAGES_JS, + CHAT_RETRY_JS, + CHAT_EDIT_JS, + ): + mutation_source = mutation_source_path.read_text(encoding="utf-8") + assert "'chat:conversation-documents-refresh'" in mutation_source + + assert "return get_message_reference_citation_buckets(message)" in export_source + assert "return get_message_source_citation_buckets(message)" in export_source + assert "source_citation_buckets = _collect_source_citation_buckets(message)" in export_source + assert "'cited_hybrid_citations': (" in export_source + assert "'cited_web_search_citations': (" in export_source + assert "Document References" in export_source + assert "Web References" in export_source + + +def test_version_is_available(): + """Verify the application includes the cited-source tracking version.""" + assert_app_version_at_least("0.250.215") + + +if __name__ == "__main__": + tests = [ + test_exact_document_and_web_references_preserve_sources, + test_explicit_source_location_matches_without_hidden_id, + test_hidden_id_prevents_same_page_source_overmatching, + test_visible_source_matching_handles_sheet_and_filename_commas, + test_schema_aware_citation_location_is_shared, + test_citation_patterns_resist_adversarial_input, + test_used_documents_collapse_cited_chunks_only, + test_reference_buckets_use_strict_empty_values_and_legacy_fallback, + test_conversation_tracking_preserves_legacy_and_merges_exact_usage, + test_rebuild_uses_active_non_deleted_tracked_assistant_messages, + test_fork_rebuilds_legacy_fallback_from_copied_messages, + test_chat_publication_paths_persist_tracking_and_aggregates, + test_lifecycle_mutations_and_forks_rebuild_exact_usage, + test_collaboration_and_workflow_propagate_tracking_contract, + test_ui_and_exports_select_cited_subsets_without_losing_sources, + test_version_is_available, + ] + for test in tests: + test() + print(f"Cited-source tracking tests passed: {len(tests)}/{len(tests)}") diff --git a/functional_tests/test_conversation_export.py b/functional_tests/test_conversation_export.py index 654059b13..a6daf5277 100644 --- a/functional_tests/test_conversation_export.py +++ b/functional_tests/test_conversation_export.py @@ -2,8 +2,8 @@ # test_conversation_export.py """ Functional test for conversation export enhancements. -Version: 0.239.029 -Implemented in: 0.239.022 (base), 0.239.023 (PDF export), 0.239.025 (tag formatting fix), 0.239.028 (summary token budget fix) +Version: 0.250.215 +Implemented in: 0.239.022 (base); Updated in: 0.250.215 This test ensures conversation export now includes normalized and raw citations, processing thoughts, transcript-style Markdown appendices, deleted-message filtering, @@ -20,9 +20,11 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'application', 'single_app')) from route_backend_conversation_export import ( # noqa: E402 + _build_message_citation_labels, _build_citation_counts, _build_pdf_html_body, _collect_raw_citation_buckets, + _collect_source_citation_buckets, _conversation_to_markdown, _conversation_to_pdf_bytes, _filter_messages_for_export, @@ -130,6 +132,130 @@ def test_sanitize_message_with_citations_and_thoughts(): return True +def test_tracked_export_references_exclude_uncited_sources(): + """Tracked exports should reference exact subsets while retaining source audit data.""" + print("Testing tracked export reference filtering...") + + cited_document = { + 'file_name': 'used-policy.pdf', + 'document_id': 'doc-used', + 'citation_id': 'doc-used_2', + 'page_number': 2, + } + uncited_document = { + 'file_name': 'retrieved-only.pdf', + 'document_id': 'doc-unused', + 'citation_id': 'doc-unused_8', + 'page_number': 8, + } + cited_web = { + 'title': 'Used web reference', + 'url': 'https://example.com/used', + } + uncited_web = { + 'title': 'Retrieved web result', + 'url': 'https://example.com/unused', + } + message = { + 'id': 'assistant-tracked', + 'role': 'assistant', + 'content': ( + 'The cited policy applies (Source: used-policy.pdf, Page: 2). ' + 'See https://example.com/used.' + ), + 'timestamp': '2026-08-14T12:00:00Z', + 'citation_tracking_version': 1, + 'hybrid_citations': [cited_document, uncited_document], + 'web_search_citations': [cited_web, uncited_web], + 'cited_hybrid_citations': [cited_document], + 'cited_web_search_citations': [cited_web], + 'agent_citations': [ + { + 'tool_name': 'Calculator', + 'function_name': 'calculate', + }, + { + 'tool_name': 'Retrieved web result', + 'function_name': 'azure_ai_foundry_web_search', + 'plugin_name': 'azure_ai_foundry', + }, + ], + } + + references = _collect_raw_citation_buckets(message) + sources = _collect_source_citation_buckets(message) + sanitized = _sanitize_message( + message, + sequence_index=1, + transcript_index=1, + thoughts=[], + ) + labels = _build_message_citation_labels(message) + + assert [item['file_name'] for item in references['hybrid']] == ['used-policy.pdf'] + assert [item['title'] for item in references['web']] == ['Used web reference'] + assert len(sources['hybrid']) == 2 + assert len(sources['web']) == 2 + assert sanitized['citation_counts'] == { + 'document': 1, + 'web': 1, + 'agent_tool': 1, + 'legacy': 0, + 'total': 3, + } + assert len(sanitized['hybrid_citations']) == 2 + assert len(sanitized['web_search_citations']) == 2 + assert len(sanitized['agent_citations']) == 2 + assert sanitized['cited_hybrid_citations'] == [cited_document] + assert sanitized['cited_web_search_citations'] == [cited_web] + assert 'used-policy.pdf — Page 2' in labels + assert 'Used web reference' in labels + assert 'Calculator' in labels + assert all('retrieved-only.pdf' not in label for label in labels) + assert all('Retrieved web result' not in label for label in labels) + + entry = { + 'conversation': { + 'id': 'conversation-tracked', + 'title': 'Tracked export', + 'message_count': 1, + 'citation_counts': sanitized['citation_counts'], + }, + 'summary_intro': {'enabled': False, 'generated': False}, + 'messages': [sanitized], + } + markdown = _conversation_to_markdown(entry) + pdf_html = _build_pdf_html_body(entry) + + assert 'Document References' in markdown + assert 'Web References' in markdown + assert 'retrieved-only.pdf' not in markdown + assert 'Retrieved web result' not in markdown + assert 'Document References' in pdf_html + assert 'Web References' in pdf_html + assert 'retrieved-only.pdf' not in pdf_html + assert 'Retrieved web result' not in pdf_html + + sheet_reference = { + 'file_name': 'Budget, 2026.xlsx', + 'document_id': 'budget-doc', + 'citation_id': 'budget-doc_Q1', + 'page_number': 'Q1', + 'sheet_name': 'Q1', + 'location_label': 'Sheet', + 'location_value': 'Q1', + } + sheet_labels = _build_message_citation_labels({ + 'citation_tracking_version': 1, + 'hybrid_citations': [sheet_reference], + 'cited_hybrid_citations': [sheet_reference], + }) + assert sheet_labels == ['Budget, 2026.xlsx — Sheet: Q1'] + + print("Tracked export reference filtering passed") + return True + + def test_conversation_metadata_and_json_shape(): """Test that conversation-level metadata captures counts for export JSON.""" print("🔍 Testing conversation metadata and JSON shape...") @@ -534,6 +660,7 @@ def test_tag_formatting(): tests = [ test_filter_messages_for_export, test_sanitize_message_with_citations_and_thoughts, + test_tracked_export_references_exclude_uncited_sources, test_conversation_metadata_and_json_shape, test_markdown_export_structure, test_safe_filename_and_zip_packaging, diff --git a/functional_tests/test_per_message_export.py b/functional_tests/test_per_message_export.py index 3b8e3ddf9..ef933df24 100644 --- a/functional_tests/test_per_message_export.py +++ b/functional_tests/test_per_message_export.py @@ -2,8 +2,8 @@ # test_per_message_export.py """ Functional tests for the per-message export feature and export route regressions. -Version: 0.241.146 -Implemented in: 0.241.019 +Version: 0.250.215 +Implemented in: 0.241.019; Updated in: 0.250.215 Covers: - Happy path: Word document built successfully from a valid message. @@ -874,6 +874,8 @@ def test_per_message_export_uses_content_override_contract(): assert 'def _apply_message_export_content_override(' in route_source assert route_source.count('message_content_override = _get_message_export_content_override(data)') >= 3 assert route_source.count('_apply_message_export_content_override(message, message_content_override)') >= 3 + assert 'get_message_reference_citation_buckets' in route_source + assert 'normalized_citations = _normalize_citations(_collect_raw_citation_buckets(message))' in route_source assert 'function buildMessageExportRequestBody(messageDiv, messageId, conversationId, role, extraFields = {})' in frontend_source assert 'requestBody.message_content_override = messageContentOverride;' in frontend_source diff --git a/functional_tests/test_per_message_powerpoint_export.py b/functional_tests/test_per_message_powerpoint_export.py index 4d6963fc6..7dc09bdc8 100644 --- a/functional_tests/test_per_message_powerpoint_export.py +++ b/functional_tests/test_per_message_powerpoint_export.py @@ -2,8 +2,8 @@ # test_per_message_powerpoint_export.py """ Functional test for per-message PowerPoint export. -Version: 0.241.146 -Implemented in: 0.241.033 +Version: 0.250.215 +Implemented in: 0.241.033; Updated in: 0.250.215 This test ensures the message export flow exposes a PowerPoint route, uses the frontend PowerPoint action hook, prefers the message model @@ -426,6 +426,7 @@ def test_powerpoint_frontend_hooks_present() -> bool: assert "fetch('/api/message/export-powerpoint'" in frontend_source, 'Expected frontend fetch for the PowerPoint export endpoint' assert 'exportMessageAsPowerPoint' in frontend_source, 'Expected frontend PowerPoint export helper' + assert 'get_message_reference_citation_buckets' in ROUTE_FILE.read_text(encoding='utf-8') assert 'message_content_override' in frontend_source, 'Expected PowerPoint export requests to support edited assistant markdown overrides' assert 'delete requestBody.message_content_override;' in frontend_source, 'Expected artifact PowerPoint export to ignore visible-message overrides' assert 'dropdown-export-ppt-btn' in menu_source, 'Expected chat message menu PowerPoint action' diff --git a/ui_tests/test_chat_conversation_contents_drawer.py b/ui_tests/test_chat_conversation_contents_drawer.py index e3dc96180..505669872 100644 --- a/ui_tests/test_chat_conversation_contents_drawer.py +++ b/ui_tests/test_chat_conversation_contents_drawer.py @@ -1,10 +1,11 @@ # test_chat_conversation_contents_drawer.py """ UI test for the conversation contents drawer. -Version: 0.250.171 +Version: 0.250.215 Implemented in: 0.250.074 Documents mode added in: 0.250.159 Compact overflow-safe layout added in: 0.250.171 +Source/cited distinction added in: 0.250.215 This test validates user-message filtering, cited-document mode, safe labels, navigation, live updates, conversation replacement, keyboard closing, and @@ -239,38 +240,53 @@ def test_chat_conversation_contents_drawer(playwright, viewport): "bank-treasury-operations-quarterly-reference-" "with-an-intentionally-long-filename.xlsx" ) + source_document_tags = [ + { + "category": "document", + "document_id": "doc-1", + "title": document_title, + "file_name": document_file_name, + "classification": "Confidential", + "chunk_ids": ["doc-1_1", "doc-1_2"], + "scope": { + "type": "group", + "id": "group-1", + "name": "Product Documentation Workspace With A Long Display Name", + }, + }, + { + "category": "document", + "document_id": "doc-2", + "title": "Release Plan", + "file_name": "release-plan.pdf", + "classification": "None", + "chunk_ids": ["doc-2_4"], + "scope": { + "type": "personal", + "id": "user-1", + "name": "Personal", + }, + }, + ] page.route( "**/api/conversations/conversation-documents-pane/metadata", lambda route: _fulfill_json(route, { "title": "Documents pane conversation", - "tags": [ - { - "category": "document", - "document_id": "doc-1", - "title": document_title, - "file_name": document_file_name, - "classification": "Confidential", - "chunk_ids": ["doc-1_1", "doc-1_2"], - "scope": { - "type": "group", - "id": "group-1", - "name": "Product Documentation Workspace With A Long Display Name", - }, - }, - { - "category": "document", - "document_id": "doc-2", - "title": "Release Plan", - "file_name": "release-plan.pdf", - "classification": "None", - "chunk_ids": ["doc-2_4"], - "scope": { - "type": "personal", - "id": "user-1", - "name": "Personal", - }, - }, - ], + "tags": source_document_tags, + "used_documents_tracking_version": 1, + "legacy_used_documents": [], + "used_documents": [{ + **source_document_tags[0], + "chunk_ids": ["doc-1_99"], + "citation_ids": ["doc-1_99"], + "page_numbers": [2], + "citation_locations": [{ + "citation_id": "doc-1_99", + "page_number": 2, + "location_label": "Page", + "location_value": 2, + }], + }], }), ) @@ -294,18 +310,20 @@ def test_chat_conversation_contents_drawer(playwright, viewport): expect(documents_toggle).to_be_visible() expect(page.locator("#conversation-contents-drawer")).to_be_visible() expect(page.locator("#conversation-contents-title")).to_have_text("Used documents") - expect(documents_entries).to_have_count(2) + expect(page.locator("#conversation-contents-subtitle")).to_have_text( + "Documents cited in active responses" + ) + expect(documents_entries).to_have_count(1) expect(documents_entries.nth(0)).to_contain_text(document_title) expect(documents_entries.nth(0)).to_contain_text(document_file_name) expect(documents_entries.nth(0)).to_contain_text("Confidential") - expect(documents_entries.nth(0)).to_contain_text("Pages: 1, 2") + expect(documents_entries.nth(0)).to_contain_text("Pages: 2") expect(documents_entries.nth(0)).to_contain_text( "group scope: Product Documentation Workspace With A Long Display Name" ) expect(documents_entries.nth(0)).not_to_contain_text("chunks") expect(documents_entries.nth(0)).not_to_contain_text("ID:") - expect(documents_entries.nth(1)).to_contain_text("Release Plan") - expect(page.locator("#conversation-documents-count")).to_have_text("2") + expect(page.locator("#conversation-documents-count")).to_have_text("1") expect(page.locator("#conversation-documents-panel img[src='x']")).to_have_count(0) assert page.evaluate("() => window.__documentsPaneXss") is False _assert_no_horizontal_overflow( @@ -314,6 +332,39 @@ def test_chat_conversation_contents_drawer(playwright, viewport): ) _assert_no_horizontal_overflow(documents_entries.nth(0), "document entry") + documents_toggle.click() + expect(page.locator("#conversation-contents-drawer")).to_be_hidden() + + page.route( + "**/api/conversations/conversation-documents-legacy/metadata", + lambda route: _fulfill_json(route, { + "title": "Legacy documents pane conversation", + "tags": source_document_tags, + }), + ) + page.evaluate( + """ + () => { + window.currentConversationId = 'conversation-documents-legacy'; + window.dispatchEvent(new CustomEvent('chat:conversation-documents-refresh', { + detail: { + conversationId: 'conversation-documents-legacy', + autoOpen: true + } + })); + } + """ + ) + expect(documents_entries).to_have_count(2) + expect(documents_entries.nth(1)).to_contain_text("Release Plan") + expect(page.locator("#conversation-documents-count")).to_have_text("2") + expect(page.locator("#conversation-contents-drawer")).to_be_hidden() + expect(page.locator("#conversation-contents-subtitle")).to_have_text( + "Documents associated with this historical conversation" + ) + + documents_toggle.click() + expect(page.locator("#conversation-contents-drawer")).to_be_visible() page.locator("#conversation-contents-mode-contents").click() expect(page.locator("#conversation-contents-title")).to_have_text("Conversation contents") expect(page.locator("#conversation-contents-panel")).to_be_visible() diff --git a/ui_tests/test_chat_markdown_citation_lookup_payload.py b/ui_tests/test_chat_markdown_citation_lookup_payload.py index 4516667b8..b2a5cf5ba 100644 --- a/ui_tests/test_chat_markdown_citation_lookup_payload.py +++ b/ui_tests/test_chat_markdown_citation_lookup_payload.py @@ -2,8 +2,8 @@ """ UI test for markdown citation lookup payloads. -Version: 0.241.021 -Implemented in: 0.241.021 +Version: 0.250.215 +Implemented in: 0.241.021; Updated in: 0.250.215 This test ensures markdown citation buttons send document, page, and chunk context to `/api/get_citation` so text citation lookup can recover when the @@ -108,11 +108,92 @@ def handle_get_citation(route): }, true ); + messagesModule.appendMessage( + 'AI', + 'Workbook answer (Source: Budget, 2026.xlsx, Sheet: Q1, Final (Approved)) [#budget-doc_Q1]', + null, + 'assistant-sheet-1', + true, + [ + { + file_name: 'Budget, 2026.xlsx', + document_id: 'budget-doc', + citation_id: 'budget-doc_Q1', + chunk_id: 'Q1', + page_number: 'Q1, Final (Approved)', + sheet_name: 'Q1, Final (Approved)', + location_label: 'Sheet', + location_value: 'Q1, Final (Approved)', + }, + ], + [], + [], + null, + null, + { + id: 'assistant-sheet-1', + role: 'assistant', + content: 'Workbook answer (Source: Budget, 2026.xlsx, Sheet: Q1, Final (Approved)) [#budget-doc_Q1]', + }, + true + ); + messagesModule.appendMessage( + 'AI', + 'A (Source: A.pdf, Page: 1). B (Source: B.pdf, Page: 2) [#b-doc_2]', + null, + 'assistant-adjacent-1', + true, + [ + { + file_name: 'B.pdf', + document_id: 'b-doc', + citation_id: 'b-doc_2', + chunk_id: '2', + page_number: 2, + location_label: 'Page', + location_value: 2, + }, + ], + [], + [], + null, + null, + { + id: 'assistant-adjacent-1', + role: 'assistant', + content: 'A (Source: A.pdf, Page: 1). B (Source: B.pdf, Page: 2) [#b-doc_2]', + }, + true + ); } """ ) message = page.locator('.message[data-message-id="assistant-md-1"]') + sheet_message = page.locator( + '.message[data-message-id="assistant-sheet-1"]' + ) + sheet_link = sheet_message.locator( + 'a.citation-link[data-citation-id="budget-doc_Q1"]' + ) + expect(sheet_link).to_be_visible() + expect(sheet_link).to_have_text("Q1, Final (Approved)") + expect(sheet_link).to_have_attribute( + "data-sheet-name", + "Q1, Final (Approved)" + ) + adjacent_message = page.locator( + '.message[data-message-id="assistant-adjacent-1"]' + ) + expect(adjacent_message.locator(".message-text")).to_contain_text( + "(Source: A.pdf, Page: 1)" + ) + adjacent_link = adjacent_message.locator( + 'a.citation-link[data-citation-id="b-doc_2"]' + ) + expect(adjacent_link).to_be_visible() + expect(adjacent_link).to_have_text("2") + citation_toggle = message.locator('.citation-toggle-btn') expect(citation_toggle).to_be_visible() citation_toggle.click() diff --git a/ui_tests/test_chat_scope_lock_and_conversation_details_escaping.py b/ui_tests/test_chat_scope_lock_and_conversation_details_escaping.py index e622b94dc..f4cf6e6dc 100644 --- a/ui_tests/test_chat_scope_lock_and_conversation_details_escaping.py +++ b/ui_tests/test_chat_scope_lock_and_conversation_details_escaping.py @@ -1,8 +1,8 @@ # test_chat_scope_lock_and_conversation_details_escaping.py """ UI test for chat scope-lock and conversation-details escaping. -Version: 0.241.019 -Implemented in: 0.241.019 +Version: 0.250.215 +Implemented in: 0.241.019; Updated in: 0.250.215 This test ensures malicious workspace names and conversation metadata render as inert text in the chat scope-lock modal and conversation-details modal. @@ -92,6 +92,18 @@ def test_chat_scope_lock_and_conversation_details_escape_malicious_metadata(play "name": document_scope_name, }, }, + { + "category": "document", + "document_id": "doc-2", + "title": "Retrieved but not cited", + "classification": "Public", + "chunk_ids": ["chunk_2_p1"], + "scope": { + "type": "personal", + "id": "user-1", + "name": "Personal", + }, + }, { "category": "semantic", "value": semantic_tag, @@ -109,6 +121,22 @@ def test_chat_scope_lock_and_conversation_details_escape_malicious_metadata(play "value": web_source, }, ], + "used_documents_tracking_version": 1, + "legacy_used_documents": [], + "used_documents": [ + { + "category": "document", + "document_id": "doc-1", + "title": document_title, + "classification": classification_label, + "chunk_ids": ["chunk_1_p2"], + "scope": { + "type": "group", + "id": "group-1", + "name": document_scope_name, + }, + }, + ], "strict": False, "classification": [classification_label], "last_updated": "2026-05-05T12:00:00Z", @@ -202,7 +230,19 @@ def fulfill_empty_docs_or_tags(route): expect(details_content).to_contain_text(participant_name) expect(details_content).to_contain_text(participant_email) expect(details_content).to_contain_text(document_title) + expect(details_content).to_contain_text("Retrieved but not cited") expect(details_content).to_contain_text(document_scope_name) + expect(details_content).to_contain_text("Source documents") + expect( + page.locator( + "#conversation-details-modal .conversation-document-cited-badge" + ) + ).to_have_count(1) + expect( + page.locator( + "#conversation-details-modal .conversation-document-cited-badge" + ) + ).to_have_text("Cited") expect(details_content).to_contain_text(classification_label) expect(details_content).to_contain_text(semantic_tag) expect(details_content).to_contain_text(model_tag)