diff --git a/application/single_app/collaboration_models.py b/application/single_app/collaboration_models.py
index 0c98dd940..e2f322ea2 100644
--- a/application/single_app/collaboration_models.py
+++ b/application/single_app/collaboration_models.py
@@ -678,6 +678,9 @@ def build_collaboration_message_doc_from_legacy(
'augmented',
'hybrid_citations',
'web_search_citations',
+ 'citation_tracking_version',
+ 'cited_hybrid_citations',
+ 'cited_web_search_citations',
'agent_citations',
'agent_display_name',
'agent_name',
diff --git a/application/single_app/config.py b/application/single_app/config.py
index 534b9a812..3661cf2b0 100644
--- a/application/single_app/config.py
+++ b/application/single_app/config.py
@@ -96,7 +96,7 @@
EXECUTOR_TYPE = 'thread'
EXECUTOR_MAX_WORKERS = 30
SESSION_TYPE = 'filesystem'
-VERSION = "0.250.214"
+VERSION = "0.250.215"
IS_DEVELOPMENT = is_development_env_enabled()
SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax')
diff --git a/application/single_app/functions_citation_tracking.py b/application/single_app/functions_citation_tracking.py
new file mode 100644
index 000000000..b2da0ab74
--- /dev/null
+++ b/application/single_app/functions_citation_tracking.py
@@ -0,0 +1,691 @@
+# functions_citation_tracking.py
+"""Track retrieved sources separately from explicitly cited references."""
+
+import html
+import re
+from copy import deepcopy
+from typing import Any, Dict, Iterable, List, Mapping, Optional
+from urllib.parse import urlsplit, urlunsplit
+
+
+CITATION_TRACKING_VERSION = 1
+USED_DOCUMENTS_TRACKING_VERSION = 1
+
+_INLINE_DOCUMENT_CITATION_GROUP_PATTERN = re.compile(
+ r"\[\s{0,8}#([^\]\r\n]{1,1024})\]",
+ re.IGNORECASE,
+)
+_INLINE_DOCUMENT_SOURCE_PATTERN = re.compile(
+ r"\(Source:((?:(?!\(Source:).){1,256}?),"
+ r"\s{0,8}(Page(?:s)?|Sheet(?:s)?|Location):"
+ r"((?:(?!\(Source:).){0,256}?)\)"
+ r"(?=\s{0,8}(?:\[#|$|[.;,!? \n]))",
+ re.IGNORECASE,
+)
+_PAGE_RANGE_PATTERN = re.compile(
+ r"(\d{1,9})\s{0,8}[-\u2013\u2014]\s{0,8}(\d{1,9})"
+)
+_HTTP_URL_PATTERN = re.compile(r"https?://[^\s<>\"']+", re.IGNORECASE)
+_TRAILING_URL_PUNCTUATION = ".,;:!?"
+_MERGED_DOCUMENT_LIST_FIELDS = (
+ "chunk_ids",
+ "citation_ids",
+ "page_numbers",
+ "sheet_names",
+)
+
+
+def _as_dict_list(value: Any) -> List[Dict[str, Any]]:
+ if not isinstance(value, list):
+ return []
+ return [deepcopy(item) for item in value if isinstance(item, dict)]
+
+
+def _append_unique(values: List[Any], value: Any) -> None:
+ if value in (None, "") or value in values:
+ return
+ values.append(deepcopy(value))
+
+
+def _append_unique_mapping(values: List[Dict[str, Any]], value: Dict[str, Any]) -> None:
+ if not value or value in values:
+ return
+ values.append(deepcopy(value))
+
+
+def _normalize_reference_id(value: Any) -> str:
+ return str(value or "").strip().lstrip("#").strip()
+
+
+def extract_explicit_document_citation_ids(content: Any) -> List[str]:
+ """Return ordered, unique citation IDs from explicit ``[#id]`` groups."""
+ citation_ids: List[str] = []
+ for citation_group in _INLINE_DOCUMENT_CITATION_GROUP_PATTERN.findall(
+ str(content or "")
+ ):
+ for raw_citation_id in re.split(r"[;,]", citation_group):
+ citation_id = _normalize_reference_id(raw_citation_id)
+ if citation_id and citation_id not in citation_ids:
+ citation_ids.append(citation_id)
+ return citation_ids
+
+
+def _normalize_source_text(value: Any) -> str:
+ return re.sub(r"\s+", " ", html.unescape(str(value or ""))).strip().lower()
+
+
+def _extract_explicit_document_source_references(
+ content: Any,
+) -> List[Dict[str, str]]:
+ references = []
+ for file_name, location_label, location_value in (
+ _INLINE_DOCUMENT_SOURCE_PATTERN.findall(str(content or ""))
+ ):
+ normalized_file_name = _normalize_source_text(file_name)
+ normalized_location_label = (
+ _normalize_source_text(location_label).rstrip("s")
+ )
+ normalized_location_value = _normalize_source_text(location_value)
+ if (
+ normalized_file_name
+ and normalized_location_label
+ and normalized_location_value
+ ):
+ references.append({
+ "file_name": normalized_file_name,
+ "location_label": normalized_location_label,
+ "location_value": normalized_location_value,
+ })
+ return references
+
+
+def _parse_numeric_location(value: Any) -> Optional[int]:
+ normalized_value = _normalize_source_text(value)
+ if not normalized_value or not normalized_value.isdigit():
+ return None
+ return int(normalized_value)
+
+
+def _location_reference_matches(
+ explicit_location: str,
+ citation_location: Any,
+) -> bool:
+ normalized_citation_location = _normalize_source_text(citation_location)
+ if not normalized_citation_location:
+ return False
+ if normalized_citation_location == explicit_location:
+ return True
+
+ explicit_tokens = [
+ token.strip()
+ for token in re.split(r"[,;]", explicit_location)
+ if token.strip()
+ ]
+ if normalized_citation_location in explicit_tokens:
+ return True
+
+ citation_number = _parse_numeric_location(normalized_citation_location)
+ if citation_number is None:
+ return False
+
+ for token in explicit_tokens:
+ range_match = _PAGE_RANGE_PATTERN.fullmatch(token)
+ if not range_match:
+ continue
+ range_start, range_end = (
+ int(range_match.group(1)),
+ int(range_match.group(2)),
+ )
+ lower_bound = min(range_start, range_end)
+ upper_bound = max(range_start, range_end)
+ if lower_bound <= citation_number <= upper_bound:
+ return True
+ return False
+
+
+def resolve_citation_location(
+ page_number: Any = None,
+ chunk_text: Any = None,
+ sheet_name: Any = None,
+ is_tabular: bool = False,
+) -> tuple[str, str]:
+ """Return a schema-, sheet-, or page-aware citation location."""
+ if sheet_name not in (None, ""):
+ return "Sheet", str(sheet_name)
+
+ normalized_chunk_text = str(chunk_text or "").strip()
+ if is_tabular and (
+ normalized_chunk_text.startswith("Tabular workbook:")
+ or normalized_chunk_text.startswith("Tabular data file:")
+ ):
+ return "Location", "Workbook Schema"
+
+ return "Page", str(page_number or 1)
+
+
+def _source_reference_matches_citation(
+ source_reference: Mapping[str, str],
+ citation: Mapping[str, Any],
+) -> bool:
+ citation_file_name = _normalize_source_text(
+ citation.get("file_name")
+ or citation.get("title")
+ )
+ if (
+ not citation_file_name
+ or citation_file_name != source_reference.get("file_name")
+ ):
+ return False
+
+ explicit_label = source_reference.get("location_label")
+ citation_label = _normalize_source_text(citation.get("location_label"))
+ if not citation_label:
+ citation_label = "sheet" if citation.get("sheet_name") else "page"
+ citation_label = citation_label.rstrip("s")
+ if explicit_label != "location" and citation_label != explicit_label:
+ return False
+
+ location_candidates = (
+ citation.get("location_value"),
+ citation.get("sheet_name"),
+ citation.get("page_number"),
+ citation.get("chunk_sequence"),
+ )
+ return any(
+ _location_reference_matches(
+ source_reference.get("location_value", ""),
+ candidate,
+ )
+ for candidate in location_candidates
+ )
+
+
+def _trim_url_candidate(value: Any) -> str:
+ candidate = html.unescape(str(value or "")).strip()
+ while candidate and candidate[-1] in _TRAILING_URL_PUNCTUATION:
+ candidate = candidate[:-1]
+ while candidate.endswith(")") and candidate.count(")") > candidate.count("("):
+ candidate = candidate[:-1]
+ while candidate.endswith("]") and candidate.count("]") > candidate.count("["):
+ candidate = candidate[:-1]
+ return candidate
+
+
+def normalize_http_url(value: Any) -> str:
+ """Normalize an HTTP(S) URL for exact response-to-source matching."""
+ candidate = _trim_url_candidate(value)
+ if not candidate:
+ return ""
+
+ try:
+ parsed = urlsplit(candidate)
+ except ValueError:
+ return ""
+
+ scheme = parsed.scheme.lower()
+ if scheme not in {"http", "https"} or not parsed.hostname:
+ return ""
+
+ hostname = parsed.hostname.lower()
+ try:
+ port = parsed.port
+ except ValueError:
+ return ""
+ if port and not (
+ (scheme == "http" and port == 80)
+ or (scheme == "https" and port == 443)
+ ):
+ hostname = f"{hostname}:{port}"
+
+ path = parsed.path or ""
+ if path == "/":
+ path = ""
+
+ return urlunsplit((scheme, hostname, path, parsed.query, ""))
+
+
+def extract_explicit_web_urls(content: Any) -> List[str]:
+ """Return ordered, unique normalized HTTP(S) URLs from final response text."""
+ normalized_urls: List[str] = []
+ for raw_url in _HTTP_URL_PATTERN.findall(str(content or "")):
+ normalized_url = normalize_http_url(raw_url)
+ if normalized_url and normalized_url not in normalized_urls:
+ normalized_urls.append(normalized_url)
+ return normalized_urls
+
+
+def _get_document_citation_id(citation: Mapping[str, Any]) -> str:
+ return _normalize_reference_id(
+ citation.get("citation_id")
+ or citation.get("id")
+ )
+
+
+def _get_web_citation_url(citation: Mapping[str, Any]) -> str:
+ return normalize_http_url(
+ citation.get("url")
+ or citation.get("href")
+ or citation.get("link")
+ )
+
+
+def build_cited_source_subsets(
+ content: Any,
+ hybrid_citations: Any = None,
+ web_search_citations: Any = None,
+) -> Dict[str, Any]:
+ """Match explicit final-response references to returned source records."""
+ document_citation_ids = set(extract_explicit_document_citation_ids(content))
+ document_source_references = _extract_explicit_document_source_references(
+ content
+ )
+ explicit_web_urls = set(extract_explicit_web_urls(content))
+
+ hybrid_source_records = _as_dict_list(hybrid_citations)
+ exact_document_citations = [
+ citation
+ for citation in hybrid_source_records
+ if _get_document_citation_id(citation) in document_citation_ids
+ ]
+ unmatched_source_references = [
+ source_reference
+ for source_reference in document_source_references
+ if not any(
+ _source_reference_matches_citation(
+ source_reference,
+ exact_citation,
+ )
+ for exact_citation in exact_document_citations
+ )
+ ]
+
+ cited_hybrid_citations: List[Dict[str, Any]] = []
+ seen_document_citations = set()
+ for citation in hybrid_source_records:
+ citation_id = _get_document_citation_id(citation)
+ matched_by_id = bool(
+ citation_id
+ and citation_id in document_citation_ids
+ )
+ matched_by_source = any(
+ _source_reference_matches_citation(source_reference, citation)
+ for source_reference in unmatched_source_references
+ )
+ if not matched_by_id and not matched_by_source:
+ continue
+ if matched_by_id:
+ identity = (
+ "citation_id",
+ citation_id,
+ str(citation.get("document_id") or ""),
+ str(citation.get("chunk_id") or ""),
+ )
+ else:
+ identity = (
+ "source_location",
+ _extract_document_id(citation),
+ _normalize_source_text(
+ citation.get("file_name")
+ or citation.get("title")
+ ),
+ _normalize_source_text(
+ citation.get("location_value")
+ or citation.get("sheet_name")
+ or citation.get("page_number")
+ or citation.get("chunk_sequence")
+ ),
+ )
+ if identity in seen_document_citations:
+ continue
+ seen_document_citations.add(identity)
+ cited_hybrid_citations.append(citation)
+
+ cited_web_search_citations: List[Dict[str, Any]] = []
+ seen_web_urls = set()
+ for citation in _as_dict_list(web_search_citations):
+ normalized_url = _get_web_citation_url(citation)
+ if (
+ not normalized_url
+ or normalized_url not in explicit_web_urls
+ or normalized_url in seen_web_urls
+ ):
+ continue
+ seen_web_urls.add(normalized_url)
+ cited_web_search_citations.append(citation)
+
+ return {
+ "citation_tracking_version": CITATION_TRACKING_VERSION,
+ "cited_hybrid_citations": cited_hybrid_citations,
+ "cited_web_search_citations": cited_web_search_citations,
+ }
+
+
+def _message_has_citation_tracking(message: Mapping[str, Any]) -> bool:
+ tracking_version = message.get("citation_tracking_version")
+ if isinstance(tracking_version, int) and tracking_version >= 1:
+ return True
+ return (
+ "cited_hybrid_citations" in message
+ or "cited_web_search_citations" in message
+ )
+
+
+def get_message_source_citation_buckets(
+ message: Optional[Mapping[str, Any]],
+) -> Dict[str, List[Any]]:
+ """Return complete stored source/execution buckets for audit and Sources UI."""
+ safe_message = message if isinstance(message, Mapping) else {}
+
+ def ensure_list(value: Any) -> List[Any]:
+ if not value:
+ return []
+ return deepcopy(value) if isinstance(value, list) else [deepcopy(value)]
+
+ return {
+ "legacy": ensure_list(safe_message.get("citations")),
+ "hybrid": ensure_list(safe_message.get("hybrid_citations")),
+ "web": ensure_list(safe_message.get("web_search_citations")),
+ "agent": ensure_list(safe_message.get("agent_citations")),
+ }
+
+
+def get_message_reference_citation_buckets(
+ message: Optional[Mapping[str, Any]],
+) -> Dict[str, List[Any]]:
+ """Return exact references, with broad source fallback for legacy messages."""
+ safe_message = message if isinstance(message, Mapping) else {}
+ source_buckets = get_message_source_citation_buckets(safe_message)
+ if not _message_has_citation_tracking(safe_message):
+ return source_buckets
+
+ referenced_agent_citations = []
+ for citation in source_buckets["agent"]:
+ if (
+ isinstance(citation, Mapping)
+ and str(citation.get("function_name") or "").strip().lower()
+ == "azure_ai_foundry_web_search"
+ ):
+ continue
+ referenced_agent_citations.append(deepcopy(citation))
+
+ return {
+ "legacy": source_buckets["legacy"],
+ "hybrid": _as_dict_list(safe_message.get("cited_hybrid_citations")),
+ "web": _as_dict_list(safe_message.get("cited_web_search_citations")),
+ "agent": referenced_agent_citations,
+ }
+
+
+def _extract_document_id(citation: Mapping[str, Any]) -> str:
+ document_id = str(citation.get("document_id") or "").strip()
+ if document_id:
+ return document_id
+
+ citation_id = _get_document_citation_id(citation)
+ if not citation_id:
+ return ""
+ if "_" in citation_id:
+ return citation_id.rsplit("_", 1)[0]
+ return citation_id
+
+
+def _get_document_tags(conversation: Mapping[str, Any]) -> List[Dict[str, Any]]:
+ tags = conversation.get("tags")
+ if not isinstance(tags, list):
+ return []
+ return [
+ deepcopy(tag)
+ for tag in tags
+ if isinstance(tag, dict)
+ and tag.get("category") == "document"
+ and str(tag.get("document_id") or "").strip()
+ ]
+
+
+def _scope_from_citation(citation: Mapping[str, Any]) -> Dict[str, Any]:
+ raw_scope = citation.get("scope")
+ if isinstance(raw_scope, dict):
+ scope_type = str(
+ raw_scope.get("type")
+ or raw_scope.get("scope")
+ or ""
+ ).strip()
+ scope_id = str(
+ raw_scope.get("id")
+ or raw_scope.get("scope_id")
+ or ""
+ ).strip()
+ if scope_type and scope_id:
+ scope = {
+ "type": scope_type,
+ "id": scope_id,
+ }
+ scope_name = str(raw_scope.get("name") or "").strip()
+ if scope_name:
+ scope["name"] = scope_name
+ return scope
+
+ public_workspace_id = str(
+ citation.get("public_workspace_id") or ""
+ ).strip()
+ if public_workspace_id:
+ return {"type": "public", "id": public_workspace_id}
+
+ group_id = str(citation.get("group_id") or "").strip()
+ if group_id:
+ return {"type": "group", "id": group_id}
+
+ user_id = str(citation.get("user_id") or "").strip()
+ if user_id:
+ return {"type": "personal", "id": user_id}
+ return {}
+
+
+def _merge_document_record(
+ target: Dict[str, Any],
+ incoming: Mapping[str, Any],
+) -> Dict[str, Any]:
+ for field_name in _MERGED_DOCUMENT_LIST_FIELDS:
+ target_values = target.setdefault(field_name, [])
+ incoming_values = incoming.get(field_name)
+ if not isinstance(incoming_values, list):
+ continue
+ for value in incoming_values:
+ _append_unique(target_values, value)
+
+ target_locations = target.setdefault("citation_locations", [])
+ incoming_locations = incoming.get("citation_locations")
+ if isinstance(incoming_locations, list):
+ for location in incoming_locations:
+ if isinstance(location, dict):
+ _append_unique_mapping(target_locations, location)
+
+ for field_name in (
+ "title",
+ "file_name",
+ "classification",
+ "scope",
+ ):
+ current_value = target.get(field_name)
+ incoming_value = incoming.get(field_name)
+ if current_value in (None, "", {}, "Unknown Document") and incoming_value not in (
+ None,
+ "",
+ {},
+ ):
+ target[field_name] = deepcopy(incoming_value)
+ return target
+
+
+def build_used_documents(
+ cited_hybrid_citations: Any,
+ source_document_tags: Any = None,
+) -> List[Dict[str, Any]]:
+ """Collapse exact cited chunks into document-level used-document records."""
+ tag_by_document_id = {
+ str(tag.get("document_id") or "").strip(): tag
+ for tag in _as_dict_list(source_document_tags)
+ if str(tag.get("document_id") or "").strip()
+ }
+ documents_by_id: Dict[str, Dict[str, Any]] = {}
+
+ for citation in _as_dict_list(cited_hybrid_citations):
+ document_id = _extract_document_id(citation)
+ if not document_id:
+ continue
+
+ document = documents_by_id.get(document_id)
+ if document is None:
+ source_tag = tag_by_document_id.get(document_id)
+ document = deepcopy(source_tag) if source_tag else {
+ "category": "document",
+ "document_id": document_id,
+ }
+ document["category"] = "document"
+ document["document_id"] = document_id
+ document["chunk_ids"] = []
+ document["citation_ids"] = []
+ document["page_numbers"] = []
+ document["sheet_names"] = []
+ document["citation_locations"] = []
+ documents_by_id[document_id] = document
+
+ citation_id = _get_document_citation_id(citation)
+ _append_unique(document["citation_ids"], citation_id)
+ _append_unique(
+ document["chunk_ids"],
+ citation_id or citation.get("chunk_id"),
+ )
+ _append_unique(document["page_numbers"], citation.get("page_number"))
+ _append_unique(document["sheet_names"], citation.get("sheet_name"))
+
+ location = {
+ key: deepcopy(citation.get(key))
+ for key in (
+ "citation_id",
+ "chunk_id",
+ "page_number",
+ "sheet_name",
+ "location_label",
+ "location_value",
+ )
+ if citation.get(key) not in (None, "")
+ }
+ _append_unique_mapping(document["citation_locations"], location)
+
+ incoming_metadata = {
+ "title": citation.get("title") or citation.get("file_name"),
+ "file_name": citation.get("file_name") or citation.get("title"),
+ "classification": (
+ citation.get("classification")
+ or citation.get("document_classification")
+ ),
+ "scope": _scope_from_citation(citation),
+ }
+ _merge_document_record(document, incoming_metadata)
+
+ return list(documents_by_id.values())
+
+
+def merge_used_documents(
+ existing_documents: Any,
+ incoming_documents: Any,
+) -> List[Dict[str, Any]]:
+ """Merge exact used-document aggregates without losing cited locations."""
+ merged_by_id: Dict[str, Dict[str, Any]] = {}
+ for document in [
+ *_as_dict_list(existing_documents),
+ *_as_dict_list(incoming_documents),
+ ]:
+ document_id = str(document.get("document_id") or "").strip()
+ if not document_id:
+ continue
+ if document_id not in merged_by_id:
+ merged_by_id[document_id] = deepcopy(document)
+ continue
+ _merge_document_record(merged_by_id[document_id], document)
+ return list(merged_by_id.values())
+
+
+def initialize_conversation_used_document_tracking(
+ conversation: Dict[str, Any],
+) -> bool:
+ """Snapshot legacy document tags before the first strictly tracked turn."""
+ if not isinstance(conversation, dict):
+ return False
+ tracking_version = conversation.get("used_documents_tracking_version")
+ if isinstance(tracking_version, int) and tracking_version >= 1:
+ return False
+
+ conversation["used_documents_tracking_version"] = (
+ USED_DOCUMENTS_TRACKING_VERSION
+ )
+ conversation["legacy_used_documents"] = _get_document_tags(conversation)
+ conversation["used_documents"] = []
+ return True
+
+
+def merge_cited_documents_into_conversation(
+ conversation: Dict[str, Any],
+ cited_hybrid_citations: Any,
+) -> List[Dict[str, Any]]:
+ """Merge one finalized assistant response into exact conversation usage."""
+ initialize_conversation_used_document_tracking(conversation)
+ source_document_tags = _get_document_tags(conversation)
+ incoming_documents = build_used_documents(
+ cited_hybrid_citations,
+ source_document_tags=source_document_tags,
+ )
+ conversation["used_documents"] = merge_used_documents(
+ conversation.get("used_documents"),
+ incoming_documents,
+ )
+ return deepcopy(conversation["used_documents"])
+
+
+def _message_is_active_assistant(message: Mapping[str, Any]) -> bool:
+ if message.get("role") != "assistant":
+ return False
+ metadata = message.get("metadata")
+ metadata = metadata if isinstance(metadata, Mapping) else {}
+ if metadata.get("is_deleted") is True:
+ return False
+ if metadata.get("is_generated_chat_artifact") is True:
+ return False
+ thread_info = metadata.get("thread_info")
+ thread_info = thread_info if isinstance(thread_info, Mapping) else {}
+ return thread_info.get("active_thread") is not False
+
+
+def rebuild_conversation_used_documents(
+ conversation: Dict[str, Any],
+ messages: Iterable[Mapping[str, Any]],
+ rebuild_legacy: bool = False,
+) -> List[Dict[str, Any]]:
+ """Rebuild exact usage after a retry, switch, deletion, or fork mutation."""
+ initialize_conversation_used_document_tracking(conversation)
+ cited_hybrid_citations: List[Dict[str, Any]] = []
+ legacy_hybrid_citations: List[Dict[str, Any]] = []
+ for message in messages or []:
+ if not isinstance(message, Mapping) or not _message_is_active_assistant(message):
+ continue
+ if _message_has_citation_tracking(message):
+ cited_hybrid_citations.extend(
+ _as_dict_list(message.get("cited_hybrid_citations"))
+ )
+ elif rebuild_legacy:
+ legacy_hybrid_citations.extend(
+ _as_dict_list(message.get("hybrid_citations"))
+ )
+
+ source_document_tags = _get_document_tags(conversation)
+ conversation["used_documents"] = build_used_documents(
+ cited_hybrid_citations,
+ source_document_tags=source_document_tags,
+ )
+ if rebuild_legacy:
+ conversation["legacy_used_documents"] = build_used_documents(
+ legacy_hybrid_citations,
+ source_document_tags=source_document_tags,
+ )
+ return deepcopy(conversation["used_documents"])
diff --git a/application/single_app/functions_collaboration.py b/application/single_app/functions_collaboration.py
index d434cb62a..75e9e78c0 100644
--- a/application/single_app/functions_collaboration.py
+++ b/application/single_app/functions_collaboration.py
@@ -40,6 +40,11 @@
log_conversation_deletion,
)
from functions_appinsights import log_event
+from functions_citation_tracking import (
+ initialize_conversation_used_document_tracking,
+ merge_cited_documents_into_conversation,
+ rebuild_conversation_used_documents,
+)
from functions_conversation_cache import bump_conversation_cache_version, invalidate_conversation_cache_for_item
from functions_documents import sync_chat_upload_workspace_document_sharing_for_collaboration
from functions_group import (
@@ -64,6 +69,21 @@
COLLABORATION_EVENT_PUBLISHERS = []
+CITATION_TRACKING_CONVERSATION_FIELDS = (
+ 'used_documents_tracking_version',
+ 'legacy_used_documents',
+ 'used_documents',
+)
+
+
+def _copy_citation_tracking_conversation_fields(target, source):
+ if not isinstance(target, dict) or not isinstance(source, dict):
+ return target
+ for field_name in CITATION_TRACKING_CONVERSATION_FIELDS:
+ if field_name in source:
+ target[field_name] = deepcopy(source.get(field_name))
+ return target
+
def register_collaboration_event_publisher(event_publisher):
if callable(event_publisher) and event_publisher not in COLLABORATION_EVENT_PUBLISHERS:
@@ -355,7 +375,7 @@ def serialize_collaboration_message(message_doc):
message_doc.get('id'),
) or serialized_content
- return {
+ payload = {
'id': message_doc.get('id'),
'conversation_id': message_doc.get('conversation_id'),
'role': serialized_role,
@@ -382,6 +402,14 @@ def serialize_collaboration_message(message_doc):
'extracted_text': message_doc.get('extracted_text'),
'vision_analysis': message_doc.get('vision_analysis'),
}
+ for field_name in (
+ 'citation_tracking_version',
+ 'cited_hybrid_citations',
+ 'cited_web_search_citations',
+ ):
+ if field_name in message_doc:
+ payload[field_name] = deepcopy(message_doc.get(field_name))
+ return payload
def _get_collaboration_source_message(message_doc):
@@ -729,6 +757,15 @@ def build_collaboration_message_metadata_payload(message_doc, conversation_doc):
'extracted_text': message_doc.get('extracted_text') or payload.get('extracted_text'),
'vision_analysis': message_doc.get('vision_analysis') or payload.get('vision_analysis'),
})
+ for field_name in (
+ 'citation_tracking_version',
+ 'cited_hybrid_citations',
+ 'cited_web_search_citations',
+ ):
+ if field_name in message_doc:
+ payload[field_name] = deepcopy(message_doc.get(field_name))
+ elif field_name in (source_message_doc or {}):
+ payload[field_name] = deepcopy(source_message_doc.get(field_name))
payload['metadata'] = merged_metadata
return payload
@@ -820,6 +857,9 @@ def serialize_collaboration_conversation(conversation_doc, current_user_id, user
'is_hidden': bool((user_state or {}).get('is_hidden', False)),
'classification': list(conversation_doc.get('classification', []) or []),
'tags': list(conversation_doc.get('tags', []) or []),
+ 'used_documents_tracking_version': conversation_doc.get('used_documents_tracking_version'),
+ 'legacy_used_documents': deepcopy(list(conversation_doc.get('legacy_used_documents', []) or [])),
+ 'used_documents': deepcopy(list(conversation_doc.get('used_documents', []) or [])),
'strict': bool(conversation_doc.get('strict', False)),
'summary': conversation_doc.get('summary'),
'has_unread_assistant_response': False,
@@ -959,6 +999,10 @@ def ensure_personal_collaboration_for_legacy_conversation(source_conversation_id
collaboration_conversation_doc['source_conversation_id'] = source_conversation_id
collaboration_conversation_doc['classification'] = list(source_conversation_doc.get('classification', []) or [])
collaboration_conversation_doc['tags'] = list(source_conversation_doc.get('tags', []) or [])
+ _copy_citation_tracking_conversation_fields(
+ collaboration_conversation_doc,
+ source_conversation_doc,
+ )
collaboration_conversation_doc['strict'] = bool(source_conversation_doc.get('strict', False))
collaboration_conversation_doc['summary'] = source_conversation_doc.get('summary')
@@ -1145,6 +1189,10 @@ def ensure_group_collaboration_for_legacy_conversation(source_conversation_id, o
collaboration_conversation_doc['classification'] = list(source_conversation_doc.get('classification', []) or [])
collaboration_conversation_doc['tags'] = list(source_conversation_doc.get('tags', []) or [])
+ _copy_citation_tracking_conversation_fields(
+ collaboration_conversation_doc,
+ source_conversation_doc,
+ )
collaboration_conversation_doc['strict'] = bool(source_conversation_doc.get('strict', False))
collaboration_conversation_doc['summary'] = source_conversation_doc.get('summary')
collaboration_conversation_doc['legacy_source_conversation_id'] = source_conversation_id
@@ -1749,6 +1797,19 @@ def _save_collaboration_message_doc(conversation_doc, message_doc):
cosmos_collaboration_messages_container.upsert_item(message_doc)
+ if (
+ str(message_doc.get('role') or '').strip().lower() == 'assistant'
+ and (
+ message_doc.get('citation_tracking_version') is not None
+ or 'cited_hybrid_citations' in message_doc
+ )
+ ):
+ initialize_conversation_used_document_tracking(conversation_doc)
+ merge_cited_documents_into_conversation(
+ conversation_doc,
+ message_doc.get('cited_hybrid_citations'),
+ )
+
conversation_doc['last_message_at'] = message_doc.get('timestamp')
conversation_doc['last_message_preview'] = (
message_doc.get('metadata', {}).get('last_message_preview', '')
@@ -1814,6 +1875,11 @@ def sync_collaboration_conversation_metadata_from_source(conversation_doc, sourc
'classification': deepcopy(list(source_conversation_doc.get('classification', []) or [])),
'summary': deepcopy(source_conversation_doc.get('summary')),
}
+ for field_name in CITATION_TRACKING_CONVERSATION_FIELDS:
+ if field_name in source_conversation_doc:
+ metadata_fields[field_name] = deepcopy(
+ source_conversation_doc.get(field_name)
+ )
updated = False
for field_name, field_value in metadata_fields.items():
@@ -1868,6 +1934,10 @@ def ensure_collaboration_source_conversation(conversation_doc, current_user):
'collaboration_conversation_id': (conversation_doc or {}).get('id'),
'is_hidden': True,
}
+ _copy_citation_tracking_conversation_fields(
+ source_conversation_doc,
+ conversation_doc,
+ )
source_updated = True
else:
synchronized_values = {
@@ -1884,6 +1954,11 @@ def ensure_collaboration_source_conversation(conversation_doc, current_user):
'collaboration_conversation_id': (conversation_doc or {}).get('id'),
'is_hidden': True,
}
+ for field_name in CITATION_TRACKING_CONVERSATION_FIELDS:
+ if field_name in (conversation_doc or {}):
+ synchronized_values[field_name] = deepcopy(
+ conversation_doc.get(field_name)
+ )
for field_name, field_value in synchronized_values.items():
if source_conversation_doc.get(field_name) != field_value:
source_conversation_doc[field_name] = field_value
@@ -1952,6 +2027,7 @@ def _refresh_collaboration_conversation_message_summary(conversation_doc):
raise ValueError('conversation_id is required')
remaining_messages = list_collaboration_messages(conversation_id)
+ rebuild_conversation_used_documents(conversation_doc, remaining_messages)
conversation_doc['message_count'] = len(remaining_messages)
if remaining_messages:
diff --git a/application/single_app/functions_simplechat_operations.py b/application/single_app/functions_simplechat_operations.py
index 03da3a6e5..2aff415c3 100644
--- a/application/single_app/functions_simplechat_operations.py
+++ b/application/single_app/functions_simplechat_operations.py
@@ -35,6 +35,7 @@
log_workflow_creation,
)
from functions_appinsights import log_event
+from functions_citation_tracking import rebuild_conversation_used_documents
from functions_authentication import (
get_current_user_info,
get_graph_endpoint,
@@ -93,6 +94,8 @@
PERSONAL_FORK_CONVERSATION_FIELDS = (
"context",
"tags",
+ "used_documents_tracking_version",
+ "legacy_used_documents",
"strict",
"classification",
"scope_locked",
@@ -808,6 +811,11 @@ def fork_personal_conversation_for_user(
normalized_user_id,
destination_chat_type,
)
+ rebuild_conversation_used_documents(
+ fork_conversation,
+ fork_documents,
+ rebuild_legacy=True,
+ )
written_message_ids = []
created_blob_targets: List[Tuple[str, str]] = []
diff --git a/application/single_app/functions_workflow_runner.py b/application/single_app/functions_workflow_runner.py
index 83ccfd1ad..2c36b99f0 100644
--- a/application/single_app/functions_workflow_runner.py
+++ b/application/single_app/functions_workflow_runner.py
@@ -39,6 +39,7 @@
)
from config import (
SECRET_KEY,
+ TABULAR_EXTENSIONS,
VERSION,
cognitive_services_scope,
cosmos_conversations_container,
@@ -54,6 +55,12 @@
build_conversation_context_system_message,
serialize_conversation_context_snapshot,
)
+from functions_citation_tracking import (
+ build_cited_source_subsets,
+ initialize_conversation_used_document_tracking,
+ merge_cited_documents_into_conversation,
+ resolve_citation_location,
+)
from functions_activity_logging import log_conversation_creation, log_token_usage, log_workflow_run
from functions_appinsights import log_event
from functions_assistant_table_exports import (
@@ -4470,8 +4477,26 @@ def _mirror_assistant_message_to_personal_conversation(
previous_thread_id,
),
}
+ for field_name in (
+ 'citation_tracking_version',
+ 'cited_hybrid_citations',
+ 'cited_web_search_citations',
+ ):
+ if field_name in source_assistant_doc:
+ if field_name == 'citation_tracking_version':
+ mirrored_assistant_doc[field_name] = source_assistant_doc.get(field_name)
+ else:
+ mirrored_assistant_doc[field_name] = list(
+ source_assistant_doc.get(field_name) or []
+ )
cosmos_messages_container.upsert_item(mirrored_assistant_doc)
+ if 'citation_tracking_version' in mirrored_assistant_doc:
+ initialize_conversation_used_document_tracking(conversation_doc)
+ merge_cited_documents_into_conversation(
+ conversation_doc,
+ mirrored_assistant_doc.get('cited_hybrid_citations'),
+ )
conversation_doc['last_updated'] = timestamp
conversation_doc['has_unread_assistant_response'] = True
conversation_doc['last_unread_assistant_message_id'] = mirrored_message_id
@@ -5833,6 +5858,12 @@ def _create_assistant_message(conversation, workflow, result, trigger_source, ru
if generated_file_output.get('output_format') == 'csv':
generated_tabular_outputs.append(generated_file_output)
web_search_citations = list(result.get('web_search_citations') or [])
+ hybrid_citations = list(result.get('hybrid_citations') or [])
+ citation_tracking = build_cited_source_subsets(
+ result.get('reply', ''),
+ hybrid_citations=hybrid_citations,
+ web_search_citations=web_search_citations,
+ )
source_review_metadata = result.get('source_review') if isinstance(result.get('source_review'), dict) else {}
url_access_metadata = result.get('url_access') if isinstance(result.get('url_access'), dict) else {}
prepared_agent_citations = _persist_agent_citation_artifacts(
@@ -5851,10 +5882,11 @@ def _create_assistant_message(conversation, workflow, result, trigger_source, ru
'content': result.get('reply', ''),
'timestamp': timestamp,
'model_deployment_name': result.get('model_deployment_name'),
- 'augmented': bool(result.get('augmented') or result.get('hybrid_citations')),
- 'hybrid_citations': list(result.get('hybrid_citations') or []),
+ 'augmented': bool(result.get('augmented') or hybrid_citations),
+ 'hybrid_citations': hybrid_citations,
'agent_citations': prepared_agent_citations,
'web_search_citations': web_search_citations,
+ **citation_tracking,
'agent_display_name': result.get('agent_display_name'),
'agent_name': result.get('agent_name'),
'workspace_type': workspace_type,
@@ -5927,6 +5959,11 @@ def _create_assistant_message(conversation, workflow, result, trigger_source, ru
conversation['has_unread_assistant_response'] = True
conversation['last_unread_assistant_message_id'] = assistant_message_id
conversation['last_unread_assistant_at'] = timestamp
+ initialize_conversation_used_document_tracking(conversation)
+ merge_cited_documents_into_conversation(
+ conversation,
+ citation_tracking['cited_hybrid_citations'],
+ )
cosmos_conversations_container.upsert_item(conversation)
return assistant_doc
@@ -6418,12 +6455,25 @@ def _build_workflow_search_citation(result):
document_id = str(result.get('document_id') or '').strip()
if not document_id:
document_id = '_'.join(str(citation_id).split('_')[:-1]) if '_' in str(citation_id) else str(citation_id)
+ sheet_name = result.get('sheet_name')
+ page_number = result.get('page_number') or result.get('chunk_sequence') or 1
+ file_name = result.get('file_name') or result.get('title') or 'Unknown document'
+ _, extension = os.path.splitext(str(file_name).strip().lower())
+ location_label, location_value = resolve_citation_location(
+ page_number=page_number,
+ chunk_text=result.get('chunk_text'),
+ sheet_name=sheet_name,
+ is_tabular=extension.lstrip('.') in TABULAR_EXTENSIONS,
+ )
return {
- 'file_name': result.get('file_name') or result.get('title') or 'Unknown document',
+ 'file_name': file_name,
'document_id': document_id,
'citation_id': citation_id,
- 'page_number': result.get('page_number'),
+ 'page_number': page_number,
+ 'sheet_name': sheet_name,
+ 'location_label': location_label,
+ 'location_value': location_value,
'chunk_id': result.get('chunk_id'),
'chunk_sequence': result.get('chunk_sequence'),
'score': result.get('score'),
@@ -6445,9 +6495,18 @@ def _format_workflow_search_results(results):
file_name = str(result.get('file_name') or result.get('title') or 'Unknown document').strip() or 'Unknown document'
page_number = result.get('page_number') or result.get('chunk_sequence') or 1
+ sheet_name = result.get('sheet_name')
+ _, extension = os.path.splitext(file_name.lower())
+ location_label, location_value = resolve_citation_location(
+ page_number=page_number,
+ chunk_text=chunk_text,
+ sheet_name=sheet_name,
+ is_tabular=extension.lstrip('.') in TABULAR_EXTENSIONS,
+ )
citation_id = result.get('id') or result.get('chunk_id') or f'workflow-search-{index}'
result_lines.append(
- f'[{index}] {file_name}, page {page_number}, citation #{citation_id}\n{chunk_text}'
+ f'[{index}] {chunk_text}\n'
+ f'(Source: {file_name}, {location_label}: {location_value}) [#{citation_id}]'
)
citations.append(_build_workflow_search_citation(result))
@@ -6468,7 +6527,8 @@ def _build_workflow_search_prompt(task_prompt, search_context):
prompt_sections = [
'[Workflow document search context]\n'
'Use the bounded native-engine evidence below as grounding for the workflow task. '
- 'When the evidence is insufficient or source coverage is partial, say what is missing instead of guessing.'
+ 'When the evidence is insufficient or source coverage is partial, say what is missing instead of guessing. '
+ 'Preserve the exact (Source: ...) [#citation-id] reference for every excerpt used in the response.'
]
if retrieved_content:
prompt_sections.append(
diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py
index d675d7292..bad4c20dc 100644
--- a/application/single_app/route_backend_chats.py
+++ b/application/single_app/route_backend_chats.py
@@ -179,6 +179,12 @@
inject_conversation_context_message,
serialize_conversation_context_snapshot,
)
+from functions_citation_tracking import (
+ build_cited_source_subsets,
+ initialize_conversation_used_document_tracking,
+ merge_cited_documents_into_conversation,
+ resolve_citation_location,
+)
from functions_conversation_metadata import collect_conversation_metadata, update_conversation_with_metadata
from functions_conversation_unread import mark_conversation_unread
from functions_image_messages import build_image_message_documents, decode_image_content
@@ -4298,7 +4304,7 @@ def build_tabular_fallback_system_message(tabular_filenames_str, execution_mode=
def build_search_augmentation_system_prompt(retrieved_content):
"""Build the retrieval augmentation prompt without blocking later tool-backed results."""
- return f"""You are an AI assistant. Use the following retrieved document excerpts to answer the user's question. Cite sources using the format (Source: filename, Page: page number).
+ return f"""You are an AI assistant. Use the following retrieved document excerpts to answer the user's question. Cite sources using the format (Source: filename, Page: page number) [#citation-id]. Copy the exact bracketed citation ID shown after the supporting excerpt.
Retrieved Excerpts:
{retrieved_content}
@@ -4308,7 +4314,7 @@ def build_search_augmentation_system_prompt(retrieved_content):
Example
User: What is the policy on double dipping?
- Assistant: The policy prohibits entities from using federal funds received through one program to apply for additional funds through another program, commonly known as 'double dipping' (Source: PolicyDocument.pdf, Page: 12)
+ Assistant: The policy prohibits entities from using federal funds received through one program to apply for additional funds through another program, commonly known as 'double dipping' (Source: PolicyDocument.pdf, Page: 12) [#example-citation-id]
"""
@@ -13046,17 +13052,12 @@ def is_tabular_filename(filename):
def get_citation_location(file_name, page_number=None, chunk_text=None, sheet_name=None):
"""Return a display label/value pair for a citation location."""
- if sheet_name:
- return 'Sheet', str(sheet_name)
-
- normalized_chunk_text = (chunk_text or '').strip()
- if is_tabular_filename(file_name) and (
- normalized_chunk_text.startswith('Tabular workbook:')
- or normalized_chunk_text.startswith('Tabular data file:')
- ):
- return 'Location', 'Workbook Schema'
-
- return 'Page', str(page_number or 1)
+ return resolve_citation_location(
+ page_number=page_number,
+ chunk_text=chunk_text,
+ sheet_name=sheet_name,
+ is_tabular=is_tabular_filename(file_name),
+ )
def get_document_container_for_scope(document_scope):
@@ -14220,6 +14221,9 @@ def normalize_terminal_chat_payload(payload):
'augmented': payload.get('augmented', False),
'hybrid_citations': payload.get('hybrid_citations', []),
'web_search_citations': payload.get('web_search_citations', []),
+ 'citation_tracking_version': payload.get('citation_tracking_version'),
+ 'cited_hybrid_citations': payload.get('cited_hybrid_citations', []),
+ 'cited_web_search_citations': payload.get('cited_web_search_citations', []),
'agent_citations': payload.get('agent_citations', []),
'agent_display_name': payload.get('agent_display_name'),
'agent_name': payload.get('agent_name'),
@@ -15400,6 +15404,11 @@ def execute_document_action_chat_request(
generated_analysis_artifacts=document_generated_analysis_artifacts,
generated_tabular_outputs=document_generated_tabular_outputs,
)
+ document_action_citation_tracking = build_cited_source_subsets(
+ document_action_reply_content,
+ hybrid_citations=hybrid_citations_list,
+ web_search_citations=[],
+ )
document_action_capability_usage = _build_capability_usage_metadata(
workspace_search_used=True,
workspace_search_result_count=len(hybrid_citations_list or []),
@@ -15419,6 +15428,7 @@ def execute_document_action_chat_request(
'augmented': False,
'hybrid_citations': hybrid_citations_list,
'web_search_citations': [],
+ **document_action_citation_tracking,
'hybridsearch_query': None,
'agent_citations': prepared_agent_citations,
'model_deployment_name': execution_result.get('model_deployment_name'),
@@ -15521,6 +15531,7 @@ def execute_document_action_chat_request(
conversation_item['last_updated'] = datetime.utcnow().isoformat()
conversation_item['chat_type'] = data.get('chat_type') or conversation_item.get('chat_type') or 'new'
+ initialize_conversation_used_document_tracking(conversation_item)
try:
conversation_item = collect_conversation_metadata(
@@ -15548,6 +15559,10 @@ def execute_document_action_chat_request(
except Exception as exc:
debug_print(f'[CHAT_DOCUMENT_ANALYSIS] Conversation metadata update failed: {exc}')
+ merge_cited_documents_into_conversation(
+ conversation_item,
+ document_action_citation_tracking['cited_hybrid_citations'],
+ )
cosmos_conversations_container.upsert_item(conversation_item)
invalidate_conversation_cache_for_item(conversation_item, reason="document_action_chat_completed")
debug_print(
@@ -15580,6 +15595,7 @@ def execute_document_action_chat_request(
'augmented': False,
'hybrid_citations': hybrid_citations_list,
'web_search_citations': [],
+ **document_action_citation_tracking,
'agent_citations': prepared_agent_citations,
'reload_messages': False,
'kernel_fallback_notice': None,
@@ -17610,6 +17626,9 @@ def result_requires_message_reload(result: Any) -> bool:
"document_id": source_doc.get("document_id"),
"citation_id": source_doc.get("citation_id"), # Seems like a useful identifier
"page_number": source_doc.get("page_number"),
+ "sheet_name": source_doc.get("sheet_name"),
+ "location_label": source_doc.get("location_label"),
+ "location_value": source_doc.get("location_value"),
"chunk_id": source_doc.get("chunk_id"), # Specific chunk identifier
"chunk_sequence": source_doc.get("chunk_sequence"), # Order within document/group
"score": source_doc.get("score"), # Relevance score from search
@@ -19807,6 +19826,11 @@ def gpt_error(e):
deep_research_query_count=_deep_research_query_count(deep_research_query_plan, deep_research_web_search_runs),
)
agent_runtime_metadata = _build_foundry_runtime_metadata(selected_agent) if selected_agent else {}
+ citation_tracking = build_cited_source_subsets(
+ ai_message,
+ hybrid_citations=hybrid_citations_list,
+ web_search_citations=web_search_citations_list,
+ )
assistant_doc = make_json_serializable({
'id': assistant_message_id,
@@ -19817,6 +19841,7 @@ def gpt_error(e):
'augmented': bool(system_messages_for_augmentation),
'hybrid_citations': hybrid_citations_list, # <--- SIMPLIFIED: Directly use the list
'web_search_citations': web_search_citations_list,
+ **citation_tracking,
'hybridsearch_query': search_query if search_results else None, # Log query when any bounded document retrieval produced results
'agent_citations': prepared_agent_citations,
'model_deployment_name': actual_model_used,
@@ -19928,6 +19953,7 @@ def gpt_error(e):
# Update conversation's last_updated timestamp one last time
conversation_item['last_updated'] = datetime.utcnow().isoformat()
+ initialize_conversation_used_document_tracking(conversation_item)
# Collect comprehensive conversation metadata
try:
@@ -19971,6 +19997,10 @@ def gpt_error(e):
debug_print(f"Error collecting conversation metadata: {e}")
# Continue even if metadata collection fails
+ merge_cited_documents_into_conversation(
+ conversation_item,
+ citation_tracking['cited_hybrid_citations'],
+ )
# Add any other final updates to conversation_item if needed (like classifications if not done earlier)
cosmos_conversations_container.upsert_item(conversation_item)
invalidate_conversation_cache_for_item(conversation_item, reason="chat_completed")
@@ -19999,6 +20029,7 @@ def gpt_error(e):
'augmented': bool(system_messages_for_augmentation),
'hybrid_citations': hybrid_citations_list,
'web_search_citations': web_search_citations_list,
+ **citation_tracking,
'source_review': compact_source_review_result_for_metadata(source_review_result),
'deep_research': deep_research_result,
'agent_citations': prepared_agent_citations,
@@ -20134,6 +20165,9 @@ def normalize_legacy_chat_payload(payload):
'augmented': payload.get('augmented', False),
'hybrid_citations': payload.get('hybrid_citations', []),
'web_search_citations': payload.get('web_search_citations', []),
+ 'citation_tracking_version': payload.get('citation_tracking_version'),
+ 'cited_hybrid_citations': payload.get('cited_hybrid_citations', []),
+ 'cited_web_search_citations': payload.get('cited_web_search_citations', []),
'agent_citations': payload.get('agent_citations', []),
'agent_display_name': payload.get('agent_display_name'),
'agent_name': payload.get('agent_name'),
@@ -20634,6 +20668,39 @@ def build_streaming_capability_usage():
deep_research_query_count=_deep_research_query_count(deep_research_query_plan, deep_research_web_search_runs),
)
+ def collect_stream_response_conversation_metadata():
+ nonlocal conversation_item
+ source_continuity_refs = None
+ if (
+ is_mixed_source_conversation_continuity_enabled(settings)
+ and mixed_source_manifest
+ ):
+ source_continuity_refs = _build_mixed_source_continuity_refs(
+ mixed_source_manifest,
+ mixed_source_evidence_envelopes,
+ effective_mixed_source_selection_mode,
+ )
+ conversation_item = collect_conversation_metadata(
+ user_message=user_message,
+ conversation_id=conversation_id,
+ user_id=user_id,
+ active_group_id=effective_active_group_id,
+ active_group_ids=effective_active_group_ids,
+ document_scope=effective_document_scope,
+ selected_document_id=effective_selected_document_id,
+ model_deployment=final_model_used if use_agent_streaming else gpt_model,
+ hybrid_search_enabled=mixed_source_document_context_active,
+ image_gen_enabled=False,
+ selected_documents=combined_documents if combined_documents else None,
+ selected_agent=agent_name_used if use_agent_streaming else None,
+ selected_agent_details=selected_agent_metadata if use_agent_streaming else None,
+ search_results=search_results if search_results else None,
+ conversation_item=conversation_item,
+ active_public_workspace_id=effective_active_public_workspace_id,
+ active_public_workspace_ids=effective_active_public_workspace_ids,
+ source_continuity_refs=source_continuity_refs,
+ )
+
# Initialize GPT client (simplified version)
gpt_model = ""
gpt_client = None
@@ -21876,6 +21943,9 @@ def record_and_publish_streaming_thought(thought_payload):
"document_id": document_id,
"citation_id": citation_id,
"page_number": page_number,
+ "sheet_name": sheet_name,
+ "location_label": location_label,
+ "location_value": location_value,
"chunk_id": chunk_id,
"chunk_sequence": chunk_sequence,
"score": score,
@@ -22934,6 +23004,7 @@ def finalize_cancelled_stream_response():
cancel_reason = stream_session.get_cancel_reason() if stream_session else 'user_requested'
partial_content = accumulated_content.strip()
message_persisted = False
+ partial_citation_tracking = {}
cancel_metadata = {
'incomplete': True,
'canceled': True,
@@ -22960,6 +23031,11 @@ def finalize_cancelled_stream_response():
if partial_content:
assistant_timestamp = datetime.utcnow().isoformat()
+ partial_citation_tracking = build_cited_source_subsets(
+ partial_content,
+ hybrid_citations=hybrid_citations_list,
+ web_search_citations=web_search_citations_list,
+ )
prepared_agent_citations = persist_agent_citation_artifacts(
conversation_id=conversation_id,
assistant_message_id=assistant_message_id,
@@ -22980,6 +23056,7 @@ def finalize_cancelled_stream_response():
'augmented': bool(system_messages_for_augmentation),
'hybrid_citations': hybrid_citations_list,
'web_search_citations': web_search_citations_list,
+ **partial_citation_tracking,
'hybridsearch_query': search_query if hybrid_search_enabled and search_results else None,
'agent_citations': prepared_agent_citations,
'model_deployment_name': final_model_used if use_agent_streaming else gpt_model,
@@ -23016,6 +23093,24 @@ def finalize_cancelled_stream_response():
})
cosmos_messages_container.upsert_item(assistant_doc)
conversation_item['last_updated'] = datetime.utcnow().isoformat()
+ initialize_conversation_used_document_tracking(conversation_item)
+ try:
+ collect_stream_response_conversation_metadata()
+ except Exception as metadata_error:
+ log_event(
+ '[STREAMING] Failed to collect canceled response metadata',
+ extra={
+ 'conversation_id': conversation_id,
+ 'message_id': assistant_message_id,
+ 'error_type': type(metadata_error).__name__,
+ },
+ level=logging.WARNING,
+ exceptionTraceback=True,
+ )
+ merge_cited_documents_into_conversation(
+ conversation_item,
+ partial_citation_tracking['cited_hybrid_citations'],
+ )
cosmos_conversations_container.upsert_item(conversation_item)
invalidate_conversation_cache_for_item(conversation_item, reason="chat_stream_stopped")
message_persisted = True
@@ -23043,6 +23138,7 @@ def finalize_cancelled_stream_response():
'augmented': bool(system_messages_for_augmentation),
'hybrid_citations': hybrid_citations_list,
'web_search_citations': web_search_citations_list,
+ **partial_citation_tracking,
'agent_citations': agent_citations_list,
'model_deployment_name': final_model_used if use_agent_streaming else gpt_model,
'model_icon': gpt_model_icon,
@@ -23666,6 +23762,11 @@ def finalize_cancelled_agent_stream_response():
generated_tabular_outputs=generated_tabular_outputs_list,
)
agent_runtime_metadata = _build_foundry_runtime_metadata(selected_agent) if use_agent_streaming else {}
+ stream_citation_tracking = build_cited_source_subsets(
+ accumulated_content,
+ hybrid_citations=hybrid_citations_list,
+ web_search_citations=web_search_citations_list,
+ )
assistant_doc = make_json_serializable({
'id': assistant_message_id,
@@ -23676,6 +23777,7 @@ def finalize_cancelled_agent_stream_response():
'augmented': bool(system_messages_for_augmentation),
'hybrid_citations': hybrid_citations_list,
'web_search_citations': web_search_citations_list,
+ **stream_citation_tracking,
'hybridsearch_query': search_query if search_results else None,
'agent_citations': prepared_agent_citations,
'model_deployment_name': final_model_used if use_agent_streaming else gpt_model,
@@ -23778,6 +23880,7 @@ def finalize_cancelled_agent_stream_response():
# Update conversation
conversation_item['last_updated'] = datetime.utcnow().isoformat()
+ initialize_conversation_used_document_tracking(conversation_item)
try:
user_message_doc = cosmos_messages_container.read_item(
@@ -23795,39 +23898,14 @@ def finalize_cancelled_agent_stream_response():
debug_print(f"Warning: Could not update streaming user message metadata: {e}")
try:
- source_continuity_refs = None
- if (
- is_mixed_source_conversation_continuity_enabled(settings)
- and mixed_source_manifest
- ):
- source_continuity_refs = _build_mixed_source_continuity_refs(
- mixed_source_manifest,
- mixed_source_evidence_envelopes,
- effective_mixed_source_selection_mode,
- )
- conversation_item = collect_conversation_metadata(
- user_message=user_message,
- conversation_id=conversation_id,
- user_id=user_id,
- active_group_id=effective_active_group_id,
- active_group_ids=effective_active_group_ids,
- document_scope=effective_document_scope,
- selected_document_id=effective_selected_document_id,
- model_deployment=final_model_used if use_agent_streaming else gpt_model,
- hybrid_search_enabled=mixed_source_document_context_active,
- image_gen_enabled=False,
- selected_documents=combined_documents if combined_documents else None,
- selected_agent=agent_name_used if use_agent_streaming else None,
- selected_agent_details=selected_agent_metadata if use_agent_streaming else None,
- search_results=search_results if search_results else None,
- conversation_item=conversation_item,
- active_public_workspace_id=effective_active_public_workspace_id,
- active_public_workspace_ids=effective_active_public_workspace_ids,
- source_continuity_refs=source_continuity_refs,
- )
+ collect_stream_response_conversation_metadata()
except Exception as e:
debug_print(f"Error collecting conversation metadata: {e}")
+ merge_cited_documents_into_conversation(
+ conversation_item,
+ stream_citation_tracking['cited_hybrid_citations'],
+ )
if is_personal_chat_conversation(conversation_item):
conversation_item = mark_conversation_unread(
conversation_item,
@@ -23871,6 +23949,7 @@ def finalize_cancelled_agent_stream_response():
'augmented': bool(system_messages_for_augmentation),
'hybrid_citations': hybrid_citations_list,
'web_search_citations': web_search_citations_list,
+ **stream_citation_tracking,
'source_review': compact_source_review_result_for_metadata(source_review_result),
'deep_research': deep_research_result,
'agent_citations': prepared_agent_citations,
@@ -23942,10 +24021,18 @@ def finalize_cancelled_agent_stream_response():
debug_print(f"Error during streaming: {error_msg}")
# Save partial response if we have content
+ interrupted_message_persisted = False
+ interrupted_citation_tracking = {}
+ interrupted_agent_citations = []
if accumulated_content:
current_assistant_thread_id = str(uuid.uuid4())
assistant_timestamp = datetime.utcnow().isoformat()
- prepared_agent_citations = persist_agent_citation_artifacts(
+ interrupted_citation_tracking = build_cited_source_subsets(
+ accumulated_content,
+ hybrid_citations=hybrid_citations_list,
+ web_search_citations=web_search_citations_list,
+ )
+ interrupted_agent_citations = persist_agent_citation_artifacts(
conversation_id=conversation_id,
assistant_message_id=assistant_message_id,
agent_citations=agent_citations_list,
@@ -23966,8 +24053,9 @@ def finalize_cancelled_agent_stream_response():
'augmented': bool(system_messages_for_augmentation),
'hybrid_citations': hybrid_citations_list,
'web_search_citations': web_search_citations_list,
+ **interrupted_citation_tracking,
'hybridsearch_query': search_query if hybrid_search_enabled and search_results else None,
- 'agent_citations': prepared_agent_citations,
+ 'agent_citations': interrupted_agent_citations,
'model_deployment_name': final_model_used if use_agent_streaming else gpt_model,
'agent_display_name': agent_display_name_used if use_agent_streaming else None,
'agent_name': agent_name_used if use_agent_streaming else None,
@@ -23991,12 +24079,68 @@ def finalize_cancelled_agent_stream_response():
})
try:
cosmos_messages_container.upsert_item(assistant_doc)
- except Exception as ex:
- pass
+ interrupted_message_persisted = True
+ conversation_item['last_updated'] = assistant_timestamp
+ initialize_conversation_used_document_tracking(
+ conversation_item
+ )
+ try:
+ collect_stream_response_conversation_metadata()
+ except Exception as metadata_error:
+ log_event(
+ '[STREAMING] Failed to collect interrupted response metadata',
+ extra={
+ 'conversation_id': conversation_id,
+ 'message_id': assistant_message_id,
+ 'error_type': type(
+ metadata_error
+ ).__name__,
+ },
+ level=logging.WARNING,
+ exceptionTraceback=True,
+ )
+ merge_cited_documents_into_conversation(
+ conversation_item,
+ interrupted_citation_tracking[
+ 'cited_hybrid_citations'
+ ],
+ )
+ cosmos_conversations_container.upsert_item(
+ conversation_item
+ )
+ invalidate_conversation_cache_for_item(
+ conversation_item,
+ reason="chat_stream_interrupted",
+ )
+ except Exception as persistence_error:
+ log_event(
+ '[STREAMING] Failed to persist interrupted response metadata',
+ extra={
+ 'conversation_id': conversation_id,
+ 'message_id': assistant_message_id,
+ 'error_type': type(
+ persistence_error
+ ).__name__,
+ },
+ level=logging.WARNING,
+ exceptionTraceback=True,
+ )
yield build_stream_error_event(
CLIENT_SAFE_STREAM_ERROR_MESSAGE,
partial_content=accumulated_content,
+ conversation_id=conversation_id,
+ user_message_id=user_message_id,
+ message_id=(
+ assistant_message_id
+ if interrupted_message_persisted
+ else None
+ ),
+ message_persisted=interrupted_message_persisted,
+ hybrid_citations=hybrid_citations_list,
+ web_search_citations=web_search_citations_list,
+ agent_citations=interrupted_agent_citations,
+ **interrupted_citation_tracking,
)
except Exception as e:
diff --git a/application/single_app/route_backend_collaboration.py b/application/single_app/route_backend_collaboration.py
index bf1f20af3..6b45af1bc 100644
--- a/application/single_app/route_backend_collaboration.py
+++ b/application/single_app/route_backend_collaboration.py
@@ -1507,7 +1507,13 @@ def transform_event_block(event_block):
except json.JSONDecodeError:
return normalized_event_block + '\n\n'
- if stream_payload.get('error'):
+ if (
+ stream_payload.get('error')
+ and not (
+ stream_payload.get('message_persisted')
+ and stream_payload.get('message_id')
+ )
+ ):
return _serialize_stream_error(
stream_payload.get('error'),
partial_content=stream_payload.get('partial_content'),
@@ -1515,6 +1521,8 @@ def transform_event_block(event_block):
message_persisted=True,
conversation_id=conversation_id,
)
+ if stream_payload.get('error'):
+ stream_payload['done'] = True
if stream_payload.get('type') == USER_MESSAGE_PERSISTED_EVENT_TYPE:
return None
@@ -1627,6 +1635,7 @@ def transform_event_block(event_block):
**stream_payload,
'conversation_id': conversation_id,
'conversation_title': serialized_final_conversation.get('title'),
+ 'conversation_kind': serialized_final_conversation.get('conversation_kind'),
'chat_type': serialized_final_conversation.get('chat_type'),
'classification': serialized_final_conversation.get('classification', []),
'context': serialized_final_conversation.get('context', []),
@@ -1638,12 +1647,15 @@ def transform_event_block(event_block):
'augmented': serialized_assistant_message.get('augmented', False),
'hybrid_citations': serialized_assistant_message.get('hybrid_citations', []),
'web_search_citations': serialized_assistant_message.get('web_search_citations', []),
+ 'citation_tracking_version': serialized_assistant_message.get('citation_tracking_version'),
+ 'cited_hybrid_citations': serialized_assistant_message.get('cited_hybrid_citations', []),
+ 'cited_web_search_citations': serialized_assistant_message.get('cited_web_search_citations', []),
'agent_citations': serialized_assistant_message.get('agent_citations', []),
'agent_display_name': serialized_assistant_message.get('agent_display_name'),
'agent_name': serialized_assistant_message.get('agent_name'),
'full_content': serialized_assistant_message.get('content') if serialized_assistant_message.get('role') != 'image' else stream_payload.get('full_content', ''),
'image_url': serialized_assistant_message.get('content') if serialized_assistant_message.get('role') == 'image' else stream_payload.get('image_url'),
- 'reload_messages': False,
+ 'reload_messages': bool(stream_payload.get('error')),
}
return f'data: {json.dumps(make_json_serializable(transformed_payload))}\n\n'
diff --git a/application/single_app/route_backend_conversation_export.py b/application/single_app/route_backend_conversation_export.py
index 68b2b855d..56af5c671 100644
--- a/application/single_app/route_backend_conversation_export.py
+++ b/application/single_app/route_backend_conversation_export.py
@@ -30,6 +30,10 @@
is_collaboration_conversation,
list_collaboration_messages,
)
+from functions_citation_tracking import (
+ get_message_reference_citation_buckets,
+ get_message_source_citation_buckets,
+)
from functions_conversation_metadata import update_conversation_with_metadata
from functions_debug import debug_print
from functions_group import get_group_model_endpoints, get_user_groups
@@ -565,8 +569,9 @@ def _sanitize_message(
) -> Dict[str, Any]:
role = message.get('role', '')
content = message.get('content', '')
- raw_citation_buckets = _collect_raw_citation_buckets(message)
- normalized_citations = _normalize_citations(raw_citation_buckets)
+ reference_citation_buckets = _collect_raw_citation_buckets(message)
+ source_citation_buckets = _collect_source_citation_buckets(message)
+ normalized_citations = _normalize_citations(reference_citation_buckets)
citation_counts = _build_citation_counts(normalized_citations)
details = _curate_message_details(message, citation_counts, len(thoughts))
@@ -585,10 +590,21 @@ def _sanitize_message(
'citations': normalized_citations,
'citation_counts': citation_counts,
'thoughts': thoughts,
- 'legacy_citations': raw_citation_buckets['legacy'],
- 'hybrid_citations': raw_citation_buckets['hybrid'],
- 'web_search_citations': raw_citation_buckets['web'],
- 'agent_citations': raw_citation_buckets['agent']
+ 'citation_tracking_version': message.get('citation_tracking_version'),
+ 'cited_hybrid_citations': (
+ list(message.get('cited_hybrid_citations'))
+ if isinstance(message.get('cited_hybrid_citations'), list)
+ else []
+ ),
+ 'cited_web_search_citations': (
+ list(message.get('cited_web_search_citations'))
+ if isinstance(message.get('cited_web_search_citations'), list)
+ else []
+ ),
+ 'legacy_citations': source_citation_buckets['legacy'],
+ 'hybrid_citations': source_citation_buckets['hybrid'],
+ 'web_search_citations': source_citation_buckets['web'],
+ 'agent_citations': source_citation_buckets['agent']
}
@@ -604,17 +620,13 @@ def _sanitize_thought(thought: Dict[str, Any]) -> Dict[str, Any]:
def _collect_raw_citation_buckets(message: Dict[str, Any]) -> Dict[str, List[Any]]:
- def ensure_list(value: Any) -> List[Any]:
- if not value:
- return []
- return value if isinstance(value, list) else [value]
+ """Return citation records that are valid references for this message."""
+ return get_message_reference_citation_buckets(message)
- return {
- 'legacy': ensure_list(message.get('citations')),
- 'hybrid': ensure_list(message.get('hybrid_citations')),
- 'web': ensure_list(message.get('web_search_citations')),
- 'agent': ensure_list(message.get('agent_citations'))
- }
+
+def _collect_source_citation_buckets(message: Dict[str, Any]) -> Dict[str, List[Any]]:
+ """Return complete source and execution records for JSON audit output."""
+ return get_message_source_citation_buckets(message)
def _normalize_citations(raw_citation_buckets: Dict[str, List[Any]]) -> List[Dict[str, Any]]:
@@ -627,7 +639,11 @@ def _normalize_citations(raw_citation_buckets: Dict[str, List[Any]]) -> List[Dic
'label': _build_document_citation_label(citation),
'file_name': citation.get('file_name'),
'title': citation.get('title') or citation.get('file_name'),
+ 'document_id': citation.get('document_id'),
'page_number': citation.get('page_number'),
+ 'sheet_name': citation.get('sheet_name'),
+ 'location_label': citation.get('location_label'),
+ 'location_value': citation.get('location_value'),
'citation_id': citation.get('citation_id'),
'chunk_id': citation.get('chunk_id'),
'metadata_type': citation.get('metadata_type'),
@@ -668,6 +684,9 @@ def _normalize_citations(raw_citation_buckets: Dict[str, List[Any]]) -> List[Dic
'tool_name': citation.get('tool_name'),
'function_name': citation.get('function_name'),
'plugin_name': citation.get('plugin_name'),
+ 'function_arguments': citation.get('function_arguments'),
+ 'function_result': citation.get('function_result'),
+ 'error_message': citation.get('error_message'),
'success': citation.get('success'),
'timestamp': citation.get('timestamp')
})
@@ -701,12 +720,20 @@ def _normalize_citations(raw_citation_buckets: Dict[str, List[Any]]) -> List[Dic
def _build_document_citation_label(citation: Dict[str, Any]) -> str:
file_name = citation.get('file_name') or citation.get('title') or 'Document source'
metadata_type = citation.get('metadata_type')
- page_number = citation.get('page_number')
+ location_label = (
+ citation.get('location_label')
+ or ('Sheet' if citation.get('sheet_name') else 'Page')
+ )
+ location_value = (
+ citation.get('location_value')
+ or citation.get('sheet_name')
+ or citation.get('page_number')
+ )
if metadata_type:
return f"{file_name} — {metadata_type.replace('_', ' ').title()}"
- if page_number not in (None, ''):
- return f"{file_name} — Page {page_number}"
+ if location_value not in (None, ''):
+ return f"{file_name} — {location_label}: {location_value}"
return file_name
@@ -1624,7 +1651,7 @@ def _conversation_to_markdown(entry: Dict[str, Any]) -> str:
def _append_citations_markdown(lines: List[str], message: Dict[str, Any]):
document_citations = [citation for citation in message.get('citations', []) if citation.get('citation_type') == 'document']
web_citations = [citation for citation in message.get('citations', []) if citation.get('citation_type') == 'web']
- agent_citations = message.get('agent_citations', []) or []
+ agent_citations = [citation for citation in message.get('citations', []) if citation.get('citation_type') == 'agent_tool']
legacy_citations = [citation for citation in message.get('citations', []) if citation.get('citation_type') == 'legacy']
if not any([document_citations, web_citations, agent_citations, legacy_citations]):
@@ -1632,13 +1659,16 @@ def _append_citations_markdown(lines: List[str], message: Dict[str, Any]):
return
if document_citations:
- lines.append('#### Document Sources')
+ lines.append('#### Document References')
lines.append('')
for index, citation in enumerate(document_citations, start=1):
lines.append(f"{index}. **{citation.get('label', 'Document source')}**")
detail_mapping = _remove_empty_values({
'citation_id': citation.get('citation_id'),
'page_number': citation.get('page_number'),
+ 'sheet_name': citation.get('sheet_name'),
+ 'location_label': citation.get('location_label'),
+ 'location_value': citation.get('location_value'),
'classification': citation.get('classification'),
'score': citation.get('score'),
'metadata_type': citation.get('metadata_type')
@@ -1650,7 +1680,7 @@ def _append_citations_markdown(lines: List[str], message: Dict[str, Any]):
lines.append('')
if web_citations:
- lines.append('#### Web Sources')
+ lines.append('#### Web References')
lines.append('')
for index, citation in enumerate(web_citations, start=1):
title = citation.get('title') or citation.get('label') or 'Web source'
@@ -5565,7 +5595,7 @@ def _append_html_citations(parts: List[str], message: Dict[str, Any]):
legacy_citations = [c for c in citations if c.get('citation_type') == 'legacy']
if doc_citations:
- parts.append('
Document Sources
')
+ parts.append('Document References
')
parts.append('')
for citation in doc_citations:
parts.append(
@@ -5574,6 +5604,9 @@ def _append_html_citations(parts: List[str], message: Dict[str, Any]):
detail_items = _remove_empty_values({
'citation_id': citation.get('citation_id'),
'page_number': citation.get('page_number'),
+ 'sheet_name': citation.get('sheet_name'),
+ 'location_label': citation.get('location_label'),
+ 'location_value': citation.get('location_value'),
'classification': citation.get('classification'),
'score': citation.get('score'),
'metadata_type': citation.get('metadata_type')
@@ -5591,7 +5624,7 @@ def _append_html_citations(parts: List[str], message: Dict[str, Any]):
parts.append('
')
if web_citations:
- parts.append('Web Sources
')
+ parts.append('Web References
')
parts.append('')
for citation in web_citations:
title = _escape_html(
diff --git a/application/single_app/route_backend_conversations.py b/application/single_app/route_backend_conversations.py
index b286afc04..d00b78534 100644
--- a/application/single_app/route_backend_conversations.py
+++ b/application/single_app/route_backend_conversations.py
@@ -31,6 +31,7 @@
tag_conversation_feed_source,
)
from functions_conversation_metadata import get_conversation_metadata, update_conversation_with_metadata
+from functions_citation_tracking import rebuild_conversation_used_documents
from functions_conversation_unread import clear_conversation_unread, normalize_conversation_unread_state
from functions_conversation_cache import (
build_conversation_cache_key,
@@ -822,6 +823,63 @@ def _authorize_personal_conversation_read(user_id, conversation_id):
return conversation_item
+def _rebuild_authorized_personal_conversation_used_documents(
+ user_id,
+ conversation_id,
+):
+ """Persist exact used documents after an authorized message mutation."""
+ try:
+ conversation_item = _authorize_personal_conversation_read(
+ user_id,
+ conversation_id,
+ )
+ messages = list(cosmos_messages_container.query_items(
+ query=(
+ "SELECT * FROM c "
+ "WHERE c.conversation_id = @conversation_id"
+ ),
+ parameters=[
+ {
+ "name": "@conversation_id",
+ "value": conversation_id,
+ },
+ ],
+ partition_key=conversation_id,
+ ))
+ rebuild_conversation_used_documents(conversation_item, messages)
+ conversation_item.pop('used_documents_rebuild_required', None)
+ conversation_item['last_updated'] = datetime.utcnow().isoformat()
+ cosmos_conversations_container.upsert_item(conversation_item)
+ return conversation_item
+ except Exception as rebuild_error:
+ if 'conversation_item' in locals() and isinstance(conversation_item, dict):
+ conversation_item['used_documents_rebuild_required'] = True
+ try:
+ cosmos_conversations_container.upsert_item(conversation_item)
+ except Exception as marker_error:
+ log_event(
+ "[CONVERSATION_METADATA] Failed to mark used documents for rebuild",
+ extra={
+ "conversation_id": conversation_id,
+ "user_id": user_id,
+ "error_type": type(marker_error).__name__,
+ },
+ level=logging.WARNING,
+ exceptionTraceback=True,
+ )
+ log_event(
+ "[CONVERSATION_METADATA] Failed to rebuild used documents after message mutation",
+ extra={
+ "conversation_id": conversation_id,
+ "user_id": user_id,
+ "error_type": type(rebuild_error).__name__,
+ },
+ level=logging.WARNING,
+ exceptionTraceback=True,
+ )
+ return None
+
+
def _invalidate_conversation_cache_after_message_mutation(conversation_id, user_id, reason):
"""Invalidate conversation caches after message-level changes without failing the caller."""
try:
@@ -1800,6 +1858,16 @@ def get_conversation_metadata_api(conversation_id):
# Ensure that the conversation belongs to the current user
if conversation_item.get('user_id') != user_id:
return jsonify({'error': 'Forbidden'}), 403
+
+ if conversation_item.get('used_documents_rebuild_required') is True:
+ rebuilt_conversation = (
+ _rebuild_authorized_personal_conversation_used_documents(
+ user_id,
+ conversation_id,
+ )
+ )
+ if rebuilt_conversation:
+ conversation_item = rebuilt_conversation
_, updated = normalize_chat_type(conversation_item)
if updated:
@@ -1828,6 +1896,9 @@ def get_conversation_metadata_api(conversation_id):
"classification": conversation_item.get('classification', []),
"context": conversation_item.get('context', []),
"tags": conversation_item.get('tags', []),
+ "used_documents_tracking_version": conversation_item.get('used_documents_tracking_version'),
+ "legacy_used_documents": conversation_item.get('legacy_used_documents', []),
+ "used_documents": conversation_item.get('used_documents', []),
"strict": conversation_item.get('strict', False),
"is_pinned": conversation_item.get('is_pinned', False),
"is_hidden": conversation_item.get('is_hidden', False),
@@ -2609,6 +2680,10 @@ def delete_message(message_id):
deleted_message_ids.append(msg_id)
+ _rebuild_authorized_personal_conversation_used_documents(
+ user_id,
+ conversation_id,
+ )
_invalidate_conversation_cache_after_message_mutation(
conversation_id,
user_id,
@@ -2787,6 +2862,10 @@ def retry_message(message_id):
}
cosmos_messages_container.upsert_item(new_user_message)
+ _rebuild_authorized_personal_conversation_used_documents(
+ user_id,
+ conversation_id,
+ )
_invalidate_conversation_cache_after_message_mutation(
conversation_id,
user_id,
@@ -3006,6 +3085,10 @@ def edit_message(message_id):
}
cosmos_messages_container.upsert_item(new_user_message)
+ _rebuild_authorized_personal_conversation_used_documents(
+ user_id,
+ conversation_id,
+ )
_invalidate_conversation_cache_after_message_mutation(
conversation_id,
user_id,
@@ -3170,6 +3253,10 @@ def switch_attempt(message_id):
msg['metadata']['thread_info']['active_thread'] = (msg_attempt == target_attempt)
cosmos_messages_container.upsert_item(msg)
+ _rebuild_authorized_personal_conversation_used_documents(
+ user_id,
+ conversation_id,
+ )
_invalidate_conversation_cache_after_message_mutation(
conversation_id,
user_id,
diff --git a/application/single_app/static/js/chat/chat-citations.js b/application/single_app/static/js/chat/chat-citations.js
index c51927233..edd6ac293 100644
--- a/application/single_app/static/js/chat/chat-citations.js
+++ b/application/single_app/static/js/chat/chat-citations.js
@@ -42,7 +42,7 @@ export function parseDocIdAndPage(citationId) {
export function parseCitations(message) {
// ... (keep existing implementation)
- const citationRegex = /\(Source:\s*([^,]+),\s*(Page(?:s)?|Sheet(?:s)?|Location):\s*([^)]+)\)\s*((?:\[#.*?\]\s*)+)/gi;
+ const citationRegex = /\(Source:\s*((?:(?!\(Source:).)+?),\s*(Page(?:s)?|Sheet(?:s)?|Location):\s*((?:(?!\(Source:).)+?)\)\s*((?:\[#.*?\]\s*)+)/gi;
let result = message.replace(citationRegex, (whole, filename, locationLabel, locations, bracketSection) => {
const trimmedFilename = filename.trim();
@@ -80,7 +80,12 @@ export function parseCitations(message) {
}
const normalizedLocationLabel = locationLabel.toLowerCase();
- const locationTokens = locations.split(/,/).map(tok => tok.trim());
+ const locationTokens = (
+ normalizedLocationLabel === 'sheet'
+ || normalizedLocationLabel === 'location'
+ )
+ ? [locations.trim()]
+ : locations.split(/,/).map(tok => tok.trim());
const linkedTokens = locationTokens.map((token, index) => {
if (!normalizedLocationLabel.startsWith('page')) {
const ref = orderedRefs[index] || orderedRefs[0];
diff --git a/application/single_app/static/js/chat/chat-collaboration.js b/application/single_app/static/js/chat/chat-collaboration.js
index 1dff3c9f1..d2ae11ac5 100644
--- a/application/single_app/static/js/chat/chat-collaboration.js
+++ b/application/single_app/static/js/chat/chat-collaboration.js
@@ -1074,6 +1074,24 @@ function handleConversationEvent(eventEnvelope = {}) {
const decoratedMessage = decorateReplyMessage(payload.message);
cacheCollaborationMessage(payload.message);
+ if (String(payload.message.role || '').trim().toLowerCase() === 'assistant') {
+ const conversationId = (
+ eventEnvelope.conversation_id
+ || payload.message.conversation_id
+ || payload.conversation?.id
+ || ''
+ );
+ window.dispatchEvent(new CustomEvent(
+ 'chat:conversation-documents-refresh',
+ {
+ detail: {
+ conversationId,
+ autoOpen: false,
+ conversationKind: 'collaborative',
+ },
+ }
+ ));
+ }
if (reconcilePendingCollaborativeUserMessage(payload.message)) {
const messageKind = String(
payload.message.message_kind || payload.message.metadata?.message_kind || ''
@@ -1112,6 +1130,16 @@ function handleConversationEvent(eventEnvelope = {}) {
if (eventEnvelope.event_type === 'collaboration.message.deleted' && payload.message_id) {
removeCollaborationMessage(payload.message_id);
+ window.dispatchEvent(new CustomEvent(
+ 'chat:conversation-documents-refresh',
+ {
+ detail: {
+ conversationId: eventEnvelope.conversation_id || '',
+ autoOpen: false,
+ conversationKind: 'collaborative',
+ },
+ }
+ ));
if (payload.deleted_by_user_id && payload.deleted_by_user_id !== getCurrentUserId()) {
showToast('A shared message was deleted.', 'info');
}
@@ -1937,6 +1965,7 @@ window.chatCollaboration = {
deactivateConversation,
fetchCollaborationConversationList,
fetchConversationMetadata,
+ loadConversationMessages,
getPendingMessageContext,
handleComposerBlur,
handleComposerInput,
diff --git a/application/single_app/static/js/chat/chat-conversation-contents.js b/application/single_app/static/js/chat/chat-conversation-contents.js
index 97d1a9016..3034d119e 100644
--- a/application/single_app/static/js/chat/chat-conversation-contents.js
+++ b/application/single_app/static/js/chat/chat-conversation-contents.js
@@ -3,7 +3,10 @@
import {
extractPageNumbers,
fetchConversationMetadata,
- getConversationDocumentTags,
+ getConversationExactUsedDocuments,
+ getConversationLegacyUsedDocuments,
+ getConversationUsedDocuments,
+ hasConversationUsedDocumentTracking,
} from "./chat-conversation-details.js";
import { isColorLight } from "./chat-utils.js";
@@ -47,6 +50,9 @@ let restoreFocusAfterClose = true;
let metadataRequestToken = 0;
let documentLoadState = "idle";
let activeDocumentConversationId = "";
+let documentUsageTrackingAvailable = false;
+let exactDocumentUsageAvailable = false;
+let legacyDocumentFallbackAvailable = false;
const autoOpenedDocumentConversationIds = new Set();
export function normalizeConversationContentsLabel(value, fallbackLabel = "User message") {
@@ -178,9 +184,15 @@ function syncDrawerModeControls() {
drawerTitle.textContent = contentsActive ? "Conversation contents" : "Used documents";
}
if (drawerSubtitle) {
+ let documentsSubtitle = "Documents associated with this historical conversation";
+ if (documentUsageTrackingAvailable && exactDocumentUsageAvailable) {
+ documentsSubtitle = legacyDocumentFallbackAvailable
+ ? "Cited documents and historical associations"
+ : "Documents cited in active responses";
+ }
drawerSubtitle.textContent = contentsActive
? "Jump to a user message"
- : "Documents cited in this conversation";
+ : documentsSubtitle;
}
contentsPanel?.classList.toggle("d-none", !contentsActive);
@@ -428,9 +440,62 @@ function createDocumentMetaLine(iconName, text) {
return line;
}
+function getDocumentCitationLocationLines(doc) {
+ const groupedLocations = new Map();
+ const appendLocation = (label, value) => {
+ const normalizedLabel = String(label || "Location").trim() || "Location";
+ const normalizedValue = String(value ?? "").trim();
+ if (!normalizedValue) {
+ return;
+ }
+ const values = groupedLocations.get(normalizedLabel) || [];
+ if (!values.includes(normalizedValue)) {
+ values.push(normalizedValue);
+ groupedLocations.set(normalizedLabel, values);
+ }
+ };
+
+ const citationLocations = Array.isArray(doc?.citation_locations)
+ ? doc.citation_locations
+ : [];
+ citationLocations.forEach(location => {
+ if (!location || typeof location !== "object") {
+ return;
+ }
+ const label = location.location_label
+ || (location.sheet_name ? "Sheet" : "Page");
+ const value = location.location_value
+ || location.sheet_name
+ || location.page_number;
+ appendLocation(label, value);
+ });
+
+ if (groupedLocations.size === 0) {
+ (Array.isArray(doc?.sheet_names) ? doc.sheet_names : [])
+ .forEach(value => appendLocation("Sheet", value));
+ (Array.isArray(doc?.page_numbers) ? doc.page_numbers : [])
+ .forEach(value => appendLocation("Page", value));
+ }
+
+ if (groupedLocations.size === 0) {
+ const chunkIds = Array.isArray(doc?.chunk_ids) ? doc.chunk_ids : [];
+ extractPageNumbers(chunkIds).forEach(value => appendLocation("Page", value));
+ }
+
+ return Array.from(groupedLocations.entries()).map(([label, values]) => {
+ const normalizedLabel = label.toLowerCase();
+ const displayLabel = normalizedLabel === "page"
+ ? "Pages"
+ : normalizedLabel === "sheet"
+ ? "Sheets"
+ : normalizedLabel === "location"
+ ? "Locations"
+ : label;
+ return `${displayLabel}: ${values.join(", ")}`;
+ });
+}
+
function createDocumentEntry(doc) {
- const chunkIds = Array.isArray(doc?.chunk_ids) ? doc.chunk_ids : [];
- const chunkPages = extractPageNumbers(chunkIds);
const documentId = String(doc?.document_id || "Unknown Document");
const documentTitle = String(doc?.title || doc?.file_name || documentId);
const documentFileName = String(doc?.file_name || "");
@@ -460,9 +525,9 @@ function createDocumentEntry(doc) {
) {
entry.appendChild(createDocumentMetaLine("file-earmark", documentFileName));
}
- if (chunkPages.length > 0) {
- entry.appendChild(createDocumentMetaLine("file-earmark-text", `Pages: ${chunkPages.join(", ")}`));
- }
+ getDocumentCitationLocationLines(doc).forEach(locationLine => {
+ entry.appendChild(createDocumentMetaLine("file-earmark-text", locationLine));
+ });
entry.appendChild(createDocumentMetaLine(
getScopeIcon(scopeType),
`${scopeType} scope: ${scopeName}`
@@ -503,6 +568,7 @@ function renderConversationDocuments(documents) {
async function refreshConversationDocuments(options = {}) {
const conversationId = String(options.conversationId || getCurrentConversationId()).trim();
+ const conversationKind = String(options.conversationKind || "").trim();
const requestToken = ++metadataRequestToken;
const autoOpen = Boolean(options.autoOpen);
const conversationChanged = conversationId !== activeDocumentConversationId;
@@ -510,6 +576,9 @@ async function refreshConversationDocuments(options = {}) {
activeDocumentConversationId = conversationId;
if (!conversationId) {
documentLoadState = "idle";
+ documentUsageTrackingAvailable = false;
+ exactDocumentUsageAvailable = false;
+ legacyDocumentFallbackAvailable = false;
renderConversationDocuments([]);
return;
}
@@ -527,18 +596,26 @@ async function refreshConversationDocuments(options = {}) {
}
try {
- const metadata = await fetchConversationMetadata(conversationId);
+ const metadata = await fetchConversationMetadata(
+ conversationId,
+ { conversationKind }
+ );
if (requestToken !== metadataRequestToken || conversationId !== activeDocumentConversationId) {
return;
}
documentLoadState = "ready";
- const documents = getConversationDocumentTags(metadata);
+ documentUsageTrackingAvailable = hasConversationUsedDocumentTracking(metadata);
+ const exactDocuments = getConversationExactUsedDocuments(metadata);
+ const legacyDocuments = getConversationLegacyUsedDocuments(metadata);
+ exactDocumentUsageAvailable = exactDocuments.length > 0;
+ legacyDocumentFallbackAvailable = legacyDocuments.length > 0;
+ const documents = getConversationUsedDocuments(metadata);
renderConversationDocuments(documents);
if (
autoOpen
- && documents.length > 0
+ && exactDocuments.length > 0
&& conversationId === getCurrentConversationId()
&& !autoOpenedDocumentConversationIds.has(conversationId)
) {
@@ -551,6 +628,9 @@ async function refreshConversationDocuments(options = {}) {
}
documentLoadState = "error";
+ documentUsageTrackingAvailable = false;
+ exactDocumentUsageAvailable = false;
+ legacyDocumentFallbackAvailable = false;
documentEntries = [];
documentsList?.replaceChildren();
documentsEmptyState?.classList.add("d-none");
@@ -658,6 +738,7 @@ function initializeConversationContents() {
void refreshConversationDocuments({
conversationId: event.detail?.conversationId || "",
autoOpen: false,
+ conversationKind: event.detail?.conversationKind || "",
});
});
window.addEventListener("chat:conversation-documents-refresh", event => {
@@ -668,6 +749,7 @@ function initializeConversationContents() {
void refreshConversationDocuments({
conversationId,
autoOpen: Boolean(event.detail?.autoOpen),
+ conversationKind: event.detail?.conversationKind || "",
});
});
diff --git a/application/single_app/static/js/chat/chat-conversation-details.js b/application/single_app/static/js/chat/chat-conversation-details.js
index da3f195b3..3101e0791 100644
--- a/application/single_app/static/js/chat/chat-conversation-details.js
+++ b/application/single_app/static/js/chat/chat-conversation-details.js
@@ -41,14 +41,17 @@ function getConversationMetadataDomItem(conversationId) {
|| document.querySelector(`.sidebar-conversation-item[data-conversation-id="${escapedConversationId}"]`);
}
-export async function fetchConversationMetadata(conversationId) {
+export async function fetchConversationMetadata(conversationId, options = {}) {
const normalizedConversationId = String(conversationId || '').trim();
if (!normalizedConversationId) {
throw new Error('Conversation ID is required');
}
const conversationItem = getConversationMetadataDomItem(normalizedConversationId);
- const isCollaborativeConversation = conversationItem?.dataset?.conversationKind === 'collaborative';
+ const isCollaborativeConversation = (
+ options.conversationKind === 'collaborative'
+ || conversationItem?.dataset?.conversationKind === 'collaborative'
+ );
if (isCollaborativeConversation && window.chatCollaboration?.fetchConversationMetadata) {
return window.chatCollaboration.fetchConversationMetadata(normalizedConversationId);
@@ -68,6 +71,57 @@ export function getConversationDocumentTags(metadata = {}) {
return tags.filter(tag => tag?.category === 'document');
}
+export function hasConversationUsedDocumentTracking(metadata = {}) {
+ return Number(metadata.used_documents_tracking_version || 0) >= 1;
+}
+
+export function getConversationExactUsedDocuments(metadata = {}) {
+ if (!hasConversationUsedDocumentTracking(metadata)) {
+ return [];
+ }
+ return Array.isArray(metadata.used_documents) ? metadata.used_documents : [];
+}
+
+export function getConversationLegacyUsedDocuments(metadata = {}) {
+ if (!hasConversationUsedDocumentTracking(metadata)) {
+ return getConversationDocumentTags(metadata);
+ }
+ return Array.isArray(metadata.legacy_used_documents)
+ ? metadata.legacy_used_documents
+ : [];
+}
+
+function mergeConversationDocumentCollections(...collections) {
+ const documentsById = new Map();
+ collections.forEach(collection => {
+ if (!Array.isArray(collection)) {
+ return;
+ }
+ collection.forEach(doc => {
+ const documentId = String(doc?.document_id || '').trim();
+ if (!documentId) {
+ return;
+ }
+ const existingDocument = documentsById.get(documentId) || {};
+ documentsById.set(documentId, {
+ ...existingDocument,
+ ...doc,
+ });
+ });
+ });
+ return Array.from(documentsById.values());
+}
+
+export function getConversationUsedDocuments(metadata = {}) {
+ if (!hasConversationUsedDocumentTracking(metadata)) {
+ return getConversationDocumentTags(metadata);
+ }
+ return mergeConversationDocumentCollections(
+ getConversationLegacyUsedDocuments(metadata),
+ getConversationExactUsedDocuments(metadata),
+ );
+}
+
export function hideConversationDetails() {
const { modal } = getConversationDetailsModalElements();
const modalInstance = getConversationDetailsModalInstance();
@@ -355,6 +409,14 @@ function renderConversationMetadata(metadata, conversationId) {
}
const documentTags = getConversationDocumentTags(metadata);
+ const exactUsedDocumentIds = new Set(
+ getConversationExactUsedDocuments(metadata)
+ .map(doc => String(doc?.document_id || '').trim())
+ .filter(Boolean)
+ );
+ const documentUsageNote = hasConversationUsedDocumentTracking(metadata)
+ ? 'All returned source documents are listed. Cited marks documents referenced by active responses.'
+ : 'All associated source documents are listed. Exact citation use was not tracked for this historical conversation.';
// Documents Section
if (documentTags.length > 0) {
@@ -362,10 +424,11 @@ function renderConversationMetadata(metadata, conversationId) {
- ${renderDocumentsSection(documentTags)}
+
${documentUsageNote}
+ ${renderDocumentsSection(documentTags, exactUsedDocumentIds)}
@@ -748,7 +811,7 @@ function renderModelsAndAgentsSection(models, agents) {
/**
* Render documents section
*/
-function renderDocumentsSection(documents) {
+function renderDocumentsSection(documents, citedDocumentIds = new Set()) {
let html = '';
documents.forEach(doc => {
@@ -761,6 +824,10 @@ function renderDocumentsSection(documents) {
const safeDocumentTitle = escapeHtml(documentTitle);
const safeScopeName = escapeHtml(scopeName);
const safeScopeType = escapeHtml(doc.scope?.type || 'Unknown');
+ const isCited = citedDocumentIds.has(String(doc.document_id || '').trim());
+ const citedBadgeHtml = isCited
+ ? 'Cited'
+ : '';
// Format document classification with custom colors
const allCategories = window.classification_categories || [];
@@ -778,7 +845,10 @@ function renderDocumentsSection(documents) {
${safeDocumentTitle}
- ${classificationHtml}
+
+ ${citedBadgeHtml}
+ ${classificationHtml}
+
diff --git a/application/single_app/static/js/chat/chat-edit.js b/application/single_app/static/js/chat/chat-edit.js
index f8d109a77..f4173cc6f 100644
--- a/application/single_app/static/js/chat/chat-edit.js
+++ b/application/single_app/static/js/chat/chat-edit.js
@@ -147,6 +147,15 @@ window.executeMessageEdit = function() {
console.log(' retry_thread_id:', data.chat_request.retry_thread_id);
console.log(' retry_thread_attempt:', data.chat_request.retry_thread_attempt);
console.log(' Full chat_request:', data.chat_request);
+ window.dispatchEvent(new CustomEvent(
+ 'chat:conversation-documents-refresh',
+ {
+ detail: {
+ conversationId: data.chat_request.conversation_id,
+ autoOpen: false,
+ },
+ }
+ ));
sendMessageWithStreaming(
data.chat_request,
diff --git a/application/single_app/static/js/chat/chat-messages.js b/application/single_app/static/js/chat/chat-messages.js
index ae90c097e..da6310144 100644
--- a/application/single_app/static/js/chat/chat-messages.js
+++ b/application/single_app/static/js/chat/chat-messages.js
@@ -9571,6 +9571,17 @@ function executeMessageDeletion(deleteThread = false) {
if (typeof loadConversations === 'function') {
loadConversations();
}
+ if (window.currentConversationId) {
+ window.dispatchEvent(new CustomEvent(
+ 'chat:conversation-documents-refresh',
+ {
+ detail: {
+ conversationId: window.currentConversationId,
+ autoOpen: false,
+ },
+ }
+ ));
+ }
} else {
showToast('Failed to delete message', 'error');
}
diff --git a/application/single_app/static/js/chat/chat-retry.js b/application/single_app/static/js/chat/chat-retry.js
index 6dcdebd9e..99d3cfdf3 100644
--- a/application/single_app/static/js/chat/chat-retry.js
+++ b/application/single_app/static/js/chat/chat-retry.js
@@ -320,6 +320,15 @@ window.executeMessageRetry = function() {
console.log(' retry_thread_id:', data.chat_request.retry_thread_id);
console.log(' retry_thread_attempt:', data.chat_request.retry_thread_attempt);
console.log(' Full chat_request:', data.chat_request);
+ window.dispatchEvent(new CustomEvent(
+ 'chat:conversation-documents-refresh',
+ {
+ detail: {
+ conversationId: data.chat_request.conversation_id,
+ autoOpen: false,
+ },
+ }
+ ));
sendMessageWithStreaming(
data.chat_request,
@@ -393,6 +402,15 @@ export function handleCarouselNavigation(messageDiv, messageId, direction) {
})
.then(data => {
console.log(`✅ Switched to attempt ${data.new_active_attempt}:`, data);
+ window.dispatchEvent(new CustomEvent(
+ 'chat:conversation-documents-refresh',
+ {
+ detail: {
+ conversationId: window.currentConversationId || '',
+ autoOpen: false,
+ },
+ }
+ ));
// Reload messages to show new active attempt
if (window.currentConversationId) {
diff --git a/application/single_app/static/js/chat/chat-streaming.js b/application/single_app/static/js/chat/chat-streaming.js
index 15ab2aeff..a2a09b493 100644
--- a/application/single_app/static/js/chat/chat-streaming.js
+++ b/application/single_app/static/js/chat/chat-streaming.js
@@ -1,6 +1,7 @@
// chat-streaming.js
import {
appendMessage,
+ loadMessages,
markUserMessageMetadataFinalizationUnconfirmed,
markUserMessageMetadataUnconfirmed,
refreshUserMessageMetadata,
@@ -374,6 +375,11 @@ function notifyConversationDocumentsMayHaveChanged(conversationId, autoOpen = fa
}));
}
+function hasCitedWorkspaceDocuments(payload) {
+ return Array.isArray(payload?.cited_hybrid_citations)
+ && payload.cited_hybrid_citations.length > 0;
+}
+
async function requestStreamCancellation(streamContext = currentStreamContext) {
if (!streamContext || streamContext.cancellationRequested) {
return false;
@@ -697,6 +703,36 @@ function consumeStreamingResponse(requestFactory, tempAiMessageId, tempUserMessa
error_message: data.error,
});
handleStreamError(tempAiMessageId, data.partial_content || accumulatedContent, data.error, data);
+ if (
+ data.message_persisted
+ && data.message_id
+ && data.conversation_id
+ ) {
+ notifyConversationDocumentsMayHaveChanged(
+ data.conversation_id,
+ false
+ );
+ if (
+ data.conversation_kind === 'collaborative'
+ && typeof window.chatCollaboration?.loadConversationMessages === 'function'
+ ) {
+ void window.chatCollaboration
+ .loadConversationMessages(data.conversation_id)
+ .catch(error => {
+ console.warn(
+ 'Failed to reload persisted collaborative response:',
+ error
+ );
+ });
+ } else {
+ void loadMessages(data.conversation_id).catch(error => {
+ console.warn(
+ 'Failed to reload persisted response:',
+ error
+ );
+ });
+ }
+ }
clearCurrentStreamController(abortController);
if (typeof onError === 'function') {
onError(data.error, data);
@@ -1276,6 +1312,10 @@ function finalizeCancelledStreamingMessage(messageId, userMessageId, finalData,
document.querySelector(`[data-message-id="${finalData.message_id}"]`),
Boolean(String(partialContent || '').trim())
);
+ notifyConversationDocumentsMayHaveChanged(
+ finalData.conversation_id,
+ false
+ );
} else if (messageElement) {
renderStoppedContent(messageElement, partialContent);
}
@@ -1377,7 +1417,10 @@ function finalizeStreamingMessage(messageId, userMessageId, finalData, fallbackA
if (existingFinalMessage) {
markStreamingConversationReadIfActive(finalData.conversation_id, 'live streaming completion');
- notifyConversationDocumentsMayHaveChanged(finalData.conversation_id, Boolean(finalData.augmented));
+ notifyConversationDocumentsMayHaveChanged(
+ finalData.conversation_id,
+ hasCitedWorkspaceDocuments(finalData)
+ );
return;
}
@@ -1406,7 +1449,10 @@ function finalizeStreamingMessage(messageId, userMessageId, finalData, fallbackA
if (finalData.reload_messages && finalData.conversation_id && typeof window.chatMessages?.loadMessages === 'function') {
window.chatMessages.loadMessages(finalData.conversation_id);
}
- notifyConversationDocumentsMayHaveChanged(finalData.conversation_id, Boolean(finalData.augmented));
+ notifyConversationDocumentsMayHaveChanged(
+ finalData.conversation_id,
+ hasCitedWorkspaceDocuments(finalData)
+ );
return;
}
@@ -1470,7 +1516,10 @@ function finalizeStreamingMessage(messageId, userMessageId, finalData, fallbackA
window.chatMessages.loadMessages(finalData.conversation_id);
}
- notifyConversationDocumentsMayHaveChanged(finalData.conversation_id, Boolean(finalData.augmented));
+ notifyConversationDocumentsMayHaveChanged(
+ finalData.conversation_id,
+ hasCitedWorkspaceDocuments(finalData)
+ );
markStreamingConversationReadIfActive(finalData.conversation_id, 'live streaming completion');
}
diff --git a/docs/explanation/features/CONVERSATION_CONTENTS_DRAWER.md b/docs/explanation/features/CONVERSATION_CONTENTS_DRAWER.md
index 6b8cbccbd..bd7b03b01 100644
--- a/docs/explanation/features/CONVERSATION_CONTENTS_DRAWER.md
+++ b/docs/explanation/features/CONVERSATION_CONTENTS_DRAWER.md
@@ -7,6 +7,7 @@ The conversation contents drawer provides a compact index of persisted user mess
Implemented in version: **0.250.074**
Used documents mode added in version: **0.250.159**
Compact overflow-safe layout added in version: **0.250.171**
+Source/cited distinction added in version: **0.250.215**
### Dependencies
@@ -21,7 +22,9 @@ Compact overflow-safe layout added in version: **0.250.171**
The contents mode is derived entirely from messages already loaded into the chat DOM. It does not add a conversation-index API or a second datastore. `chat-conversation-contents.js` observes persisted user-message elements, renders labels with `textContent`, and tracks the message nearest the current scroll position.
-The documents mode uses the same conversation metadata source as the details modal. `functions_conversation_metadata.py` retains each document's user-facing title and source filename in the conversation tag. `chat-conversation-details.js` exposes shared helpers for fetching conversation metadata and filtering `tags` where `category === "document"`. `chat-conversation-contents.js` renders those document tags into a vertical row list, so selected-but-unused documents are excluded and the side pane stays aligned with the details modal document card.
+The documents mode uses the same conversation metadata source as the details modal but applies a different selector. Document tags remain the complete source inventory. New assistant messages persist exact cited document records, and conversation metadata maintains a compact `used_documents` aggregate for active responses. `chat-conversation-contents.js` renders exact used documents plus any explicit legacy fallback, while `chat-conversation-details.js` keeps the complete source list and marks exact matches with a **Cited** badge.
+
+`functions_citation_tracking.py` matches final-response citation IDs without mutating full source arrays. Historical conversations without versioned tracking continue to use their existing document tags in the drawer; they are not falsely marked as exact citations in the details modal.
The application setting `enable_conversation_contents_drawer` is the authoritative global gate and defaults to `true`. The user setting `conversationContentsDrawerEnabled` also defaults to `true`; it is only effective while the admin gate is enabled.
@@ -38,6 +41,8 @@ The application setting `enable_conversation_contents_drawer` is the authoritati
- Contents and Documents share the same right-side drawer. Opening either mode swaps the drawer content instead of creating side-by-side panes.
- After a streamed response completes, the drawer refreshes full conversation metadata and auto-opens Documents once per conversation when cited documents first appear.
- Document rows show title, classification, source filename when it differs from the title, cited page references, and workspace scope.
+- The full conversation details modal lists every source document and marks only exact tracked use with a **Cited** badge.
+- Retrieved-only documents remain available under each response's **Sources** disclosure but do not enter the strict Used documents list.
- Chunk counts and backend document IDs are intentionally omitted from the user-facing drawer.
- Both lists use compact typography and constrained grid/flex tracks so long labels and metadata truncate instead of creating horizontal scrolling.
@@ -46,6 +51,8 @@ The application setting `enable_conversation_contents_drawer` is the authoritati
- `application/single_app/functions_settings.py`
- `application/single_app/functions_conversation_contents.py`
- `application/single_app/functions_conversation_metadata.py`
+- `application/single_app/functions_citation_tracking.py`
+- `application/single_app/route_backend_conversations.py`
- `application/single_app/route_frontend_admin_settings.py`
- `application/single_app/route_backend_users.py`
- `application/single_app/route_frontend_chats.py`
@@ -69,12 +76,15 @@ The application setting `enable_conversation_contents_drawer` is the authoritati
- `functional_tests/test_conversation_contents_drawer_settings.py` validates global and user defaults, persistence wiring, validation, and gate precedence.
- `functional_tests/test_document_action_conversation_scope_metadata.py` validates that source filenames survive conversation document-tag creation and refresh.
+- `functional_tests/test_chat_cited_source_tracking.py` validates exact citation matching, source preservation, legacy compatibility, and active-attempt aggregation.
- `ui_tests/test_chat_conversation_contents_drawer.py` covers filtering, ordering, safe 30-character labels, fallback labels, compact cited-document rendering, documents-mode swapping, auto-open behavior, navigation, focus, live updates, conversation replacement, keyboard closing, and desktop/mobile horizontal-overflow measurements.
+- `ui_tests/test_chat_scope_lock_and_conversation_details_escaping.py` verifies the full source inventory and safe exact Cited badge.
- The contents list is generated locally from authorized messages and inserts user-authored labels only through safe text APIs. The documents list also renders metadata through DOM APIs and `textContent`.
### Known limitations
- Contents mode indexes user messages only.
-- Documents mode includes cited/used conversation metadata documents only; documents merely selected in the picker but not used for citations are intentionally excluded.
+- New tracked responses include exact cited/used documents only; retrieved-only documents are intentionally excluded.
+- Historical conversations retain the prior document list as a compatibility fallback because exact citation use was not persisted before version `0.250.215`.
- Search, grouping, generated summaries, assistant-message entries, and document filtering are not included.
- Only messages currently loaded into the active timeline can appear in the contents list.
diff --git a/docs/explanation/features/v0.238.024/CITATION_IMPROVEMENTS.md b/docs/explanation/features/v0.238.024/CITATION_IMPROVEMENTS.md
index dd2029c02..66ca169c9 100644
--- a/docs/explanation/features/v0.238.024/CITATION_IMPROVEMENTS.md
+++ b/docs/explanation/features/v0.238.024/CITATION_IMPROVEMENTS.md
@@ -2,6 +2,7 @@
**Feature**: Citation Improvements
**Version**: v0.238.024
+**Updated**: v0.250.215
**Dependencies**: Enhanced Citations System, chat-citations.js
## Overview and Purpose
@@ -15,6 +16,7 @@ This release includes several improvements to the citation system in the chat in
- **Improved Error Handling**: Graceful JSON parsing with explicit error messages in citation processing
- **Robust Citation Links**: Support for both inline citation links and hybrid citation buttons
- **Metadata Citations**: New metadata citation type for viewing extracted metadata (OCR text, vision analysis, detected objects)
+- **Source and Reference Separation**: Complete returned sources remain inspectable while exact document and web citations are persisted separately for Used documents, details badges, and exports
## Technical Specifications
@@ -37,6 +39,18 @@ The citation system supports multiple citation types:
| **Agent Citations** | Tool execution details from Semantic Kernel agents | Expandable section with function args and results |
| **Metadata Citations** | Extracted metadata from document processing | Expandable section with OCR text, vision analysis, detected objects |
+### Retrieved sources and cited references
+
+Starting in version `0.250.215`, SimpleChat distinguishes retrieval disclosure from final-response references:
+
+- `hybrid_citations` and `web_search_citations` retain all returned sources for the **Sources** control and JSON audit output.
+- `cited_hybrid_citations` contains only returned document records whose citation IDs appear in the final response, with strict filename/location matching for explicit legacy-style source references that omit the ID.
+- `cited_web_search_citations` contains only returned web records whose normalized URLs appear in the final response.
+- Agent citations continue to represent tool executions that actually occurred; Foundry web-source proxy entries are deduplicated into web references for exports.
+- Conversation details list all source documents and mark exact use with **Cited**.
+- Used documents and reference-producing exports consume exact cited subsets for new responses.
+- Historical records without versioned tracking use the previous broad fallback and require no migration.
+
### Enhanced Citation Modal
Citations that reference documents open specialized viewers based on file type:
@@ -62,6 +76,10 @@ The document selection dropdown in the chat interface was widened to accommodate
| `static/js/chat/chat-citations.js` | Core citation parsing, link generation, and rendering |
| `static/js/chat/chat-enhanced-citations.js` | Enhanced citation modal for PDF/image/video/audio viewing |
| `static/js/chat/chat-documents.js` | `getDocumentMetadata()` for metadata citations; dropdown width updates |
+| `functions_citation_tracking.py` | Exact document/web citation matching and compatibility-aware reference selection |
+| `route_backend_conversation_export.py` | Citation-accurate conversation and per-message reference sections |
+| `static/js/chat/chat-conversation-details.js` | Full source inventory and exact Cited annotations |
+| `static/js/chat/chat-conversation-contents.js` | Exact Used documents selection with historical fallback |
## Commits
@@ -81,3 +99,5 @@ The document selection dropdown in the chat interface was widened to accommodate
- **Timestamp seeking**: Verify video and audio citations navigate to the correct timestamp
- **Metadata citations**: Verify extracted metadata (OCR, vision, objects) displays correctly in expandable sections
- **Error handling**: Verify malformed citation strings are handled gracefully without breaking the chat interface
+- **Source/reference distinction**: Verify retrieved-only sources remain under Sources but are excluded from new Used documents and export references
+- **Compatibility**: Verify historical messages and conversations retain their legacy fallback without read-time history parsing
diff --git a/docs/explanation/fixes/SOURCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md b/docs/explanation/fixes/SOURCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md
new file mode 100644
index 000000000..e09b91e12
--- /dev/null
+++ b/docs/explanation/fixes/SOURCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md
@@ -0,0 +1,96 @@
+# Retrieved Sources and Cited References Distinction Fix
+
+## Header information
+
+**Issue:** Retrieved document and web search results were presented as citations even when the final assistant response did not reference them.
+
+**Root cause:** SimpleChat stored complete retrieval result arrays in `hybrid_citations` and `web_search_citations`, then reused those arrays for the per-message Sources disclosure, the Used documents drawer, conversation details, and export reference sections. The data model did not preserve a separate exact-citation subset.
+
+Fixed/Implemented in version: **0.250.215**
+
+Related config.py update: `VERSION = "0.250.215"`
+
+Associated issue: `microsoft/simplechat#1249`
+
+## Technical details
+
+### Persisted contract
+
+New assistant responses preserve both source and reference data:
+
+| Field | Meaning |
+|---|---|
+| `hybrid_citations` | Every returned workspace document source |
+| `web_search_citations` | Every returned web source |
+| `agent_citations` | Tool and agent executions that occurred |
+| `citation_tracking_version` | Indicates exact citation subsets are authoritative |
+| `cited_hybrid_citations` | Returned document records matched by exact citation IDs, with strict filename/location matching for explicit legacy-style source references |
+| `cited_web_search_citations` | Returned web records whose normalized URLs appear in the final response |
+
+Conversation records persist `used_documents` for exact active-response usage and `legacy_used_documents` for pre-tracking compatibility. Normal metadata reads use these compact fields and do not reparse message history.
+
+### Files modified
+
+- `application/single_app/functions_citation_tracking.py`
+- `application/single_app/route_backend_chats.py`
+- `application/single_app/route_backend_conversations.py`
+- `application/single_app/functions_simplechat_operations.py`
+- `application/single_app/functions_collaboration.py`
+- `application/single_app/collaboration_models.py`
+- `application/single_app/route_backend_collaboration.py`
+- `application/single_app/functions_workflow_runner.py`
+- `application/single_app/route_backend_conversation_export.py`
+- `application/single_app/static/js/chat/chat-citations.js`
+- `application/single_app/static/js/chat/chat-collaboration.js`
+- `application/single_app/static/js/chat/chat-conversation-details.js`
+- `application/single_app/static/js/chat/chat-conversation-contents.js`
+- `application/single_app/static/js/chat/chat-streaming.js`
+
+### Code changes
+
+- Exact `[#citation-id]` tokens are matched to returned document records after final response content is settled. Explicit `(Source: filename, Page/Sheet/Location: value)` references provide a strict fallback for response paths that omit the hidden ID.
+- Citation matching patterns use bounded quantifiers so hostile or malformed model output cannot trigger polynomial backtracking (ReDoS) while parsing references.
+- Final-response HTTP(S) URLs are normalized and matched to returned web records.
+- Full source arrays remain unchanged for the per-message **Sources** disclosure and JSON audit output.
+- The **Used documents** drawer reads exact active-response usage plus an explicit historical fallback.
+- Conversation details continue to show all source documents and add a **Cited** badge only to exact tracked documents.
+- Retry, edit, attempt switching, deletion, collaboration mirroring, workflow messages, and conversation forks maintain the exact used-document aggregate.
+- Canceled/interrupted persisted responses update source metadata and notify active clients; collaboration stream bridges mirror persisted partial responses before returning the terminal error.
+- Conversation Markdown/PDF and per-message Word/PowerPoint/email reference sections use cited document/web subsets plus executed tools. Foundry web-source proxy entries are represented once as web references rather than duplicated as tool references.
+- Historical messages without a tracking version retain their previous broad export behavior.
+
+### Testing approach
+
+- `functional_tests/test_chat_cited_source_tracking.py` validates exact document IDs, normalized web URLs, immutable source arrays, aggregation, lifecycle rebuilds, collaboration propagation, UI contracts, and export selection.
+- `functional_tests/test_conversation_export.py` validates that tracked reference appendices exclude retrieved-only document and web sources while JSON retains them.
+- `functional_tests/test_per_message_export.py` and `functional_tests/test_per_message_powerpoint_export.py` validate the shared Word, email, and PowerPoint reference path.
+- `ui_tests/test_chat_conversation_contents_drawer.py` validates strict Used documents selection and historical fallback.
+- `ui_tests/test_chat_scope_lock_and_conversation_details_escaping.py` validates the full source inventory, exact Cited badge, and safe rendering.
+
+### Impact analysis
+
+- No Cosmos migration is required.
+- Existing source links and enhanced citation behavior remain unchanged.
+- New responses gain exact reference semantics.
+- Historical conversations remain readable and keep their previous drawer/export behavior when exact data is unavailable.
+- Ordinary metadata reads remain constant-time with respect to message history; message scans occur only after uncommon lifecycle mutations.
+
+## Validation
+
+### Before
+
+- Every retrieved document could appear in **Used documents**.
+- Conversation details could not distinguish cited documents from retrieved-only documents.
+- Export reference sections could list every returned document and web result.
+
+### After
+
+- **Sources** remains the complete retrieval/execution disclosure.
+- **Used documents** reflects cited documents in active responses, with an explicit legacy fallback.
+- Conversation details list all source documents and mark exact citations.
+- Reference-producing exports contain exact cited document/web records and executed tools.
+- JSON exports preserve both complete source arrays and exact cited subsets.
+
+### User experience improvement
+
+Users can now tell what the model reviewed from what it actually cited, and exported references accurately represent the evidence visible in the generated response.
diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md
index 7ad9569c3..4ba293ab5 100644
--- a/docs/explanation/release_notes.md
+++ b/docs/explanation/release_notes.md
@@ -2,6 +2,16 @@
For feature-focused and fix-focused drill-downs by version, see [Features by Version](/explanation/features/) and [Fixes by Version](/explanation/fixes/).
+### **(v0.250.215)**
+
+#### Bug Fixes
+
+* **Retrieved Sources and Cited References**
+ * Separated complete document/web retrieval results from the exact references used in final assistant responses, while preserving all returned results under **Sources**.
+ * Used documents now follows active cited responses, conversation details marks cited items within the full source inventory, and conversation/message export references exclude retrieved-only sources.
+ * Historical conversations retain their previous fallback without a migration or ordinary read-time history parsing.
+ * (Ref: #1249, `functions_citation_tracking.py`, `route_backend_chats.py`, `route_backend_conversation_export.py`, `SOURCE_AND_CITED_REFERENCE_DISTINCTION_FIX.md`)
+
### **(v0.250.214)**
#### New Features
diff --git a/functional_tests/test_chat_cited_source_tracking.py b/functional_tests/test_chat_cited_source_tracking.py
new file mode 100644
index 000000000..3b8ba79c7
--- /dev/null
+++ b/functional_tests/test_chat_cited_source_tracking.py
@@ -0,0 +1,659 @@
+# test_chat_cited_source_tracking.py
+#!/usr/bin/env python3
+"""
+Functional test for cited-source tracking.
+Version: 0.250.215
+Implemented in: 0.250.215
+
+This test ensures returned sources remain complete while exact document and
+web references drive used-document aggregates and export reference buckets.
+"""
+
+from copy import deepcopy
+from pathlib import Path
+import sys
+import time
+
+from test_support.versioning import assert_app_version_at_least
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+APP_ROOT = REPO_ROOT / "application" / "single_app"
+CHAT_ROUTE = APP_ROOT / "route_backend_chats.py"
+CONVERSATIONS_ROUTE = APP_ROOT / "route_backend_conversations.py"
+COLLABORATION_FUNCTIONS = APP_ROOT / "functions_collaboration.py"
+COLLABORATION_ROUTE = APP_ROOT / "route_backend_collaboration.py"
+CONVERSATION_EXPORT_ROUTE = APP_ROOT / "route_backend_conversation_export.py"
+CONVERSATION_DETAILS_JS = (
+ APP_ROOT / "static" / "js" / "chat" / "chat-conversation-details.js"
+)
+CONVERSATION_CONTENTS_JS = (
+ APP_ROOT / "static" / "js" / "chat" / "chat-conversation-contents.js"
+)
+CHAT_STREAMING_JS = APP_ROOT / "static" / "js" / "chat" / "chat-streaming.js"
+CHAT_COLLABORATION_JS = (
+ APP_ROOT / "static" / "js" / "chat" / "chat-collaboration.js"
+)
+CHAT_MESSAGES_JS = APP_ROOT / "static" / "js" / "chat" / "chat-messages.js"
+CHAT_RETRY_JS = APP_ROOT / "static" / "js" / "chat" / "chat-retry.js"
+CHAT_EDIT_JS = APP_ROOT / "static" / "js" / "chat" / "chat-edit.js"
+SIMPLECHAT_OPERATIONS = APP_ROOT / "functions_simplechat_operations.py"
+WORKFLOW_RUNNER = APP_ROOT / "functions_workflow_runner.py"
+if str(APP_ROOT) not in sys.path:
+ sys.path.insert(0, str(APP_ROOT))
+
+from functions_citation_tracking import ( # noqa: E402
+ CITATION_TRACKING_VERSION,
+ USED_DOCUMENTS_TRACKING_VERSION,
+ build_cited_source_subsets,
+ build_used_documents,
+ get_message_reference_citation_buckets,
+ initialize_conversation_used_document_tracking,
+ merge_cited_documents_into_conversation,
+ rebuild_conversation_used_documents,
+ resolve_citation_location,
+)
+from collaboration_models import ( # noqa: E402
+ build_collaboration_message_doc_from_legacy,
+)
+
+
+def _hybrid_sources():
+ return [
+ {
+ "file_name": "Policy.pdf",
+ "document_id": "doc-policy",
+ "citation_id": "doc-policy_1",
+ "chunk_id": "1",
+ "page_number": 1,
+ "classification": "Internal",
+ },
+ {
+ "file_name": "Policy.pdf",
+ "document_id": "doc-policy",
+ "citation_id": "doc-policy_2",
+ "chunk_id": "2",
+ "page_number": 2,
+ "classification": "Internal",
+ },
+ {
+ "file_name": "Unused.pdf",
+ "document_id": "doc-unused",
+ "citation_id": "doc-unused_7",
+ "chunk_id": "7",
+ "page_number": 7,
+ },
+ ]
+
+
+def _web_sources():
+ return [
+ {
+ "title": "Cited web page",
+ "url": "https://Example.com:443/guidance?view=current#overview",
+ },
+ {
+ "title": "Unused web page",
+ "url": "https://example.com/unused",
+ },
+ ]
+
+
+def test_exact_document_and_web_references_preserve_sources():
+ """Match grouped document IDs and normalized URLs without mutating sources."""
+ hybrid_sources = _hybrid_sources()
+ web_sources = _web_sources()
+ original_hybrid_sources = deepcopy(hybrid_sources)
+ original_web_sources = deepcopy(web_sources)
+ content = (
+ "The policy requires review "
+ "(Source: Policy.pdf, Pages: 1, 2) [#doc-policy_1; #doc-policy_2]. "
+ "See [current guidance](https://example.com/guidance?view=current). "
+ "An unknown token [#doc-missing_9] is ignored."
+ )
+
+ tracked = build_cited_source_subsets(
+ content,
+ hybrid_citations=hybrid_sources,
+ web_search_citations=web_sources,
+ )
+
+ assert tracked["citation_tracking_version"] == CITATION_TRACKING_VERSION
+ assert [
+ citation["citation_id"]
+ for citation in tracked["cited_hybrid_citations"]
+ ] == ["doc-policy_1", "doc-policy_2"]
+ assert [
+ citation["title"]
+ for citation in tracked["cited_web_search_citations"]
+ ] == ["Cited web page"]
+ assert hybrid_sources == original_hybrid_sources
+ assert web_sources == original_web_sources
+
+ tracked["cited_hybrid_citations"][0]["file_name"] = "Changed.pdf"
+ assert hybrid_sources[0]["file_name"] == "Policy.pdf"
+
+
+def test_explicit_source_location_matches_without_hidden_id():
+ """Match the visible citation format required by the model prompt."""
+ tracked = build_cited_source_subsets(
+ "Does the second page control (Source: Policy.pdf, Page: 2)?",
+ hybrid_citations=_hybrid_sources(),
+ web_search_citations=[],
+ )
+
+ assert [
+ citation["citation_id"]
+ for citation in tracked["cited_hybrid_citations"]
+ ] == ["doc-policy_2"]
+
+
+def test_hidden_id_prevents_same_page_source_overmatching():
+ """An exact ID should win over other returned chunks from the same page."""
+ same_page_source = {
+ **_hybrid_sources()[0],
+ "citation_id": "doc-policy_1_other",
+ "chunk_id": "1-other",
+ }
+ tracked = build_cited_source_subsets(
+ (
+ "The first page controls "
+ "(Source: Policy.pdf, Page: 1) [#doc-policy_1]"
+ ),
+ hybrid_citations=[_hybrid_sources()[0], same_page_source],
+ web_search_citations=[],
+ )
+
+ assert [
+ citation["citation_id"]
+ for citation in tracked["cited_hybrid_citations"]
+ ] == ["doc-policy_1"]
+
+
+def test_visible_source_matching_handles_sheet_and_filename_commas():
+ """Visible references should support sheet locations and original filenames."""
+ tracked = build_cited_source_subsets(
+ (
+ "The Q1 sheet controls. "
+ "(Source: Budget, 2026.xlsx, Sheet: Q1, Final (Approved))"
+ ),
+ hybrid_citations=[{
+ "file_name": "Budget, 2026.xlsx",
+ "document_id": "budget-doc",
+ "citation_id": "budget-doc_Q1",
+ "page_number": "Q1, Final (Approved)",
+ "sheet_name": "Q1, Final (Approved)",
+ "location_label": "Sheet",
+ "location_value": "Q1, Final (Approved)",
+ }],
+ web_search_citations=[],
+ )
+
+ assert [
+ citation["citation_id"]
+ for citation in tracked["cited_hybrid_citations"]
+ ] == ["budget-doc_Q1"]
+
+
+def test_schema_aware_citation_location_is_shared():
+ """Workflow and chat citations should label workbook schemas consistently."""
+ assert resolve_citation_location(
+ page_number=1,
+ chunk_text="Tabular workbook: Budget.xlsx",
+ is_tabular=True,
+ ) == ("Location", "Workbook Schema")
+ assert resolve_citation_location(
+ page_number=4,
+ chunk_text="Narrative page",
+ is_tabular=False,
+ ) == ("Page", "4")
+
+
+def test_citation_patterns_resist_adversarial_input():
+ """Bounded citation patterns must not degrade on hostile model output."""
+ adversarial_inputs = (
+ "[#" + ("[#\\" * 20000),
+ "(Source:" + (" " * 40000),
+ "(Source: a, Page:" + (" " * 40000),
+ "(Source: a, Page: " + ("0" * 40000) + "-1)",
+ "[#" + ("0" * 40000),
+ )
+
+ hybrid_citations = _hybrid_sources()
+ started_at = time.monotonic()
+ for adversarial_input in adversarial_inputs:
+ tracked = build_cited_source_subsets(
+ adversarial_input,
+ hybrid_citations=hybrid_citations,
+ web_search_citations=_web_sources(),
+ )
+ assert tracked["cited_hybrid_citations"] == []
+ assert tracked["cited_web_search_citations"] == []
+ elapsed_seconds = time.monotonic() - started_at
+
+ assert elapsed_seconds < 5, (
+ f"Citation matching took {elapsed_seconds:.2f}s on adversarial input"
+ )
+
+
+def test_used_documents_collapse_cited_chunks_only():
+ """Collapse multiple cited chunks while excluding retrieved-only documents."""
+ source_tags = [
+ {
+ "category": "document",
+ "document_id": "doc-policy",
+ "title": "Policy Handbook",
+ "file_name": "Policy.pdf",
+ "classification": "Internal",
+ "chunk_ids": ["doc-policy_1", "doc-policy_2", "doc-policy_3"],
+ "scope": {
+ "type": "group",
+ "id": "group-1",
+ "name": "Policy Team",
+ },
+ },
+ ]
+
+ used_documents = build_used_documents(
+ _hybrid_sources()[:2],
+ source_document_tags=source_tags,
+ )
+
+ assert len(used_documents) == 1
+ used_document = used_documents[0]
+ assert used_document["document_id"] == "doc-policy"
+ assert used_document["title"] == "Policy Handbook"
+ assert used_document["chunk_ids"] == ["doc-policy_1", "doc-policy_2"]
+ assert used_document["citation_ids"] == ["doc-policy_1", "doc-policy_2"]
+ assert used_document["page_numbers"] == [1, 2]
+ assert used_document["scope"]["name"] == "Policy Team"
+
+
+def test_reference_buckets_use_strict_empty_values_and_legacy_fallback():
+ """Tracked messages must not fall back when exact cited subsets are empty."""
+ source_message = {
+ "citations": [{"title": "Legacy"}],
+ "hybrid_citations": _hybrid_sources(),
+ "web_search_citations": _web_sources(),
+ "agent_citations": [
+ {"tool_name": "Calculator", "function_name": "calculate"},
+ {
+ "tool_name": "Used web reference",
+ "function_name": "azure_ai_foundry_web_search",
+ },
+ ],
+ }
+
+ legacy_buckets = get_message_reference_citation_buckets(source_message)
+ assert len(legacy_buckets["hybrid"]) == 3
+ assert len(legacy_buckets["web"]) == 2
+ assert len(legacy_buckets["agent"]) == 2
+
+ tracked_message = {
+ **source_message,
+ "citation_tracking_version": CITATION_TRACKING_VERSION,
+ "cited_hybrid_citations": [],
+ "cited_web_search_citations": [],
+ }
+ tracked_buckets = get_message_reference_citation_buckets(tracked_message)
+ assert tracked_buckets["hybrid"] == []
+ assert tracked_buckets["web"] == []
+ assert tracked_buckets["agent"] == [{
+ "tool_name": "Calculator",
+ "function_name": "calculate",
+ }]
+ assert tracked_buckets["legacy"] == [{"title": "Legacy"}]
+
+
+def test_conversation_tracking_preserves_legacy_and_merges_exact_usage():
+ """Snapshot pre-tracking tags and keep new exact documents separate."""
+ conversation = {
+ "tags": [
+ {
+ "category": "document",
+ "document_id": "doc-legacy",
+ "title": "Historical Source",
+ "chunk_ids": ["doc-legacy_1"],
+ },
+ {
+ "category": "participant",
+ "user_id": "user-1",
+ },
+ ],
+ }
+
+ initialized = initialize_conversation_used_document_tracking(conversation)
+ assert initialized is True
+ assert (
+ conversation["used_documents_tracking_version"]
+ == USED_DOCUMENTS_TRACKING_VERSION
+ )
+ assert [
+ document["document_id"]
+ for document in conversation["legacy_used_documents"]
+ ] == ["doc-legacy"]
+ assert conversation["used_documents"] == []
+
+ conversation["tags"].append({
+ "category": "document",
+ "document_id": "doc-policy",
+ "title": "Policy Handbook",
+ "file_name": "Policy.pdf",
+ "chunk_ids": ["doc-policy_1", "doc-policy_2", "doc-policy_7"],
+ "scope": {
+ "type": "group",
+ "id": "group-1",
+ "name": "Policy Team",
+ },
+ })
+ merged = merge_cited_documents_into_conversation(
+ conversation,
+ _hybrid_sources()[:2],
+ )
+
+ assert [document["document_id"] for document in merged] == ["doc-policy"]
+ assert merged[0]["chunk_ids"] == ["doc-policy_1", "doc-policy_2"]
+ assert [
+ document["document_id"]
+ for document in conversation["legacy_used_documents"]
+ ] == ["doc-legacy"]
+
+
+def test_rebuild_uses_active_non_deleted_tracked_assistant_messages():
+ """Retry and deletion rebuilds must follow the active visible response set."""
+ conversation = {
+ "tags": [
+ {
+ "category": "document",
+ "document_id": "doc-policy",
+ "title": "Policy Handbook",
+ "chunk_ids": ["doc-policy_1", "doc-policy_2"],
+ },
+ {
+ "category": "document",
+ "document_id": "doc-unused",
+ "title": "Unused Source",
+ "chunk_ids": ["doc-unused_7"],
+ },
+ ],
+ "used_documents_tracking_version": USED_DOCUMENTS_TRACKING_VERSION,
+ "legacy_used_documents": [],
+ "used_documents": [],
+ }
+ messages = [
+ {
+ "role": "assistant",
+ "citation_tracking_version": CITATION_TRACKING_VERSION,
+ "cited_hybrid_citations": [_hybrid_sources()[0]],
+ "metadata": {
+ "thread_info": {
+ "active_thread": True,
+ },
+ },
+ },
+ {
+ "role": "assistant",
+ "citation_tracking_version": CITATION_TRACKING_VERSION,
+ "cited_hybrid_citations": [_hybrid_sources()[1]],
+ "metadata": {
+ "thread_info": {
+ "active_thread": False,
+ },
+ },
+ },
+ {
+ "role": "assistant",
+ "citation_tracking_version": CITATION_TRACKING_VERSION,
+ "cited_hybrid_citations": [_hybrid_sources()[2]],
+ "metadata": {
+ "is_deleted": True,
+ "thread_info": {
+ "active_thread": True,
+ },
+ },
+ },
+ {
+ "role": "assistant",
+ "hybrid_citations": [_hybrid_sources()[2]],
+ "metadata": {},
+ },
+ ]
+
+ rebuilt = rebuild_conversation_used_documents(conversation, messages)
+
+ assert [document["document_id"] for document in rebuilt] == ["doc-policy"]
+ assert rebuilt[0]["chunk_ids"] == ["doc-policy_1"]
+
+
+def test_fork_rebuilds_legacy_fallback_from_copied_messages():
+ """Fork legacy fallback must exclude documents introduced after the fork point."""
+ conversation = {
+ "tags": [
+ {
+ "category": "document",
+ "document_id": "doc-policy",
+ "title": "Policy Handbook",
+ "chunk_ids": ["doc-policy_1"],
+ },
+ {
+ "category": "document",
+ "document_id": "doc-unused",
+ "title": "Later Source",
+ "chunk_ids": ["doc-unused_7"],
+ },
+ ],
+ }
+ copied_messages = [
+ {
+ "role": "assistant",
+ "hybrid_citations": [_hybrid_sources()[0]],
+ "metadata": {
+ "thread_info": {
+ "active_thread": True,
+ },
+ },
+ },
+ ]
+
+ rebuild_conversation_used_documents(
+ conversation,
+ copied_messages,
+ rebuild_legacy=True,
+ )
+
+ assert conversation["used_documents"] == []
+ assert [
+ document["document_id"]
+ for document in conversation["legacy_used_documents"]
+ ] == ["doc-policy"]
+
+
+def test_chat_publication_paths_persist_tracking_and_aggregates():
+ """Verify each chat publication path stores exact subsets and aggregates."""
+ route_source = CHAT_ROUTE.read_text(encoding="utf-8")
+
+ required_snippets = [
+ "document_action_citation_tracking = build_cited_source_subsets(",
+ "citation_tracking = build_cited_source_subsets(",
+ "stream_citation_tracking = build_cited_source_subsets(",
+ "partial_citation_tracking = build_cited_source_subsets(",
+ "interrupted_citation_tracking = build_cited_source_subsets(",
+ "**document_action_citation_tracking,",
+ "**citation_tracking,",
+ "**stream_citation_tracking,",
+ "initialize_conversation_used_document_tracking(conversation_item)",
+ "merge_cited_documents_into_conversation(",
+ "'hybrid_citations': hybrid_citations_list",
+ "'web_search_citations': web_search_citations_list",
+ ]
+ missing = [
+ snippet
+ for snippet in required_snippets
+ if snippet not in route_source
+ ]
+ assert not missing, f"Missing chat citation tracking wiring: {missing}"
+ assert route_source.count("merge_cited_documents_into_conversation(") >= 4
+ assert "Copy the exact bracketed citation ID shown after the supporting excerpt." in route_source
+ assert 'reason="chat_stream_interrupted"' in route_source
+ assert route_source.count("collect_stream_response_conversation_metadata()") >= 4
+ assert '"sheet_name": source_doc.get("sheet_name")' in route_source
+ assert '"sheet_name": sheet_name' in route_source
+
+
+def test_lifecycle_mutations_and_forks_rebuild_exact_usage():
+ """Verify retry/edit/switch/delete/fork paths invoke the shared rebuild."""
+ route_source = CONVERSATIONS_ROUTE.read_text(encoding="utf-8")
+ operations_source = SIMPLECHAT_OPERATIONS.read_text(encoding="utf-8")
+
+ assert (
+ route_source.count(
+ "_rebuild_authorized_personal_conversation_used_documents("
+ )
+ >= 5
+ )
+ for reason in (
+ "message_deleted",
+ "message_retry_created",
+ "message_edit_created",
+ "message_attempt_switched",
+ ):
+ assert reason in route_source
+ assert '"used_documents_tracking_version"' in operations_source
+ assert '"legacy_used_documents"' in operations_source
+ assert "rebuild_legacy=True" in operations_source
+ assert '"used_documents_tracking_version": conversation_item.get(' in route_source
+ assert '"legacy_used_documents": conversation_item.get(' in route_source
+ assert '"used_documents": conversation_item.get(' in route_source
+ assert "except Exception as rebuild_error:" in route_source
+ assert "used_documents_rebuild_required" in route_source
+
+
+def test_collaboration_and_workflow_propagate_tracking_contract():
+ """Verify shared and workflow messages retain exact cited subsets."""
+ legacy_message = {
+ "id": "assistant-source-1",
+ "role": "assistant",
+ "content": "Tracked response",
+ "timestamp": "2026-08-14T12:00:00Z",
+ "citation_tracking_version": CITATION_TRACKING_VERSION,
+ "cited_hybrid_citations": [_hybrid_sources()[0]],
+ "cited_web_search_citations": [_web_sources()[0]],
+ }
+ converted = build_collaboration_message_doc_from_legacy(
+ conversation_id="collaboration-1",
+ legacy_message=legacy_message,
+ default_sender_user={
+ "user_id": "user-1",
+ "display_name": "Test User",
+ "email": "test@example.com",
+ },
+ )
+ assert converted["citation_tracking_version"] == CITATION_TRACKING_VERSION
+ assert converted["cited_hybrid_citations"][0]["citation_id"] == "doc-policy_1"
+ assert converted["cited_web_search_citations"][0]["title"] == "Cited web page"
+
+ collaboration_source = COLLABORATION_FUNCTIONS.read_text(encoding="utf-8")
+ collaboration_route_source = COLLABORATION_ROUTE.read_text(encoding="utf-8")
+ workflow_source = WORKFLOW_RUNNER.read_text(encoding="utf-8")
+
+ for field_name in (
+ "citation_tracking_version",
+ "cited_hybrid_citations",
+ "cited_web_search_citations",
+ "used_documents_tracking_version",
+ "legacy_used_documents",
+ "used_documents",
+ ):
+ assert field_name in collaboration_source
+ assert "rebuild_conversation_used_documents(conversation_doc, remaining_messages)" in collaboration_source
+ assert "'cited_hybrid_citations': serialized_assistant_message.get(" in collaboration_route_source
+ assert "stream_payload['done'] = True" in collaboration_route_source
+ assert "'conversation_kind': serialized_final_conversation.get(" in collaboration_route_source
+ assert "citation_tracking = build_cited_source_subsets(" in workflow_source
+ assert "merge_cited_documents_into_conversation(" in workflow_source
+ assert "(Source: {file_name}, {location_label}: {location_value}) [#{citation_id}]" in workflow_source
+ assert "'sheet_name': sheet_name" in workflow_source
+ assert "resolve_citation_location(" in workflow_source
+
+
+def test_ui_and_exports_select_cited_subsets_without_losing_sources():
+ """Verify downstream consumers separate source inventory from references."""
+ details_source = CONVERSATION_DETAILS_JS.read_text(encoding="utf-8")
+ contents_source = CONVERSATION_CONTENTS_JS.read_text(encoding="utf-8")
+ export_source = CONVERSATION_EXPORT_ROUTE.read_text(encoding="utf-8")
+
+ assert "export function getConversationDocumentTags(metadata = {})" in details_source
+ assert "export function getConversationUsedDocuments(metadata = {})" in details_source
+ assert "getConversationExactUsedDocuments(metadata)" in details_source
+ assert "getConversationLegacyUsedDocuments(metadata)" in details_source
+ assert "conversation-document-cited-badge" in details_source
+ assert "Source documents" in details_source
+ assert "getConversationUsedDocuments," in contents_source
+ assert "const documents = getConversationUsedDocuments(metadata);" in contents_source
+ assert "Documents cited in active responses" in contents_source
+ assert "exactDocuments.length > 0" in contents_source
+
+ streaming_source = CHAT_STREAMING_JS.read_text(encoding="utf-8")
+ assert "function hasCitedWorkspaceDocuments(payload)" in streaming_source
+ assert "hasCitedWorkspaceDocuments(finalData)" in streaming_source
+ assert (
+ "notifyConversationDocumentsMayHaveChanged("
+ in streaming_source
+ )
+ assert "data.message_persisted" in streaming_source
+ assert "&& data.message_id" in streaming_source
+ assert "&& data.conversation_id" in streaming_source
+ assert "void loadMessages(data.conversation_id).catch" in streaming_source
+ assert "data.conversation_kind === 'collaborative'" in streaming_source
+ assert ".loadConversationMessages(data.conversation_id)" in streaming_source
+
+ collaboration_ui_source = CHAT_COLLABORATION_JS.read_text(encoding="utf-8")
+ assert "'chat:conversation-documents-refresh'" in collaboration_ui_source
+ assert "String(payload.message.role || '').trim().toLowerCase() === 'assistant'" in collaboration_ui_source
+ assert "conversationKind: 'collaborative'" in collaboration_ui_source
+ assert "loadConversationMessages," in collaboration_ui_source
+
+ for mutation_source_path in (
+ CHAT_MESSAGES_JS,
+ CHAT_RETRY_JS,
+ CHAT_EDIT_JS,
+ ):
+ mutation_source = mutation_source_path.read_text(encoding="utf-8")
+ assert "'chat:conversation-documents-refresh'" in mutation_source
+
+ assert "return get_message_reference_citation_buckets(message)" in export_source
+ assert "return get_message_source_citation_buckets(message)" in export_source
+ assert "source_citation_buckets = _collect_source_citation_buckets(message)" in export_source
+ assert "'cited_hybrid_citations': (" in export_source
+ assert "'cited_web_search_citations': (" in export_source
+ assert "Document References" in export_source
+ assert "Web References" in export_source
+
+
+def test_version_is_available():
+ """Verify the application includes the cited-source tracking version."""
+ assert_app_version_at_least("0.250.215")
+
+
+if __name__ == "__main__":
+ tests = [
+ test_exact_document_and_web_references_preserve_sources,
+ test_explicit_source_location_matches_without_hidden_id,
+ test_hidden_id_prevents_same_page_source_overmatching,
+ test_visible_source_matching_handles_sheet_and_filename_commas,
+ test_schema_aware_citation_location_is_shared,
+ test_citation_patterns_resist_adversarial_input,
+ test_used_documents_collapse_cited_chunks_only,
+ test_reference_buckets_use_strict_empty_values_and_legacy_fallback,
+ test_conversation_tracking_preserves_legacy_and_merges_exact_usage,
+ test_rebuild_uses_active_non_deleted_tracked_assistant_messages,
+ test_fork_rebuilds_legacy_fallback_from_copied_messages,
+ test_chat_publication_paths_persist_tracking_and_aggregates,
+ test_lifecycle_mutations_and_forks_rebuild_exact_usage,
+ test_collaboration_and_workflow_propagate_tracking_contract,
+ test_ui_and_exports_select_cited_subsets_without_losing_sources,
+ test_version_is_available,
+ ]
+ for test in tests:
+ test()
+ print(f"Cited-source tracking tests passed: {len(tests)}/{len(tests)}")
diff --git a/functional_tests/test_conversation_export.py b/functional_tests/test_conversation_export.py
index 654059b13..a6daf5277 100644
--- a/functional_tests/test_conversation_export.py
+++ b/functional_tests/test_conversation_export.py
@@ -2,8 +2,8 @@
# test_conversation_export.py
"""
Functional test for conversation export enhancements.
-Version: 0.239.029
-Implemented in: 0.239.022 (base), 0.239.023 (PDF export), 0.239.025 (tag formatting fix), 0.239.028 (summary token budget fix)
+Version: 0.250.215
+Implemented in: 0.239.022 (base); Updated in: 0.250.215
This test ensures conversation export now includes normalized and raw citations,
processing thoughts, transcript-style Markdown appendices, deleted-message filtering,
@@ -20,9 +20,11 @@
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'application', 'single_app'))
from route_backend_conversation_export import ( # noqa: E402
+ _build_message_citation_labels,
_build_citation_counts,
_build_pdf_html_body,
_collect_raw_citation_buckets,
+ _collect_source_citation_buckets,
_conversation_to_markdown,
_conversation_to_pdf_bytes,
_filter_messages_for_export,
@@ -130,6 +132,130 @@ def test_sanitize_message_with_citations_and_thoughts():
return True
+def test_tracked_export_references_exclude_uncited_sources():
+ """Tracked exports should reference exact subsets while retaining source audit data."""
+ print("Testing tracked export reference filtering...")
+
+ cited_document = {
+ 'file_name': 'used-policy.pdf',
+ 'document_id': 'doc-used',
+ 'citation_id': 'doc-used_2',
+ 'page_number': 2,
+ }
+ uncited_document = {
+ 'file_name': 'retrieved-only.pdf',
+ 'document_id': 'doc-unused',
+ 'citation_id': 'doc-unused_8',
+ 'page_number': 8,
+ }
+ cited_web = {
+ 'title': 'Used web reference',
+ 'url': 'https://example.com/used',
+ }
+ uncited_web = {
+ 'title': 'Retrieved web result',
+ 'url': 'https://example.com/unused',
+ }
+ message = {
+ 'id': 'assistant-tracked',
+ 'role': 'assistant',
+ 'content': (
+ 'The cited policy applies (Source: used-policy.pdf, Page: 2). '
+ 'See https://example.com/used.'
+ ),
+ 'timestamp': '2026-08-14T12:00:00Z',
+ 'citation_tracking_version': 1,
+ 'hybrid_citations': [cited_document, uncited_document],
+ 'web_search_citations': [cited_web, uncited_web],
+ 'cited_hybrid_citations': [cited_document],
+ 'cited_web_search_citations': [cited_web],
+ 'agent_citations': [
+ {
+ 'tool_name': 'Calculator',
+ 'function_name': 'calculate',
+ },
+ {
+ 'tool_name': 'Retrieved web result',
+ 'function_name': 'azure_ai_foundry_web_search',
+ 'plugin_name': 'azure_ai_foundry',
+ },
+ ],
+ }
+
+ references = _collect_raw_citation_buckets(message)
+ sources = _collect_source_citation_buckets(message)
+ sanitized = _sanitize_message(
+ message,
+ sequence_index=1,
+ transcript_index=1,
+ thoughts=[],
+ )
+ labels = _build_message_citation_labels(message)
+
+ assert [item['file_name'] for item in references['hybrid']] == ['used-policy.pdf']
+ assert [item['title'] for item in references['web']] == ['Used web reference']
+ assert len(sources['hybrid']) == 2
+ assert len(sources['web']) == 2
+ assert sanitized['citation_counts'] == {
+ 'document': 1,
+ 'web': 1,
+ 'agent_tool': 1,
+ 'legacy': 0,
+ 'total': 3,
+ }
+ assert len(sanitized['hybrid_citations']) == 2
+ assert len(sanitized['web_search_citations']) == 2
+ assert len(sanitized['agent_citations']) == 2
+ assert sanitized['cited_hybrid_citations'] == [cited_document]
+ assert sanitized['cited_web_search_citations'] == [cited_web]
+ assert 'used-policy.pdf — Page 2' in labels
+ assert 'Used web reference' in labels
+ assert 'Calculator' in labels
+ assert all('retrieved-only.pdf' not in label for label in labels)
+ assert all('Retrieved web result' not in label for label in labels)
+
+ entry = {
+ 'conversation': {
+ 'id': 'conversation-tracked',
+ 'title': 'Tracked export',
+ 'message_count': 1,
+ 'citation_counts': sanitized['citation_counts'],
+ },
+ 'summary_intro': {'enabled': False, 'generated': False},
+ 'messages': [sanitized],
+ }
+ markdown = _conversation_to_markdown(entry)
+ pdf_html = _build_pdf_html_body(entry)
+
+ assert 'Document References' in markdown
+ assert 'Web References' in markdown
+ assert 'retrieved-only.pdf' not in markdown
+ assert 'Retrieved web result' not in markdown
+ assert 'Document References' in pdf_html
+ assert 'Web References' in pdf_html
+ assert 'retrieved-only.pdf' not in pdf_html
+ assert 'Retrieved web result' not in pdf_html
+
+ sheet_reference = {
+ 'file_name': 'Budget, 2026.xlsx',
+ 'document_id': 'budget-doc',
+ 'citation_id': 'budget-doc_Q1',
+ 'page_number': 'Q1',
+ 'sheet_name': 'Q1',
+ 'location_label': 'Sheet',
+ 'location_value': 'Q1',
+ }
+ sheet_labels = _build_message_citation_labels({
+ 'citation_tracking_version': 1,
+ 'hybrid_citations': [sheet_reference],
+ 'cited_hybrid_citations': [sheet_reference],
+ })
+ assert sheet_labels == ['Budget, 2026.xlsx — Sheet: Q1']
+
+ print("Tracked export reference filtering passed")
+ return True
+
+
def test_conversation_metadata_and_json_shape():
"""Test that conversation-level metadata captures counts for export JSON."""
print("🔍 Testing conversation metadata and JSON shape...")
@@ -534,6 +660,7 @@ def test_tag_formatting():
tests = [
test_filter_messages_for_export,
test_sanitize_message_with_citations_and_thoughts,
+ test_tracked_export_references_exclude_uncited_sources,
test_conversation_metadata_and_json_shape,
test_markdown_export_structure,
test_safe_filename_and_zip_packaging,
diff --git a/functional_tests/test_per_message_export.py b/functional_tests/test_per_message_export.py
index 3b8e3ddf9..ef933df24 100644
--- a/functional_tests/test_per_message_export.py
+++ b/functional_tests/test_per_message_export.py
@@ -2,8 +2,8 @@
# test_per_message_export.py
"""
Functional tests for the per-message export feature and export route regressions.
-Version: 0.241.146
-Implemented in: 0.241.019
+Version: 0.250.215
+Implemented in: 0.241.019; Updated in: 0.250.215
Covers:
- Happy path: Word document built successfully from a valid message.
@@ -874,6 +874,8 @@ def test_per_message_export_uses_content_override_contract():
assert 'def _apply_message_export_content_override(' in route_source
assert route_source.count('message_content_override = _get_message_export_content_override(data)') >= 3
assert route_source.count('_apply_message_export_content_override(message, message_content_override)') >= 3
+ assert 'get_message_reference_citation_buckets' in route_source
+ assert 'normalized_citations = _normalize_citations(_collect_raw_citation_buckets(message))' in route_source
assert 'function buildMessageExportRequestBody(messageDiv, messageId, conversationId, role, extraFields = {})' in frontend_source
assert 'requestBody.message_content_override = messageContentOverride;' in frontend_source
diff --git a/functional_tests/test_per_message_powerpoint_export.py b/functional_tests/test_per_message_powerpoint_export.py
index 4d6963fc6..7dc09bdc8 100644
--- a/functional_tests/test_per_message_powerpoint_export.py
+++ b/functional_tests/test_per_message_powerpoint_export.py
@@ -2,8 +2,8 @@
# test_per_message_powerpoint_export.py
"""
Functional test for per-message PowerPoint export.
-Version: 0.241.146
-Implemented in: 0.241.033
+Version: 0.250.215
+Implemented in: 0.241.033; Updated in: 0.250.215
This test ensures the message export flow exposes a PowerPoint route,
uses the frontend PowerPoint action hook, prefers the message model
@@ -426,6 +426,7 @@ def test_powerpoint_frontend_hooks_present() -> bool:
assert "fetch('/api/message/export-powerpoint'" in frontend_source, 'Expected frontend fetch for the PowerPoint export endpoint'
assert 'exportMessageAsPowerPoint' in frontend_source, 'Expected frontend PowerPoint export helper'
+ assert 'get_message_reference_citation_buckets' in ROUTE_FILE.read_text(encoding='utf-8')
assert 'message_content_override' in frontend_source, 'Expected PowerPoint export requests to support edited assistant markdown overrides'
assert 'delete requestBody.message_content_override;' in frontend_source, 'Expected artifact PowerPoint export to ignore visible-message overrides'
assert 'dropdown-export-ppt-btn' in menu_source, 'Expected chat message menu PowerPoint action'
diff --git a/ui_tests/test_chat_conversation_contents_drawer.py b/ui_tests/test_chat_conversation_contents_drawer.py
index e3dc96180..505669872 100644
--- a/ui_tests/test_chat_conversation_contents_drawer.py
+++ b/ui_tests/test_chat_conversation_contents_drawer.py
@@ -1,10 +1,11 @@
# test_chat_conversation_contents_drawer.py
"""
UI test for the conversation contents drawer.
-Version: 0.250.171
+Version: 0.250.215
Implemented in: 0.250.074
Documents mode added in: 0.250.159
Compact overflow-safe layout added in: 0.250.171
+Source/cited distinction added in: 0.250.215
This test validates user-message filtering, cited-document mode, safe labels,
navigation, live updates, conversation replacement, keyboard closing, and
@@ -239,38 +240,53 @@ def test_chat_conversation_contents_drawer(playwright, viewport):
"bank-treasury-operations-quarterly-reference-"
"with-an-intentionally-long-filename.xlsx"
)
+ source_document_tags = [
+ {
+ "category": "document",
+ "document_id": "doc-1",
+ "title": document_title,
+ "file_name": document_file_name,
+ "classification": "Confidential",
+ "chunk_ids": ["doc-1_1", "doc-1_2"],
+ "scope": {
+ "type": "group",
+ "id": "group-1",
+ "name": "Product Documentation Workspace With A Long Display Name",
+ },
+ },
+ {
+ "category": "document",
+ "document_id": "doc-2",
+ "title": "Release Plan",
+ "file_name": "release-plan.pdf",
+ "classification": "None",
+ "chunk_ids": ["doc-2_4"],
+ "scope": {
+ "type": "personal",
+ "id": "user-1",
+ "name": "Personal",
+ },
+ },
+ ]
page.route(
"**/api/conversations/conversation-documents-pane/metadata",
lambda route: _fulfill_json(route, {
"title": "Documents pane conversation",
- "tags": [
- {
- "category": "document",
- "document_id": "doc-1",
- "title": document_title,
- "file_name": document_file_name,
- "classification": "Confidential",
- "chunk_ids": ["doc-1_1", "doc-1_2"],
- "scope": {
- "type": "group",
- "id": "group-1",
- "name": "Product Documentation Workspace With A Long Display Name",
- },
- },
- {
- "category": "document",
- "document_id": "doc-2",
- "title": "Release Plan",
- "file_name": "release-plan.pdf",
- "classification": "None",
- "chunk_ids": ["doc-2_4"],
- "scope": {
- "type": "personal",
- "id": "user-1",
- "name": "Personal",
- },
- },
- ],
+ "tags": source_document_tags,
+ "used_documents_tracking_version": 1,
+ "legacy_used_documents": [],
+ "used_documents": [{
+ **source_document_tags[0],
+ "chunk_ids": ["doc-1_99"],
+ "citation_ids": ["doc-1_99"],
+ "page_numbers": [2],
+ "citation_locations": [{
+ "citation_id": "doc-1_99",
+ "page_number": 2,
+ "location_label": "Page",
+ "location_value": 2,
+ }],
+ }],
}),
)
@@ -294,18 +310,20 @@ def test_chat_conversation_contents_drawer(playwright, viewport):
expect(documents_toggle).to_be_visible()
expect(page.locator("#conversation-contents-drawer")).to_be_visible()
expect(page.locator("#conversation-contents-title")).to_have_text("Used documents")
- expect(documents_entries).to_have_count(2)
+ expect(page.locator("#conversation-contents-subtitle")).to_have_text(
+ "Documents cited in active responses"
+ )
+ expect(documents_entries).to_have_count(1)
expect(documents_entries.nth(0)).to_contain_text(document_title)
expect(documents_entries.nth(0)).to_contain_text(document_file_name)
expect(documents_entries.nth(0)).to_contain_text("Confidential")
- expect(documents_entries.nth(0)).to_contain_text("Pages: 1, 2")
+ expect(documents_entries.nth(0)).to_contain_text("Pages: 2")
expect(documents_entries.nth(0)).to_contain_text(
"group scope: Product Documentation Workspace With A Long Display Name"
)
expect(documents_entries.nth(0)).not_to_contain_text("chunks")
expect(documents_entries.nth(0)).not_to_contain_text("ID:")
- expect(documents_entries.nth(1)).to_contain_text("Release Plan")
- expect(page.locator("#conversation-documents-count")).to_have_text("2")
+ expect(page.locator("#conversation-documents-count")).to_have_text("1")
expect(page.locator("#conversation-documents-panel img[src='x']")).to_have_count(0)
assert page.evaluate("() => window.__documentsPaneXss") is False
_assert_no_horizontal_overflow(
@@ -314,6 +332,39 @@ def test_chat_conversation_contents_drawer(playwright, viewport):
)
_assert_no_horizontal_overflow(documents_entries.nth(0), "document entry")
+ documents_toggle.click()
+ expect(page.locator("#conversation-contents-drawer")).to_be_hidden()
+
+ page.route(
+ "**/api/conversations/conversation-documents-legacy/metadata",
+ lambda route: _fulfill_json(route, {
+ "title": "Legacy documents pane conversation",
+ "tags": source_document_tags,
+ }),
+ )
+ page.evaluate(
+ """
+ () => {
+ window.currentConversationId = 'conversation-documents-legacy';
+ window.dispatchEvent(new CustomEvent('chat:conversation-documents-refresh', {
+ detail: {
+ conversationId: 'conversation-documents-legacy',
+ autoOpen: true
+ }
+ }));
+ }
+ """
+ )
+ expect(documents_entries).to_have_count(2)
+ expect(documents_entries.nth(1)).to_contain_text("Release Plan")
+ expect(page.locator("#conversation-documents-count")).to_have_text("2")
+ expect(page.locator("#conversation-contents-drawer")).to_be_hidden()
+ expect(page.locator("#conversation-contents-subtitle")).to_have_text(
+ "Documents associated with this historical conversation"
+ )
+
+ documents_toggle.click()
+ expect(page.locator("#conversation-contents-drawer")).to_be_visible()
page.locator("#conversation-contents-mode-contents").click()
expect(page.locator("#conversation-contents-title")).to_have_text("Conversation contents")
expect(page.locator("#conversation-contents-panel")).to_be_visible()
diff --git a/ui_tests/test_chat_markdown_citation_lookup_payload.py b/ui_tests/test_chat_markdown_citation_lookup_payload.py
index 4516667b8..b2a5cf5ba 100644
--- a/ui_tests/test_chat_markdown_citation_lookup_payload.py
+++ b/ui_tests/test_chat_markdown_citation_lookup_payload.py
@@ -2,8 +2,8 @@
"""
UI test for markdown citation lookup payloads.
-Version: 0.241.021
-Implemented in: 0.241.021
+Version: 0.250.215
+Implemented in: 0.241.021; Updated in: 0.250.215
This test ensures markdown citation buttons send document, page, and chunk
context to `/api/get_citation` so text citation lookup can recover when the
@@ -108,11 +108,92 @@ def handle_get_citation(route):
},
true
);
+ messagesModule.appendMessage(
+ 'AI',
+ 'Workbook answer (Source: Budget, 2026.xlsx, Sheet: Q1, Final (Approved)) [#budget-doc_Q1]',
+ null,
+ 'assistant-sheet-1',
+ true,
+ [
+ {
+ file_name: 'Budget, 2026.xlsx',
+ document_id: 'budget-doc',
+ citation_id: 'budget-doc_Q1',
+ chunk_id: 'Q1',
+ page_number: 'Q1, Final (Approved)',
+ sheet_name: 'Q1, Final (Approved)',
+ location_label: 'Sheet',
+ location_value: 'Q1, Final (Approved)',
+ },
+ ],
+ [],
+ [],
+ null,
+ null,
+ {
+ id: 'assistant-sheet-1',
+ role: 'assistant',
+ content: 'Workbook answer (Source: Budget, 2026.xlsx, Sheet: Q1, Final (Approved)) [#budget-doc_Q1]',
+ },
+ true
+ );
+ messagesModule.appendMessage(
+ 'AI',
+ 'A (Source: A.pdf, Page: 1). B (Source: B.pdf, Page: 2) [#b-doc_2]',
+ null,
+ 'assistant-adjacent-1',
+ true,
+ [
+ {
+ file_name: 'B.pdf',
+ document_id: 'b-doc',
+ citation_id: 'b-doc_2',
+ chunk_id: '2',
+ page_number: 2,
+ location_label: 'Page',
+ location_value: 2,
+ },
+ ],
+ [],
+ [],
+ null,
+ null,
+ {
+ id: 'assistant-adjacent-1',
+ role: 'assistant',
+ content: 'A (Source: A.pdf, Page: 1). B (Source: B.pdf, Page: 2) [#b-doc_2]',
+ },
+ true
+ );
}
"""
)
message = page.locator('.message[data-message-id="assistant-md-1"]')
+ sheet_message = page.locator(
+ '.message[data-message-id="assistant-sheet-1"]'
+ )
+ sheet_link = sheet_message.locator(
+ 'a.citation-link[data-citation-id="budget-doc_Q1"]'
+ )
+ expect(sheet_link).to_be_visible()
+ expect(sheet_link).to_have_text("Q1, Final (Approved)")
+ expect(sheet_link).to_have_attribute(
+ "data-sheet-name",
+ "Q1, Final (Approved)"
+ )
+ adjacent_message = page.locator(
+ '.message[data-message-id="assistant-adjacent-1"]'
+ )
+ expect(adjacent_message.locator(".message-text")).to_contain_text(
+ "(Source: A.pdf, Page: 1)"
+ )
+ adjacent_link = adjacent_message.locator(
+ 'a.citation-link[data-citation-id="b-doc_2"]'
+ )
+ expect(adjacent_link).to_be_visible()
+ expect(adjacent_link).to_have_text("2")
+
citation_toggle = message.locator('.citation-toggle-btn')
expect(citation_toggle).to_be_visible()
citation_toggle.click()
diff --git a/ui_tests/test_chat_scope_lock_and_conversation_details_escaping.py b/ui_tests/test_chat_scope_lock_and_conversation_details_escaping.py
index e622b94dc..f4cf6e6dc 100644
--- a/ui_tests/test_chat_scope_lock_and_conversation_details_escaping.py
+++ b/ui_tests/test_chat_scope_lock_and_conversation_details_escaping.py
@@ -1,8 +1,8 @@
# test_chat_scope_lock_and_conversation_details_escaping.py
"""
UI test for chat scope-lock and conversation-details escaping.
-Version: 0.241.019
-Implemented in: 0.241.019
+Version: 0.250.215
+Implemented in: 0.241.019; Updated in: 0.250.215
This test ensures malicious workspace names and conversation metadata render as
inert text in the chat scope-lock modal and conversation-details modal.
@@ -92,6 +92,18 @@ def test_chat_scope_lock_and_conversation_details_escape_malicious_metadata(play
"name": document_scope_name,
},
},
+ {
+ "category": "document",
+ "document_id": "doc-2",
+ "title": "Retrieved but not cited",
+ "classification": "Public",
+ "chunk_ids": ["chunk_2_p1"],
+ "scope": {
+ "type": "personal",
+ "id": "user-1",
+ "name": "Personal",
+ },
+ },
{
"category": "semantic",
"value": semantic_tag,
@@ -109,6 +121,22 @@ def test_chat_scope_lock_and_conversation_details_escape_malicious_metadata(play
"value": web_source,
},
],
+ "used_documents_tracking_version": 1,
+ "legacy_used_documents": [],
+ "used_documents": [
+ {
+ "category": "document",
+ "document_id": "doc-1",
+ "title": document_title,
+ "classification": classification_label,
+ "chunk_ids": ["chunk_1_p2"],
+ "scope": {
+ "type": "group",
+ "id": "group-1",
+ "name": document_scope_name,
+ },
+ },
+ ],
"strict": False,
"classification": [classification_label],
"last_updated": "2026-05-05T12:00:00Z",
@@ -202,7 +230,19 @@ def fulfill_empty_docs_or_tags(route):
expect(details_content).to_contain_text(participant_name)
expect(details_content).to_contain_text(participant_email)
expect(details_content).to_contain_text(document_title)
+ expect(details_content).to_contain_text("Retrieved but not cited")
expect(details_content).to_contain_text(document_scope_name)
+ expect(details_content).to_contain_text("Source documents")
+ expect(
+ page.locator(
+ "#conversation-details-modal .conversation-document-cited-badge"
+ )
+ ).to_have_count(1)
+ expect(
+ page.locator(
+ "#conversation-details-modal .conversation-document-cited-badge"
+ )
+ ).to_have_text("Cited")
expect(details_content).to_contain_text(classification_label)
expect(details_content).to_contain_text(semantic_tag)
expect(details_content).to_contain_text(model_tag)