From 02f5c8a4569fe9b132c6db36e09afb15e71a473c Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 19 Aug 2026 17:19:39 -0400 Subject: [PATCH 1/2] Fix generated artifact paging, truncation disclosure, and guidance carry-forward A deployment test surfaced three defects in agent conversations producing CSV artifacts from a truncating telemetry action. The model replied "I cannot create or attach a CSV file in this interface" on the turn that answered a schema clarification, yet the server published the CSV. Version 0.260.008 carried the pending format forward for publication but not for guidance, which both chat paths still resolved from the current user message alone. A clarification answer names no format, so no publication contract ever reached the model. Both paths now resolve guidance through _resolve_generated_file_guidance_format, which falls back to the pending format. A 1,000-row file was produced for a window holding roughly 500 distinct samples. The agent re-requested the same window with an earlier stop time rather than paging forward, and action grouping concatenated both responses. Rows an earlier page of the same action already contributed are now dropped; repeats inside a single response are kept, because the action counted them as distinct records. The source action reported truncated results and the file gave no indication it was partial. Truncation is now detected from the action payload, propagated through the export payload, summary, and artifact metadata for every format, and surfaced as a Partial badge on the artifact card. CSV, DOCX, and PDF guidance now tells the model to page from the last row it holds instead of repeating the range. The telemetry fixture in test_generated_csv_uses_authorized_action_rows derived timestamps from index % 60, so its 900 samples were 60 distinct rows repeated fifteen times, making continuation pages indistinguishable from re-reads. Timestamps now roll over into minutes, matching the fixture's 15-minute window. The per-call row cap belongs to the action's own configuration and is unchanged. --- application/single_app/config.py | 2 +- .../functions_generated_file_exports.py | 130 ++++++- application/single_app/route_backend_chats.py | 25 +- .../static/js/chat/chat-messages.js | 9 + ...ERATED_ARTIFACT_PAGING_AND_GUIDANCE_FIX.md | 167 ++++++++ docs/explanation/fixes/index.md | 1 + ..._generated_artifact_paging_and_guidance.py | 355 ++++++++++++++++++ ...nerated_csv_uses_authorized_action_rows.py | 6 +- 8 files changed, 679 insertions(+), 16 deletions(-) create mode 100644 docs/explanation/fixes/GENERATED_ARTIFACT_PAGING_AND_GUIDANCE_FIX.md create mode 100644 functional_tests/test_generated_artifact_paging_and_guidance.py diff --git a/application/single_app/config.py b/application/single_app/config.py index 60b856bf6..7dab85ab9 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -96,7 +96,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.260.010" +VERSION = "0.260.011" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_generated_file_exports.py b/application/single_app/functions_generated_file_exports.py index aa1f43e79..d4f444fe5 100644 --- a/application/single_app/functions_generated_file_exports.py +++ b/application/single_app/functions_generated_file_exports.py @@ -110,6 +110,16 @@ ) # Below this a stored action reads as a lookup, not the dataset a request is asking for. PRIOR_TURN_SUBSTANTIVE_ROW_COUNT = 10 +# Actions advertise partial results in different ways, so accept the common spellings. +FUNCTION_RESULT_TRUNCATION_KEYS = { + 'truncated', + 'istruncated', + 'wastruncated', + 'resultstruncated', + 'rowstruncated', +} +FUNCTION_RESULT_TRUNCATION_MAX_DEPTH = 3 +FUNCTION_RESULT_TRUNCATION_MAX_ITEMS = 20 FUNCTION_RESULT_CONTROL_KEYS = { 'count', 'detail', @@ -224,6 +234,15 @@ 'do not tell the user to copy or save rows manually, and do not paste a sample of the rows ' 'in place of the artifact.' ) +# Agents commonly re-request an action from the same start time, which re-reads rows already held. +GENERATED_FILE_PAGING_GUIDANCE = ( + 'When an action reports that its results were truncated, request the remaining data with a ' + 'window that starts after the last row you already have instead of repeating the original ' + 'range, and say plainly that the data is partial if you cannot retrieve the rest.' +) +GENERATED_FILE_TRUNCATED_ROWS_NOTE = ( + 'The source action reported truncated results, so this file covers only the rows it returned.' +) def _normalize_question_for_artifact_detection(user_question: str) -> str: @@ -473,7 +492,11 @@ def build_generated_file_output_guidance( clarification_guidance = build_csv_output_clarification_guidance(user_question) return ' '.join( guidance_part - for guidance_part in (clarification_guidance, CSV_PUBLICATION_GUIDANCE) + for guidance_part in ( + clarification_guidance, + CSV_PUBLICATION_GUIDANCE, + GENERATED_FILE_PAGING_GUIDANCE, + ) if guidance_part ) if output_format == 'json': @@ -497,7 +520,7 @@ def build_generated_file_output_guidance( 'evidence. Structured function results from this turn may be included as labeled tables in the ' 'generated file. Do not claim that you cannot create or attach files, do not tell the user to ' 'copy or save the content manually, do not invent rows, and do not claim an attachment exists ' - 'before the file-output finalizer publishes it.' + f'before the file-output finalizer publishes it. {GENERATED_FILE_PAGING_GUIDANCE}' ) return '' @@ -539,11 +562,14 @@ def build_generated_file_export( return None function_rows = extract_authorized_function_result_rows(function_results) row_provenance = ROW_SOURCE_CURRENT_ACTION + rows_truncated = function_results_report_truncated_rows(function_results) if function_rows else False if not assistant_rows and not function_rows and prior_function_results_loader is not None: # This turn answered from evidence gathered earlier, so reuse those rows instead of none. - function_rows = extract_authorized_function_result_rows(prior_function_results_loader()) + prior_function_results = prior_function_results_loader() + function_rows = extract_authorized_function_result_rows(prior_function_results) if function_rows: row_provenance = ROW_SOURCE_EARLIER_ACTION + rows_truncated = function_results_report_truncated_rows(prior_function_results) function_passthrough = evaluate_generated_file_passthrough_eligibility( user_question, rows=function_rows, @@ -566,6 +592,7 @@ def build_generated_file_export( row_source=row_source, assistant_content=assistant_text, passthrough_reason_code=(function_passthrough.get('reason_code') if use_function_rows else None), + rows_truncated=rows_truncated if use_function_rows else False, ) if not assistant_text and not assistant_rows and not function_rows: @@ -589,6 +616,7 @@ def build_generated_file_export( passthrough_reason_code=( function_passthrough.get('reason_code') if row_source in FUNCTION_RESULT_ROW_SOURCES else None ), + rows_truncated=rows_truncated if row_source in FUNCTION_RESULT_ROW_SOURCES else False, ) @@ -605,10 +633,13 @@ def build_structured_artifact_rows_payload( function_rows = extract_authorized_function_result_rows(function_results) row_provenance = ROW_SOURCE_CURRENT_ACTION + rows_truncated = function_results_report_truncated_rows(function_results) if function_rows else False if not function_rows and prior_function_results_loader is not None: - function_rows = extract_authorized_function_result_rows(prior_function_results_loader()) + prior_function_results = prior_function_results_loader() + function_rows = extract_authorized_function_result_rows(prior_function_results) if function_rows: row_provenance = ROW_SOURCE_EARLIER_ACTION + rows_truncated = function_results_report_truncated_rows(prior_function_results) if not function_rows: return None @@ -629,6 +660,7 @@ def build_structured_artifact_rows_payload( 'row_count': len(function_rows), 'row_source': row_provenance, 'passthrough_reason_code': function_passthrough.get('reason_code'), + 'rows_truncated': rows_truncated, } @@ -659,6 +691,8 @@ def _collect_authorized_function_row_groups( ) -> List[Tuple[str, List[Dict[str, Any]]]]: """Group authorized rows by action, so paged calls to one action stay one dataset.""" grouped_rows: Dict[str, List[Dict[str, Any]]] = {} + # Agents often re-page an action from the same start, so pages overlap rather than extend. + seen_group_rows: Dict[str, set] = {} for function_result in function_results or []: if not isinstance(function_result, dict): continue @@ -670,13 +704,81 @@ def _collect_authorized_function_row_groups( ) if not structured_rows: continue - grouped_rows.setdefault( - _get_function_result_label(function_result), - [], - ).extend(structured_rows) + function_label = _get_function_result_label(function_result) + group_rows = grouped_rows.setdefault(function_label, []) + seen_rows = seen_group_rows.setdefault(function_label, set()) + page_signatures = set() + for structured_row in structured_rows: + row_signature = _build_function_result_row_signature(structured_row) + # Repeats inside one response are records the action counted, so only drop rows an + # earlier page of the same action already contributed. + if row_signature is not None: + if row_signature in seen_rows: + continue + page_signatures.add(row_signature) + group_rows.append(structured_row) + seen_rows.update(page_signatures) return list(grouped_rows.items()) +def _build_function_result_row_signature(row: Any) -> Optional[Tuple[Tuple[str, str], ...]]: + if not isinstance(row, dict): + return None + try: + return tuple(sorted( + (_normalize_function_result_key(key), _serialize_row_signature_value(value)) + for key, value in row.items() + )) + except TypeError: + return None + + +def _serialize_row_signature_value(value: Any) -> str: + if isinstance(value, (dict, list, tuple, set)): + return json.dumps(value, default=str, sort_keys=True) + return '' if value is None else str(value) + + +def function_results_report_truncated_rows( + function_results: Optional[List[Dict[str, Any]]], +) -> bool: + """Report whether a row-contributing action said it returned only part of the matching data.""" + for function_result in function_results or []: + if not isinstance(function_result, dict): + continue + if function_result.get('success') is False or _is_tabular_function_result(function_result): + continue + payload = _parse_function_result_payload(function_result.get('function_result')) + if not _extract_function_result_rows(payload): + continue + if _payload_reports_truncation(payload): + return True + return False + + +def _payload_reports_truncation(payload: Any, depth: int = 0) -> bool: + if depth > FUNCTION_RESULT_TRUNCATION_MAX_DEPTH: + return False + if isinstance(payload, dict): + for key, value in payload.items(): + if _normalize_function_result_key(key) not in FUNCTION_RESULT_TRUNCATION_KEYS: + continue + if value is True or str(value).strip().casefold() == 'true': + return True + return any( + _payload_reports_truncation(value, depth + 1) + for value in payload.values() + if isinstance(value, (dict, list)) + ) + if isinstance(payload, list): + return any( + _payload_reports_truncation(item, depth + 1) + for item in payload[:FUNCTION_RESULT_TRUNCATION_MAX_ITEMS] + if isinstance(item, (dict, list)) + ) + return False + + def _select_dominant_function_row_group( function_row_groups: Sequence[Tuple[str, List[Dict[str, Any]]]], ) -> Optional[List[Dict[str, Any]]]: @@ -917,6 +1019,8 @@ def build_generated_file_artifact_metadata( row_source = str(export_payload.get('row_source') or '').strip() if row_source: artifact_metadata['row_source'] = row_source + if export_payload.get('rows_truncated'): + artifact_metadata['rows_truncated'] = True # Surfaced immediately so the participant who asked for the file sees the pending state in # the same response instead of a download button that would be refused. @@ -939,6 +1043,7 @@ def _build_generated_file_payload( assistant_content: str, title: str = '', passthrough_reason_code: Optional[str] = None, + rows_truncated: bool = False, ) -> Dict[str, Any]: normalized_output_format = str(output_format or '').strip().lower() row_count = len(rows or []) @@ -952,12 +1057,14 @@ def _build_generated_file_payload( 'preview_rows': list(rows or [])[:GENERATED_FILE_PREVIEW_ROWS], 'preview_lines': _build_preview_lines(assistant_content), 'row_source': row_source, + 'rows_truncated': bool(rows_truncated), '_structured_rows': list(rows or []), 'summary': _build_generated_file_summary( normalized_output_format, row_count, row_source, normalized_title, + bool(rows_truncated), ), } if passthrough_reason_code: @@ -979,9 +1086,14 @@ def _build_generated_file_summary( row_count: int, row_source: str, title: str, + rows_truncated: bool = False, ) -> str: row_detail = f' with {row_count} structured row(s)' if row_count else '' - return f'Prepared {title}{row_detail} from the {row_source}.' + summary = f'Prepared {title}{row_detail} from the {row_source}.' + if rows_truncated: + # The action capped its own response, so the file is a partial view of the matching data. + summary = f'{summary} {GENERATED_FILE_TRUNCATED_ROWS_NOTE}' + return summary def _build_preview_lines(assistant_content: str) -> List[str]: diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index b0ac2418a..13411322e 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -2371,6 +2371,17 @@ def _resolve_pending_generated_file_format(user_question, conversation_id, user_ ) +def _resolve_generated_file_guidance_format(user_question, conversation_id, user_id): + """Return the artifact format this reply owes, including one carried from a clarification.""" + requested_format = str(get_tabular_generated_output_format(user_question) or '').strip().lower() + if requested_format or not conversation_id: + return requested_format + # A reply that only answers "which rows and columns?" still owes the originally requested file, + # so the publication contract has to reach the model on this turn too. + pending_format = _resolve_pending_generated_file_format(user_question, conversation_id, user_id) + return str(pending_format or '').strip().lower() + + def maybe_create_generated_file_output( user_question, assistant_content, @@ -16278,7 +16289,11 @@ def result_requires_message_reload(result: Any) -> bool: system_messages_for_augmentation = [] # Collect system messages from search generated_file_output_guidance = build_generated_file_output_guidance( user_message, - requested_format=get_tabular_generated_output_format(user_message), + requested_format=_resolve_generated_file_guidance_format( + user_message, + conversation_id, + user_id, + ), ) if generated_file_output_guidance: system_messages_for_augmentation.append({ @@ -20631,9 +20646,11 @@ def stream_cancel_requested(): generated_tabular_outputs_list = [] generated_analysis_artifacts_list = [] system_messages_for_augmentation = [] - requested_streamed_file_format = str( - get_tabular_generated_output_format(user_message) or '' - ).strip().lower() + requested_streamed_file_format = _resolve_generated_file_guidance_format( + user_message, + conversation_id, + user_id, + ) suppress_streamed_file_payload = requested_streamed_file_format in {'json', 'xml'} streamed_file_status_content = _build_streaming_assistant_file_status( requested_streamed_file_format diff --git a/application/single_app/static/js/chat/chat-messages.js b/application/single_app/static/js/chat/chat-messages.js index 5df9d3986..7faec1a20 100644 --- a/application/single_app/static/js/chat/chat-messages.js +++ b/application/single_app/static/js/chat/chat-messages.js @@ -5347,6 +5347,15 @@ function renderReplyQuoteHtml(fullMessageObject = null) { backgroundDetailElements.push(rowCountDetail); } + if (outputMetadata?.rows_truncated) { + // The completed-artifact layout hides the summary, so partial coverage needs its own badge. + const truncatedBadge = document.createElement('span'); + truncatedBadge.className = 'badge text-bg-warning'; + truncatedBadge.textContent = 'Partial'; + truncatedBadge.title = 'The source action reported truncated results, so this file covers only the rows it returned.'; + header.appendChild(truncatedBadge); + } + card.appendChild(header); if (!isCompletedTabularArtifact) { diff --git a/docs/explanation/fixes/GENERATED_ARTIFACT_PAGING_AND_GUIDANCE_FIX.md b/docs/explanation/fixes/GENERATED_ARTIFACT_PAGING_AND_GUIDANCE_FIX.md new file mode 100644 index 000000000..a0d94206d --- /dev/null +++ b/docs/explanation/fixes/GENERATED_ARTIFACT_PAGING_AND_GUIDANCE_FIX.md @@ -0,0 +1,167 @@ +# Generated Artifact Paging, Truncation, and Guidance Carry-Forward Fix + +**Fixed in version: 0.260.011** + +## Issue Description + +A deployment test of the generated-file work from `0.260.004` through `0.260.010` surfaced three +defects in agent conversations that produced CSV artifacts from a Yamcs telemetry action. + +1. **The model denied it could create a file, then the file appeared anyway.** After answering a + schema clarification with "yes and all columns", the assistant replied *"I cannot create or + attach a CSV file in this interface"* and listed the columns as prose. The server published the + CSV regardless, so the user saw a contradiction between the reply and the artifact card. +2. **A 1,000-row file was produced for a window holding roughly 500 distinct samples.** The agent + called `list_parameter_history` twice, and the server concatenated both responses. +3. **A partial file was published with no indication it was partial.** The source action returned + 500 rows and reported `truncated=True`, and the assistant said so in prose, but the artifact card + presented a plain 500-row file. + +## Root Cause Analysis + +### 1. Publication contract resolved, guidance not + +Version `0.260.008` added `resolve_pending_generated_file_format` so a reply that only answers a +clarification still *publishes* the originally requested artifact. Guidance injection was never +updated to match. Both chat paths resolved the guidance format from the current user message alone: + +```python +requested_format=get_tabular_generated_output_format(user_message) +``` + +"yes and all columns" contains no format keyword, so `build_generated_file_output_guidance` returned +an empty string and no system message was added. The model was never told that the server attaches +the file, so it fell back to its default belief that it cannot produce attachments — while the +finalizer, which *did* carry the format forward, published the CSV. + +### 2. Overlapping pages concatenated + +Version `0.260.008` grouped rows by action so paged calls to one action stayed one dataset, using +`.extend()`. The deployment log shows the agent did not page forward; it re-requested the same +window with an earlier stop time: + +``` +list_parameter_history start=20:40:57.445 stop=20:55:57.445 row_count=500 truncated=True +list_parameter_history start=20:40:57.445 stop=20:49:17.445 row_count=500 truncated=True +``` + +Both calls share a start time, and both were capped at the action's own `max_rows` limit, so the +second response returned the same first 500 samples. Concatenation produced 1,000 rows for roughly +500 distinct samples. The same pattern explains the earlier "901 acquired samples" claim +(500 truncated + 401 overlapping). + +### 3. Truncation signal discarded + +Action payloads carried `truncated: true`, but nothing read it. `_assistant_content_disclaims_complete_file` +existed for JSON and XML only and matched assistant prose rather than the action's own report, so no +format surfaced source truncation on the artifact. + +The 500-row cap itself is the action's configured `max_rows`, not a SimpleChat limit. The fix makes +the cap visible and teaches the model to page past it rather than silently changing plugin behavior. + +## Technical Details + +### Files Modified + +| File | Change | +|------|--------| +| `application/single_app/functions_generated_file_exports.py` | Cross-page row de-duplication, truncation detection and propagation, paging guidance | +| `application/single_app/route_backend_chats.py` | `_resolve_generated_file_guidance_format` helper wired into both chat paths | +| `application/single_app/static/js/chat/chat-messages.js` | `Partial` badge on the generated artifact card | +| `application/single_app/config.py` | Version bump to `0.260.011` | +| `functional_tests/test_generated_csv_uses_authorized_action_rows.py` | Telemetry fixture now uses distinct sample times | + +### Code Changes + +**Guidance carry-forward.** A new route helper resolves the format the reply owes, falling back to the +pending clarification when the message itself names no format: + +```python +def _resolve_generated_file_guidance_format(user_question, conversation_id, user_id): + requested_format = str(get_tabular_generated_output_format(user_question) or '').strip().lower() + if requested_format or not conversation_id: + return requested_format + pending_format = _resolve_pending_generated_file_format(user_question, conversation_id, user_id) + return str(pending_format or '').strip().lower() +``` + +Both the streaming and non-streaming chat paths now call it. In the streaming path it also drives the +payload-suppression decision and the "generating file" status banner, so the answer turn behaves like +the turn that made the original request. + +**Cross-page de-duplication.** `_collect_authorized_function_row_groups` now tracks a row signature per +action label. Repeats *inside* one response are kept, because the action counted them as distinct +records; rows an *earlier page of the same action* already contributed are dropped: + +```python +page_signatures = set() +for structured_row in structured_rows: + row_signature = _build_function_result_row_signature(structured_row) + if row_signature is not None: + if row_signature in seen_rows: + continue + page_signatures.add(row_signature) + group_rows.append(structured_row) +seen_rows.update(page_signatures) +``` + +**Truncation disclosure.** `function_results_report_truncated_rows` scans row-contributing action +payloads for `truncated`, `is_truncated`, `was_truncated`, `results_truncated`, or `rows_truncated`. +The flag flows into the export payload, the summary text, and the artifact metadata, and is captured +separately for reach-back rows so a file built from an earlier turn keeps that turn's signal. + +**Paging guidance.** CSV, DOCX, and PDF guidance now includes: + +> When an action reports that its results were truncated, request the remaining data with a window +> that starts after the last row you already have instead of repeating the original range, and say +> plainly that the data is partial if you cannot retrieve the rest. + +JSON and XML are excluded because their guidance requires a payload-only reply. + +### Fixture Correction + +`TELEMETRY_ROWS` in `test_generated_csv_uses_authorized_action_rows.py` previously derived timestamps +from `index % 60`, so its 900 "samples" were only 60 distinct rows repeated fifteen times. That made +genuine continuation pages indistinguishable from re-reads. Timestamps now roll over into minutes, +which matches the 15-minute high-granularity window the fixture's question asks for. + +## Validation + +### Test Results + +| Suite | Result | +|---|---| +| `test_generated_artifact_paging_and_guidance.py` | 11/11 | +| `test_generated_csv_uses_authorized_action_rows.py` | 15/15 | +| `test_generated_structured_artifact_parity.py` | 8/8 | +| `test_assistant_table_csv_artifact.py` | 36/36 | +| `test_generated_json_xml_exports.py` | 7/7 | +| `test_tabular_passthrough_heterogeneous_rows.py` | 4/4 | +| `test_tabular_background_generated_exports.py` | 8/8 | +| `test_tabular_row_orchestration_scale.py` | 15/15 | +| `route_tests/test_route_blueprint_policy_inventory.py` | 6/6 | +| `route_tests/test_route_unauthenticated_policy_contract.py` | 4/4 | + +### Before and After + +| Scenario | Before | After | +|---|---|---| +| Answering a schema clarification | No file guidance injected; model denies it can attach files, server publishes anyway | Publication contract injected; reply and artifact agree | +| Two calls re-reading one window | 1,000 rows, roughly half duplicated | Distinct rows only | +| Two calls paging forward | Both pages kept | Both pages kept (unchanged) | +| Action reports `truncated=True` | Silent, file looks complete | `Partial` badge, summary note, `rows_truncated` on the artifact | +| Repeated identical rows in one response | All kept | All kept (unchanged) | + +## Known Limitations + +- De-duplication compares full row values. A continuation page whose rows differ only in a field the + action omits from its response would still be treated as a repeat. +- The paging instruction is model guidance, not enforcement. An agent that ignores it still produces + a truncated dataset, but the file is now labeled partial. +- The per-call row cap belongs to the action's own configuration. Raising it is a plugin + configuration change, not an application change. + +## Related Documentation + +- `docs/explanation/fixes/GENERATED_JSON_XML_EXPORTS_FIX.md` +- `docs/explanation/fixes/GENERATED_STRUCTURED_ARTIFACT_INTENT_FIX.md` diff --git a/docs/explanation/fixes/index.md b/docs/explanation/fixes/index.md index d81b36e2a..201728c95 100644 --- a/docs/explanation/fixes/index.md +++ b/docs/explanation/fixes/index.md @@ -35,3 +35,4 @@ category: Version History - [Chat Citation Whitespace Collapse Fix](CHAT_CITATION_WHITESPACE_COLLAPSE_FIX.md) - [New Chat Conversation Documents Drawer Reset Fix](NEW_CHAT_CONVERSATION_DOCUMENTS_DRAWER_RESET_FIX.md) - [Collaboration Mention Tab Autocomplete Fix](COLLABORATION_MENTION_TAB_AUTOCOMPLETE_FIX.md) +- [Generated Artifact Paging, Truncation, and Guidance Carry-Forward Fix](GENERATED_ARTIFACT_PAGING_AND_GUIDANCE_FIX.md) diff --git a/functional_tests/test_generated_artifact_paging_and_guidance.py b/functional_tests/test_generated_artifact_paging_and_guidance.py new file mode 100644 index 000000000..69ac4e288 --- /dev/null +++ b/functional_tests/test_generated_artifact_paging_and_guidance.py @@ -0,0 +1,355 @@ +# test_generated_artifact_paging_and_guidance.py +#!/usr/bin/env python3 +""" +Functional test for paged action rows, truncation disclosure, and carried guidance. +Version: 0.260.011 +Implemented in: 0.260.011 + +A deployment test produced three defects that the log confirmed: + + 1. An agent answered "yes and all columns" to a schema clarification, replied + "I cannot create or attach a CSV file in this interface", and the server + published the CSV anyway. The publication contract was resolved from the + current user message only, so the answer turn injected no guidance at all + even though 0.260.008 already carried the format forward for publishing. + 2. Two `list_parameter_history` calls in one turn both started at the same + timestamp, so the second page re-read rows the first page already held. + The action grouping added in 0.260.008 concatenated them, producing a + 1,000-row file for a window that held roughly 500 distinct samples. + 3. Those same calls reported `truncated=True`, yet the published file gave no + indication that it covered only part of the matching data. + +This test ensures overlapping pages collapse to distinct rows, truncated source +results are disclosed on the artifact, and the guidance format falls back to the +clarification the reply is answering. +""" + +import sys +import traceback +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +APP_DIR = ROOT / 'application' / 'single_app' +if str(APP_DIR) not in sys.path: + sys.path.insert(0, str(APP_DIR)) +if str(Path(__file__).resolve().parent) not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from test_support.versioning import assert_app_version_at_least # noqa: E402 + +from functions_generated_file_exports import ( # noqa: E402 + GENERATED_FILE_PAGING_GUIDANCE, + GENERATED_FILE_TRUNCATED_ROWS_NOTE, + build_generated_file_artifact_metadata, + build_generated_file_export, + build_generated_file_output_guidance, + build_structured_artifact_rows_payload, + extract_authorized_function_result_rows, + function_results_report_truncated_rows, +) + + +def _telemetry_rows(start_index, count): + return [ + { + 'generation_time': f'2026-08-19T20:40:{index % 60:02d}.000Z', + 'eng_value': round(11.0 + (index * 0.001), 4), + 'raw_value': index, + 'monitoring_result': 'IN_LIMITS', + } + for index in range(start_index, start_index + count) + ] + + +def _history_call(rows, truncated): + return { + 'success': True, + 'plugin_name': 'yamcs_plugin', + 'function_name': 'list_parameter_history', + 'function_result': { + 'instance': 'simulator', + 'row_count': len(rows), + 'truncated': truncated, + 'rows': rows, + }, + } + + +def test_overlapping_pages_collapse_to_distinct_rows(): + """Re-reading an action from the same start must not double the dataset.""" + print("Testing overlapping paged action results...") + try: + full_page = _telemetry_rows(0, 500) + # The agent narrowed only the stop time, so this page repeats the first 300 rows. + overlapping_page = _telemetry_rows(0, 300) + rows = extract_authorized_function_result_rows([ + _history_call(full_page, True), + _history_call(overlapping_page, False), + ]) + + assert len(rows) == 500, f'expected 500 distinct rows, got {len(rows)}' + signatures = {tuple(sorted(row.items())) for row in rows} + assert len(signatures) == 500, 'distinct row signatures were not preserved' + print("Test passed!") + return True + except Exception as e: + print(f"Test failed: {e}") + traceback.print_exc() + return False + + +def test_non_overlapping_pages_still_combine(): + """Genuine continuation pages must still extend the dataset.""" + print("Testing non-overlapping paged action results...") + try: + rows = extract_authorized_function_result_rows([ + _history_call(_telemetry_rows(0, 500), True), + _history_call(_telemetry_rows(500, 401), False), + ]) + + assert len(rows) == 901, f'expected 901 combined rows, got {len(rows)}' + print("Test passed!") + return True + except Exception as e: + print(f"Test failed: {e}") + traceback.print_exc() + return False + + +def test_truncated_source_is_reported(): + """A capped action result must be recognizable as partial.""" + print("Testing truncation detection...") + try: + assert function_results_report_truncated_rows([ + _history_call(_telemetry_rows(0, 500), True), + ]) is True, 'truncated result was not detected' + assert function_results_report_truncated_rows([ + _history_call(_telemetry_rows(0, 401), False), + ]) is False, 'complete result was reported as truncated' + assert function_results_report_truncated_rows([]) is False, 'empty results reported truncation' + print("Test passed!") + return True + except Exception as e: + print(f"Test failed: {e}") + traceback.print_exc() + return False + + +def test_csv_export_discloses_truncated_rows(): + """The CSV artifact must say the source action returned only part of the data.""" + print("Testing CSV truncation disclosure...") + try: + export_payload = build_generated_file_export( + 'convert the parameter history to csv without changes', + 'Here is the telemetry export.', + function_results=[_history_call(_telemetry_rows(0, 500), True)], + ) + + assert export_payload is not None, 'no CSV export was produced' + assert export_payload['row_count'] == 500, f"unexpected row count {export_payload['row_count']}" + assert export_payload['rows_truncated'] is True, 'truncation flag missing from payload' + assert GENERATED_FILE_TRUNCATED_ROWS_NOTE in export_payload['summary'], ( + 'summary did not disclose the truncation' + ) + + artifact_metadata = build_generated_file_artifact_metadata( + export_payload, + {'message': {'id': 'artifact-1', 'file_name': export_payload['file_name']}}, + 'conversation-1', + ) + assert artifact_metadata is not None, 'no artifact metadata was produced' + assert artifact_metadata.get('rows_truncated') is True, 'truncation flag missing from artifact' + print("Test passed!") + return True + except Exception as e: + print(f"Test failed: {e}") + traceback.print_exc() + return False + + +def test_complete_export_is_not_marked_partial(): + """A complete action result must not be labeled partial.""" + print("Testing complete export labeling...") + try: + export_payload = build_generated_file_export( + 'convert the parameter history to csv without changes', + 'Here is the telemetry export.', + function_results=[_history_call(_telemetry_rows(0, 401), False)], + ) + + assert export_payload is not None, 'no CSV export was produced' + assert export_payload['rows_truncated'] is False, 'complete export was flagged as truncated' + assert GENERATED_FILE_TRUNCATED_ROWS_NOTE not in export_payload['summary'], ( + 'complete export summary claimed truncation' + ) + print("Test passed!") + return True + except Exception as e: + print(f"Test failed: {e}") + traceback.print_exc() + return False + + +def test_structured_payload_reports_truncation(): + """JSON and XML artifacts must carry the same truncation signal as CSV.""" + print("Testing structured artifact truncation signal...") + try: + for output_format in ('json', 'xml'): + payload = build_structured_artifact_rows_payload( + f'convert the parameter history to {output_format} without changes', + output_format, + function_results=[_history_call(_telemetry_rows(0, 500), True)], + ) + assert payload is not None, f'no {output_format} payload was produced' + assert payload['rows_truncated'] is True, f'{output_format} payload lost the truncation flag' + print("Test passed!") + return True + except Exception as e: + print(f"Test failed: {e}") + traceback.print_exc() + return False + + +def test_reachback_rows_carry_prior_truncation(): + """Rows recovered from an earlier turn must keep that turn's truncation signal.""" + print("Testing reach-back truncation signal...") + try: + prior_results = [_history_call(_telemetry_rows(0, 500), True)] + export_payload = build_generated_file_export( + 'convert those results to csv without changes', + 'Using the telemetry already retrieved.', + function_results=[], + prior_function_results_loader=lambda: prior_results, + ) + + assert export_payload is not None, 'no CSV export was produced from prior rows' + assert export_payload['row_source'] == 'earlier action result', ( + f"unexpected row source {export_payload['row_source']}" + ) + assert export_payload['rows_truncated'] is True, 'reach-back lost the truncation flag' + print("Test passed!") + return True + except Exception as e: + print(f"Test failed: {e}") + traceback.print_exc() + return False + + +def test_guidance_carries_pending_format(): + """A clarification answer must still receive the publication contract.""" + print("Testing carried guidance format...") + try: + answer_guidance = build_generated_file_output_guidance( + 'yes and all columns', + requested_format='csv', + ) + assert 'cannot create or attach files' in answer_guidance, ( + 'carried guidance omitted the publication contract' + ) + + # Without a carried format the answer turn has no artifact contract to state. + assert build_generated_file_output_guidance('yes and all columns') == '', ( + 'guidance was emitted without a requested or pending format' + ) + print("Test passed!") + return True + except Exception as e: + print(f"Test failed: {e}") + traceback.print_exc() + return False + + +def test_paging_guidance_reaches_prose_formats(): + """Formats that can explain themselves must be told how to page past truncation.""" + print("Testing paging guidance coverage...") + try: + for output_format in ('csv', 'docx', 'pdf'): + guidance = build_generated_file_output_guidance( + f'create a {output_format} of the parameter history', + ) + assert GENERATED_FILE_PAGING_GUIDANCE in guidance, ( + f'{output_format} guidance omitted the paging instruction' + ) + + # JSON and XML must return only the payload, so prose paging advice would conflict. + for output_format in ('json', 'xml'): + guidance = build_generated_file_output_guidance( + f'create a {output_format} of the parameter history', + ) + assert GENERATED_FILE_PAGING_GUIDANCE not in guidance, ( + f'{output_format} guidance added prose that conflicts with payload-only output' + ) + print("Test passed!") + return True + except Exception as e: + print(f"Test failed: {e}") + traceback.print_exc() + return False + + +def test_route_resolves_guidance_format_from_clarification(): + """The chat routes must resolve guidance from the pending format, not the raw message.""" + print("Testing route guidance format resolution...") + try: + route_source = (APP_DIR / 'route_backend_chats.py').read_text(encoding='utf-8') + + assert route_source.count('_resolve_generated_file_guidance_format(') == 3, ( + 'expected one helper definition and both chat paths to use it' + ) + assert 'requested_format=get_tabular_generated_output_format(user_message)' not in route_source, ( + 'a guidance call still resolves the format from the current message only' + ) + print("Test passed!") + return True + except Exception as e: + print(f"Test failed: {e}") + traceback.print_exc() + return False + + +def test_artifact_card_renders_the_partial_badge(): + """The completed-artifact card hides the summary, so truncation needs a visible badge.""" + print("Testing artifact card truncation badge...") + try: + card_source = ( + APP_DIR / 'static' / 'js' / 'chat' / 'chat-messages.js' + ).read_text(encoding='utf-8') + + assert "outputMetadata?.rows_truncated" in card_source, ( + 'the artifact card never reads the truncation flag' + ) + assert "truncatedBadge.textContent = 'Partial'" in card_source, ( + 'the artifact card does not render a partial badge' + ) + print("Test passed!") + return True + except Exception as e: + print(f"Test failed: {e}") + traceback.print_exc() + return False + + +if __name__ == "__main__": + assert_app_version_at_least("0.260.011") + + tests = [ + test_overlapping_pages_collapse_to_distinct_rows, + test_non_overlapping_pages_still_combine, + test_truncated_source_is_reported, + test_csv_export_discloses_truncated_rows, + test_complete_export_is_not_marked_partial, + test_structured_payload_reports_truncation, + test_reachback_rows_carry_prior_truncation, + test_guidance_carries_pending_format, + test_paging_guidance_reaches_prose_formats, + test_route_resolves_guidance_format_from_clarification, + test_artifact_card_renders_the_partial_badge, + ] + + results = [] + for test in tests: + print(f"\nRunning {test.__name__}...") + results.append(test()) + + print(f"\nResults: {sum(results)}/{len(results)} tests passed") + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_generated_csv_uses_authorized_action_rows.py b/functional_tests/test_generated_csv_uses_authorized_action_rows.py index eba77f64d..698327642 100644 --- a/functional_tests/test_generated_csv_uses_authorized_action_rows.py +++ b/functional_tests/test_generated_csv_uses_authorized_action_rows.py @@ -62,8 +62,10 @@ ) TELEMETRY_ROWS = [ { - 'generation_time_utc': f'2026-08-19T15:37:{index % 60:02d}.371000+00:00', - 'reception_time_utc': f'2026-08-19T15:37:{index % 60:02d}.372000+00:00', + # High-granularity telemetry carries a distinct acquisition time per sample, which is what + # lets 0.260.011 tell a genuine continuation page apart from a re-read of the same window. + 'generation_time_utc': f'2026-08-19T15:{37 + index // 60:02d}:{index % 60:02d}.371000+00:00', + 'reception_time_utc': f'2026-08-19T15:{37 + index // 60:02d}:{index % 60:02d}.372000+00:00', 'battery_voltage_1_v': 27 + (index % 5), 'raw_value': 27 + (index % 5), 'monitoring_result': 'CRITICAL', From e32453c3a213eefcb9dc2b437950b45dacd65e5f Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 19 Aug 2026 17:39:25 -0400 Subject: [PATCH 2/2] Add consolidated release note for generated file output work (Refs #1071) --- docs/explanation/release_notes.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 221762321..d95b7aaf2 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,24 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](/explanation/features/) and [Fixes by Version](/explanation/fixes/). +### **(v0.260.011)** + +#### Bug Fixes + +* **Reliable File Generation From Agent Action Results** + * Asking an agent for a downloadable file built from action results now produces the complete dataset in the requested format. Previously these requests could fail outright, publish a three-row sample of a large result, overwrite the assistant's written answer, or return nothing at all. Delivered across v0.260.004 through v0.260.011. + * **Files no longer fail to generate.** A CSV built from several actions in one turn could stop with `Generated output schema mismatch at row 2`, because each action returned a different set of columns. The export now pins a union of every column before the run starts and pads the missing cells, so mixed-shape results serialize instead of failing. + * **The written answer is no longer replaced by the file card.** CSV replies were suppressed alongside JSON and XML, but only JSON and XML withhold their payload from the response. CSV, DOCX, and PDF now keep the assistant's answer and append the file card beneath it. + * **Files contain the retrieved data, not a sample of it.** When the assistant pasted a few example rows above its answer, that excerpt outranked the real result set, producing a 3-row file from a 900-row query. Pasted rows are now used only when they are not an excerpt of the data actually retrieved. + * **Discovery calls no longer dilute the dataset.** A turn that lists instances, lists parameters, then retrieves history used to blend all three into one file. Rows are grouped by the action that produced them, and the action holding the substantive dataset wins. + * **Follow-up requests reuse data already gathered.** Asking "now make that a CSV" after the data was retrieved in an earlier turn no longer returns an empty result. The export reaches back through stored conversation citations, bounded by the **conversation history limit** in Admin Settings, and reuses the rows already collected instead of re-querying the source. + * **Answering a clarifying question now delivers the file.** When the assistant asks which rows and columns to include, replying "yes, all columns" now publishes the file that was originally requested. The clarification turn itself no longer publishes a placeholder file built from the question text. + * **The assistant no longer claims it cannot create files.** Every format now states the publication contract to the model, including on the turn that only answers a clarification, so replies stop saying "I cannot create or attach a file in this interface" and then producing one anyway. + * **Overlapping result pages no longer double the row count.** Agents frequently re-request a range from the same start time rather than paging forward, which produced a 1,000-row file for a window holding roughly 500 distinct records. Rows an earlier page of the same action already returned are dropped, while genuinely repeated records inside a single response are preserved. + * **Partial data is now labeled.** When an action reports that it truncated its own results, the file carries a **Partial** badge and a note explaining that it covers only the rows the action returned. Agents are also instructed to request the remainder starting after the last row they already hold, rather than repeating the original range. + * **CSV, DOCX, PDF, JSON, and XML now behave identically.** All five formats resolve rows the same way, reach back to earlier turns, decline to publish on a clarification turn, and report truncation. + * (Ref: `functions_generated_file_exports.py`, `functions_tabular_generated_exports.py`, `route_backend_chats.py`, `chat-messages.js`, [Generated Artifact Paging, Truncation, and Guidance Carry-Forward Fix](fixes/GENERATED_ARTIFACT_PAGING_AND_GUIDANCE_FIX.md), Refs #1071) + ### **(v0.260.006)** #### New Features