Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@
"audio_files_authentication_type": "key",
"audio_files_key": "",
"enable_content_safety": false,
"content_safety_violation_message": "Your message was blocked by Content Safety.",
"content_safety_include_trigger_information": true,
"require_member_of_safety_violation_admin": false,
"content_safety_endpoint": "",
"content_safety_key": "",
Expand Down
2 changes: 1 addition & 1 deletion application/single_app/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@
EXECUTOR_TYPE = 'thread'
EXECUTOR_MAX_WORKERS = 30
SESSION_TYPE = 'filesystem'
VERSION = "0.250.059"
VERSION = "0.250.061"
IS_DEVELOPMENT = is_development_env_enabled()

SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax')
Expand Down
64 changes: 64 additions & 0 deletions application/single_app/functions_content_safety.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
# functions_content_safety.py
"""Helpers for formatting user-visible Content Safety violation messages."""

CONTENT_SAFETY_VIOLATION_MESSAGE_DEFAULT = 'Your message was blocked by Content Safety.'
CONTENT_SAFETY_VIOLATION_MESSAGE_MAX_LENGTH = 3000


def normalize_content_safety_violation_message(value):
"""Return a non-empty, bounded Markdown message template."""

Check warning on line 9 in application/single_app/functions_content_safety.py

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains dynamic execution, persistence, or system access marker. Recommendation%3A Do not execute changed lifecycle scripts or installers while this finding is unresolved.
candidate = str(value or '').replace('\r\n', '\n').replace('\r', '\n').strip()
if not candidate:
return CONTENT_SAFETY_VIOLATION_MESSAGE_DEFAULT
return candidate[:CONTENT_SAFETY_VIOLATION_MESSAGE_MAX_LENGTH]


def build_content_safety_violation_message(
settings,
block_reasons,
triggered_categories,
blocklist_matches,
):
"""Build the Markdown message shown after a blocked chat request."""
safety_settings = settings if isinstance(settings, dict) else {}
message_template = normalize_content_safety_violation_message(
safety_settings.get('content_safety_violation_message')
)

if safety_settings.get('content_safety_include_trigger_information', True) is False:
return message_template

normalized_reasons = [
str(reason).strip()
for reason in (block_reasons or [])
if str(reason or '').strip()
]
trigger_lines = [
f"**Reason**: {', '.join(normalized_reasons)}",
'Triggered categories:',
]

for category in triggered_categories or []:
if not isinstance(category, dict):
continue
category_name = str(category.get('category') or '').strip()
if category_name:
trigger_lines.append(
f" - {category_name} (severity={category.get('severity')})"
)

formatted_blocklist_matches = []
for match in blocklist_matches or []:
if not isinstance(match, dict):
continue
item_text = str(match.get('blocklistItemText') or '').strip()
blocklist_name = str(match.get('blocklistName') or '').strip()
if item_text and blocklist_name:
formatted_blocklist_matches.append(
f" - {item_text} (in {blocklist_name})"
)

if formatted_blocklist_matches:
trigger_lines.extend(['', 'Blocklist Matches:', *formatted_blocklist_matches])

return f"{message_template}\n\n" + '\n'.join(trigger_lines)
5 changes: 5 additions & 0 deletions application/single_app/functions_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@

from config import *
from functions_appinsights import log_event
from functions_content_safety import (
CONTENT_SAFETY_VIOLATION_MESSAGE_DEFAULT,
)
from functions_cosmos_throughput import get_default_cosmos_throughput_settings
from functions_document_actions import get_default_document_action_capabilities
from functions_icon_utils import normalize_icon_payload
Expand Down Expand Up @@ -1063,6 +1066,8 @@ def get_settings(use_cosmos=False, include_source=False):

# Safety (Content Safety) Settings
'enable_content_safety': False,
'content_safety_violation_message': CONTENT_SAFETY_VIOLATION_MESSAGE_DEFAULT,
'content_safety_include_trigger_information': True,
'require_member_of_safety_violation_admin': False,
'require_member_of_control_center_admin': False,
'require_member_of_control_center_dashboard_reader': False,
Expand Down
40 changes: 11 additions & 29 deletions application/single_app/route_backend_chats.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
SEMANTIC_SEARCH_QUOTA_WARNING_TYPE,
SemanticSearchQuotaExceededError,
)
from functions_content_safety import build_content_safety_violation_message
from functions_settings import *
from functions_assigned_knowledge import (
ASSIGNED_KNOWLEDGE_USER_ACTION_ANALYZE,
Expand Down Expand Up @@ -2707,7 +2708,6 @@ def _build_safety_message_doc(
},
})


def _build_fact_memory_context_lines(
scope_id,
scope_type,
Expand Down Expand Up @@ -13410,21 +13410,12 @@ def result_requires_message_reload(result: Any) -> bool:
cosmos_safety_container.upsert_item(safety_item)

# Instead of 403, we'll add a "safety" message
blocked_msg_content = (
"Your message was blocked by Content Safety.\n\n"
f"**Reason**: {', '.join(block_reasons)}\n"
"Triggered categories:\n"
blocked_msg_content = build_content_safety_violation_message(
settings=settings,
block_reasons=block_reasons,
triggered_categories=triggered_categories,
blocklist_matches=blocklist_matches,
)
for cat in triggered_categories:
blocked_msg_content += (
f" - {cat['category']} (severity={cat['severity']})\n"
)
if blocklist_matches:
blocked_msg_content += (
"\nBlocklist Matches:\n" +
"\n".join([f" - {m['blocklistItemText']} (in {m['blocklistName']})"
for m in blocklist_matches])
)

# Insert a special "role": "safety" or "blocked"
safety_doc = _build_safety_message_doc(
Expand Down Expand Up @@ -17039,21 +17030,12 @@ def record_and_publish_streaming_thought(thought_payload):
cosmos_safety_container.upsert_item(safety_item)

# Build blocked message
blocked_msg_content = (
"Your message was blocked by Content Safety.\n\n"
f"**Reason**: {', '.join(block_reasons)}\n"
"Triggered categories:\n"
blocked_msg_content = build_content_safety_violation_message(
settings=settings,
block_reasons=block_reasons,
triggered_categories=triggered_categories,
blocklist_matches=blocklist_matches,
)
for cat in triggered_categories:
blocked_msg_content += (
f" - {cat['category']} (severity={cat['severity']})\n"
)
if blocklist_matches:
blocked_msg_content += (
"\nBlocklist Matches:\n" +
"\n".join([f" - {m['blocklistItemText']} (in {m['blocklistName']})"
for m in blocklist_matches])
)

# Insert safety message
safety_doc = _build_safety_message_doc(
Expand Down
10 changes: 10 additions & 0 deletions application/single_app/route_frontend_admin_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

from functions_keyvault import keyvault_model_endpoint_cleanup_helper, keyvault_model_endpoint_delete_helper, keyvault_model_endpoint_save_helper, redact_model_endpoint_secret_values
from functions_settings import *
from functions_content_safety import normalize_content_safety_violation_message
from functions_file_sync import FILE_SYNC_DEFAULTS, get_file_sync_config
from functions_source_review import SOURCE_REVIEW_DEFAULTS, get_source_review_config, get_source_review_runtime_capabilities, normalize_source_review_js_rendering_enabled, parse_source_review_list
from functions_control_center import (
Expand Down Expand Up @@ -126,6 +127,7 @@ def normalize_agents_page_text(value, fallback, max_length):
candidate = fallback
return candidate[:max_length]


def register_route_frontend_admin_settings(bp):
@bp.route('/admin/settings', methods=['GET', 'POST'])
@swagger_route(security=get_auth_security())
Expand Down Expand Up @@ -720,6 +722,12 @@ def parse_admin_int(raw_value, fallback_value, field_name="unknown", hard_defaul
file_download_allowed_public_workspace_ids = normalize_file_download_allowed_public_workspace_ids(
form_data.get('file_download_allowed_public_workspace_ids', '')
)
content_safety_violation_message = normalize_content_safety_violation_message(
form_data.get('content_safety_violation_message')
)
content_safety_include_trigger_information = form_data.get(
'content_safety_include_trigger_information'
) == 'on'
require_member_of_safety_violation_admin = form_data.get('require_member_of_safety_violation_admin') == 'on'
require_member_of_control_center_admin = form_data.get('require_member_of_control_center_admin') == 'on'
require_member_of_control_center_dashboard_reader = form_data.get('require_member_of_control_center_dashboard_reader') == 'on'
Expand Down Expand Up @@ -2105,6 +2113,8 @@ def is_valid_url(url):

# Safety (Content Safety Direct & APIM)
'enable_content_safety': form_data.get('enable_content_safety') == 'on',
'content_safety_violation_message': content_safety_violation_message,
'content_safety_include_trigger_information': content_safety_include_trigger_information,
'content_safety_endpoint': form_data.get('content_safety_endpoint', '').strip(),
'content_safety_key': admin_secret('content_safety_key'),
'content_safety_authentication_type': form_data.get('content_safety_authentication_type', 'key'),
Expand Down
82 changes: 82 additions & 0 deletions application/single_app/templates/admin_settings.html
Original file line number Diff line number Diff line change
Expand Up @@ -9062,6 +9062,37 @@ <h5>
</div>
</div>


<div class="mb-3">
<label for="content_safety_violation_message" class="form-label">
Safety Violation Message <span class="text-muted fw-normal">(Markdown supported)</span>
</label>
<textarea
class="form-control"
id="content_safety_violation_message"
name="content_safety_violation_message"
rows="4"
maxlength="3000"
placeholder="Your message was blocked by Content Safety."
>{{ settings.content_safety_violation_message or '' }}</textarea>
<div class="form-text">Displayed when Content Safety blocks a chat message.</div>
</div>

<div class="form-group form-check form-switch mb-2">
<input
type="checkbox"
class="form-check-input"
id="content_safety_include_trigger_information"
name="content_safety_include_trigger_information"
{% if settings.content_safety_include_trigger_information %}checked{% endif %}
>
<label class="form-check-label ms-2" for="content_safety_include_trigger_information">
Include Trigger Information
</label>
<i class="bi bi-info-circle ms-2" data-bs-toggle="tooltip" title="Show the reason, detected categories and severities, and blocklist matches below the safety message."></i>

Check warning on line 9092 in application/single_app/templates/admin_settings.html

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains AI, plugin, agent, or workspace boundary marker. Recommendation%3A Check whether prompts, chat history, uploaded documents, embeddings, citations, settings, or identity can cross a new boundary.
</div>
<p class="form-text text-muted mb-3">Disable this option to show only the configured message.</p>

Check warning on line 9094 in application/single_app/templates/admin_settings.html

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains security control, sanitization, or audit marker. Recommendation%3A Confirm the change does not weaken auth, CSRF, CSP, XSS defenses, settings sanitization, redaction, audit logging, or tests.

<button type="button" class="btn btn-secondary mt-3" id="test_safety_button">
Test Safety Connection
</button>
Expand Down Expand Up @@ -11129,6 +11160,7 @@ <h5 class="modal-title" id="userAgreementPreviewModalLabel">
// Existing SimpleMDE script...
// We’ll store the SimpleMDE instance globally so we can destroy or read its content
let simplemde = null;
let contentSafetyViolationMessageEditor = null;

// Grab references to the toggle, the editor container, and the preview container
const toggleLandingPageEditor = document.getElementById("toggleLandingPageEditor");
Expand Down Expand Up @@ -11186,6 +11218,56 @@ <h5 class="modal-title" id="userAgreementPreviewModalLabel">
showPreview();
}
}

function initializeContentSafetyViolationMessageEditor() {
if (contentSafetyViolationMessageEditor || typeof SimpleMDE === "undefined") {
return;
}

const contentSafetyViolationMessageInput = document.getElementById(
"content_safety_violation_message"
);
if (!contentSafetyViolationMessageInput) {
return;
}

try {
contentSafetyViolationMessageEditor = new SimpleMDE({
element: contentSafetyViolationMessageInput,
spellChecker: false,
autoDownloadFontAwesome: false,
forceSync: true
});
contentSafetyViolationMessageEditor.codemirror.on("change", () => {
if (typeof window.markFormAsModified === "function") {
window.markFormAsModified();
}
});
} catch (error) {
console.error("Failed to initialize SimpleMDE for Content Safety violation messages:", error);
}
}

function refreshContentSafetyViolationMessageEditor() {
if (!contentSafetyViolationMessageEditor?.codemirror) {
return;
}

window.setTimeout(() => {
contentSafetyViolationMessageEditor.codemirror.refresh();
}, 100);
}

initializeContentSafetyViolationMessageEditor();

const safetyTab = document.getElementById("safety-tab");
if (safetyTab) {
safetyTab.addEventListener("shown.bs.tab", refreshContentSafetyViolationMessageEditor);
}
if (document.getElementById("safety")?.classList.contains("show")) {

Check warning on line 11267 in application/single_app/templates/admin_settings.html

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains security control, sanitization, or audit marker. Recommendation%3A Confirm the change does not weaken auth, CSRF, CSP, XSS defenses, settings sanitization, redaction, audit logging, or tests.
refreshContentSafetyViolationMessageEditor();
}
window.addEventListener("load", refreshContentSafetyViolationMessageEditor);
</script>

<script>
Expand Down
42 changes: 42 additions & 0 deletions docs/explanation/features/CONTENT_SAFETY_VIOLATION_MESSAGES.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# Content Safety Violation Messages

Implemented in version: **0.250.061**

Related config.py version update: `application/single_app/config.py` is **0.250.061** for this implementation.

## Overview

Administrators can configure the Markdown message shown when Azure Content Safety blocks a chat message. They can also choose whether the response includes trigger information such as block reasons, detected categories with severity, and blocklist matches.

## Dependencies

- Content Safety must be enabled in Admin Settings.
- The existing local `marked` and `DOMPurify` assets render and sanitize the saved Markdown in chat.

Check warning on line 14 in docs/explanation/features/CONTENT_SAFETY_VIOLATION_MESSAGES.md

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains security control, sanitization, or audit marker. Recommendation%3A Confirm the change does not weaken auth, CSRF, CSP, XSS defenses, settings sanitization, redaction, audit logging, or tests.

## Technical Specifications

- `content_safety_violation_message` stores the administrator-authored Markdown template.
- `content_safety_include_trigger_information` controls whether Content Safety details are appended to the message.
- The default message is `Your message was blocked by Content Safety.` and trigger information is enabled by default, preserving previous behavior.
- Blank or missing templates fall back to the default message, and saved templates are limited to 3,000 characters.
- Normal and streaming chat requests call the same backend formatter before persisting and returning a `safety` message.
- The chat UI renders safety messages through `DOMPurify.sanitize(marked.parse(...))`; no remote browser assets are added.

Check warning on line 23 in docs/explanation/features/CONTENT_SAFETY_VIOLATION_MESSAGES.md

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains security control, sanitization, or audit marker. Recommendation%3A Confirm the change does not weaken auth, CSRF, CSP, XSS defenses, settings sanitization, redaction, audit logging, or tests.
- The editor refreshes after the hidden Safety tab becomes visible, and Markdown-only edits enable Save Settings and synchronize to the submitted form value.

Check warning on line 24 in docs/explanation/features/CONTENT_SAFETY_VIOLATION_MESSAGES.md

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Moderate - Changed line contains obfuscation, dynamic loading, or hidden payload marker. Recommendation%3A Confirm the changed code is not hiding behavior, decoding payloads, or bypassing normal review.

## Usage Instructions

Check warning on line 26 in docs/explanation/features/CONTENT_SAFETY_VIOLATION_MESSAGES.md

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains AI, plugin, agent, or workspace boundary marker. Recommendation%3A Check whether prompts, chat history, uploaded documents, embeddings, citations, settings, or identity can cross a new boundary.

1. Open **Admin Settings** and enable Content Safety.
2. Use the standard Markdown toolbar in **Safety Violation Message** to format the desired text.

Check warning on line 29 in docs/explanation/features/CONTENT_SAFETY_VIOLATION_MESSAGES.md

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains AI, plugin, agent, or workspace boundary marker. Recommendation%3A Check whether prompts, chat history, uploaded documents, embeddings, citations, settings, or identity can cross a new boundary.
3. Keep **Include Trigger Information** selected to append the block reason, categories, severities, and blocklist matches. Clear it to show only the configured message.
4. Save the settings. Newly blocked chat messages use the saved configuration.

## Testing and Validation

- `functional_tests/test_content_safety_violation_message.py` validates the Markdown editor toolbar, default output, Markdown template preservation, trigger-information suppression, and blank-template fallback.

Check warning on line 35 in docs/explanation/features/CONTENT_SAFETY_VIOLATION_MESSAGES.md

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains security control, sanitization, or audit marker. Recommendation%3A Confirm the change does not weaken auth, CSRF, CSP, XSS defenses, settings sanitization, redaction, audit logging, or tests.
- Admin Settings validation verifies message normalization, seed defaults, and the form controls.
- Version was updated in `application/single_app/config.py` to `0.250.061` for traceability.

## Known Limitations

- The setting applies to text-chat Content Safety blocks. Image-generation moderation messages continue to use their existing dedicated copy.
- Saved messages affect newly created safety messages; historical conversation messages retain the text stored when they were created.
1 change: 1 addition & 0 deletions docs/explanation/features/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ category: Version History
- [AI Voice Conversations Setup Guide](AI_VOICE_CONVERSATIONS_SETUP_GUIDE.md)
- [Activity Log Auto-Refresh](CONTROL_CENTER_ACTIVITY_LOG_AUTO_REFRESH.md)
- [Activity Log Layout Presets](ACTIVITY_LOG_LAYOUT_PRESETS.md)
- [Content Safety Violation Messages](CONTENT_SAFETY_VIOLATION_MESSAGES.md)

## Agent and Action Features

Expand Down
10 changes: 10 additions & 0 deletions docs/explanation/release_notes.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,16 @@

For feature-focused and fix-focused drill-downs by version, see [Features by Version](/explanation/features/) and [Fixes by Version](/explanation/fixes/).

### **(v0.250.061)**

#### New Features

* **Configurable Content Safety Violation Messages**
* Administrators can now configure the Markdown message shown when Content Safety blocks a chat request using the standard Markdown editor toolbar.
* A new setting controls whether the block reason, detected categories and severities, and blocklist matches are included beneath the custom message.
* The editor now renders correctly when the hidden Safety tab opens, and Markdown-only edits activate Save Settings before submission.
* (Ref: microsoft/simplechat#989, `functions_content_safety.py`, `admin_settings.html`, `route_backend_chats.py`)

### **(v0.250.059)**

#### New Features
Expand Down
Loading
Loading