Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions application/single_app/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -432,6 +432,28 @@
cosmos_database_name = "SimpleChat"
cosmos_database = cosmos_client.create_database_if_not_exists(cosmos_database_name)

_cosmos_create_container_if_not_exists = cosmos_database.create_container_if_not_exists

Check warning on line 435 in application/single_app/config.py

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains secret or sensitive data source marker. Recommendation%3A Pair this source with any nearby network, logging, serialization, or process execution sink before approving.


def _create_container_if_not_exists_with_conflict_recovery(*args, **kwargs):
"""Return the existing container when concurrent startup creates it first."""
try:

Check warning on line 440 in application/single_app/config.py

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains security control, sanitization, or audit marker. Recommendation%3A Confirm the change does not weaken auth, CSRF, CSP, XSS defenses, settings sanitization, redaction, audit logging, or tests.
return _cosmos_create_container_if_not_exists(*args, **kwargs)

Check warning on line 441 in application/single_app/config.py

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains secret or sensitive data source marker. Recommendation%3A Pair this source with any nearby network, logging, serialization, or process execution sink before approving.
except exceptions.CosmosResourceExistsError:

Check warning on line 442 in application/single_app/config.py

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains security control, sanitization, or audit marker. Recommendation%3A Confirm the change does not weaken auth, CSRF, CSP, XSS defenses, settings sanitization, redaction, audit logging, or tests.

Check warning on line 442 in application/single_app/config.py

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains secret or sensitive data source marker. Recommendation%3A Pair this source with any nearby network, logging, serialization, or process execution sink before approving.
container_id = kwargs.get('id')
if container_id is None and args:
container_id = args[0]

if not container_id:
raise

container = cosmos_database.get_container_client(container_id)

Check warning on line 450 in application/single_app/config.py

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains secret or sensitive data source marker. Recommendation%3A Pair this source with any nearby network, logging, serialization, or process execution sink before approving.
container.read()
return container


cosmos_database.create_container_if_not_exists = _create_container_if_not_exists_with_conflict_recovery

Check warning on line 455 in application/single_app/config.py

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains secret or sensitive data source marker. Recommendation%3A Pair this source with any nearby network, logging, serialization, or process execution sink before approving.

cosmos_conversations_container_name = "conversations"
cosmos_conversations_container = cosmos_database.create_container_if_not_exists(
id=cosmos_conversations_container_name,
Expand Down
38 changes: 38 additions & 0 deletions docs/explanation/fixes/COSMOS_CONTAINER_STARTUP_CONFLICT_FIX.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
# Cosmos Container Startup Conflict Fix

Check warning on line 1 in docs/explanation/fixes/COSMOS_CONTAINER_STARTUP_CONFLICT_FIX.md

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains secret or sensitive data source marker. Recommendation%3A Pair this source with any nearby network, logging, serialization, or process execution sink before approving.

Fixed in version: **0.250.058**

## Issue Description

Local Docker startup could fail when gunicorn booted multiple workers and more than one worker imported `config.py` while Cosmos containers were still being initialized. The observed failure was a Cosmos `NotFound` during `create_container_if_not_exists`, followed by a `Conflict` because another worker created the same container first.

Check warning on line 7 in docs/explanation/fixes/COSMOS_CONTAINER_STARTUP_CONFLICT_FIX.md

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains secret or sensitive data source marker. Recommendation%3A Pair this source with any nearby network, logging, serialization, or process execution sink before approving.

## Root Cause

`config.py` called the Azure Cosmos SDK `create_container_if_not_exists` helper directly for each application container at import time. The SDK helper performs a read-then-create flow, so concurrent workers can race when a container is missing: one worker creates the container after another worker has already observed `NotFound`, causing the second worker's create operation to receive `CosmosResourceExistsError` and crash during app import.

Check warning on line 11 in docs/explanation/fixes/COSMOS_CONTAINER_STARTUP_CONFLICT_FIX.md

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review

Important - Changed line contains secret or sensitive data source marker. Recommendation%3A Pair this source with any nearby network, logging, serialization, or process execution sink before approving.

## Technical Details

Files modified:

- `application/single_app/config.py`
- `functional_tests/test_cosmos_container_conflict_recovery.py`

Code changes:

- Added a local wrapper around `cosmos_database.create_container_if_not_exists`.
- Preserved the normal SDK helper behavior for successful reads and creates.
- On `CosmosResourceExistsError`, re-read the container by id and return the now-existing container.
- Left conflicts without a resolvable container id unchanged so unexpected errors are not hidden.
- Updated `VERSION` in `config.py` from `0.250.057` to `0.250.058`.

## Validation

Test results:

- `python -m py_compile application/single_app/config.py` passed.
- `python functional_tests/test_cosmos_container_conflict_recovery.py` passed with 2/2 tests.

User impact:

- Local Docker gunicorn workers can now tolerate concurrent first-run Cosmos container creation instead of failing the whole app startup.
- Existing single-worker and already-provisioned Cosmos startup behavior remains unchanged.
7 changes: 7 additions & 0 deletions docs/explanation/release_notes.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,13 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Ver

### **(v0.250.062)**

#### Bug Fixes

* **Cosmos Container Startup Conflict Recovery**
* Fixed a local Docker startup failure where multiple gunicorn workers could race while creating first-run Cosmos containers, causing a `NotFound` followed by a `Conflict` during app import.
* Container initialization now re-reads and returns the existing container when another worker creates it first, preserving normal startup behavior for already-provisioned environments.
* (Ref: `config.py`, `test_cosmos_container_conflict_recovery.py`, `COSMOS_CONTAINER_STARTUP_CONFLICT_FIX.md`)

#### New Features

* **Workflow Run Cancellation**
Expand Down
125 changes: 125 additions & 0 deletions functional_tests/test_cosmos_container_conflict_recovery.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
#!/usr/bin/env python3
"""
Functional test for Cosmos container startup conflict recovery.
Version: 0.250.058
Implemented in: 0.250.058

This test ensures local Docker gunicorn workers recover when concurrent startup
creates the same Cosmos container between the SDK's read and create calls.
"""

# test_cosmos_container_conflict_recovery.py
import ast
import os
import sys
import traceback
import types


REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
CONFIG_PATH = os.path.join(REPO_ROOT, 'application', 'single_app', 'config.py')


class FakeCosmosResourceExistsError(Exception):
"""Stand-in for the Azure Cosmos conflict exception."""


class FakeContainer:
def __init__(self, container_id):
self.container_id = container_id
self.read_count = 0

def read(self):
self.read_count += 1
return {'id': self.container_id}


class FakeDatabase:
def __init__(self):
self.requested_container_ids = []
self.containers = {}

def get_container_client(self, container_id):
self.requested_container_ids.append(container_id)
container = FakeContainer(container_id)
self.containers[container_id] = container
return container


def load_conflict_recovery_function(original_create, database):
with open(CONFIG_PATH, 'r', encoding='utf-8') as config_file:
config_tree = ast.parse(config_file.read(), filename=CONFIG_PATH)

helper_node = next(
node for node in config_tree.body
if isinstance(node, ast.FunctionDef)
and node.name == '_create_container_if_not_exists_with_conflict_recovery'
)

module = ast.Module(body=[helper_node], type_ignores=[])
ast.fix_missing_locations(module)
namespace = {
'_cosmos_create_container_if_not_exists': original_create,
'cosmos_database': database,
'exceptions': types.SimpleNamespace(
CosmosResourceExistsError=FakeCosmosResourceExistsError,
),
}
exec(compile(module, CONFIG_PATH, 'exec'), namespace)
return namespace['_create_container_if_not_exists_with_conflict_recovery']


def test_conflict_recovery_returns_existing_container():
print('Testing Cosmos container conflict recovery...')
database = FakeDatabase()

def original_create(*args, **kwargs):
raise FakeCosmosResourceExistsError('container already exists')

create_container = load_conflict_recovery_function(original_create, database)

container = create_container(id='document_access_index', partition_key='ignored')

assert container.container_id == 'document_access_index'
assert container.read_count == 1
assert database.requested_container_ids == ['document_access_index']
print('Test passed!')
return True


def test_conflict_without_container_id_is_not_swallowed():
print('Testing Cosmos conflict without container id is re-raised...')
database = FakeDatabase()

def original_create(*args, **kwargs):
raise FakeCosmosResourceExistsError('container already exists')

create_container = load_conflict_recovery_function(original_create, database)

try:
create_container(partition_key='ignored')
except FakeCosmosResourceExistsError:
print('Test passed!')
return True

raise AssertionError('Expected conflict to be re-raised when container id is missing')


if __name__ == '__main__':
tests = [
test_conflict_recovery_returns_existing_container,
test_conflict_without_container_id_is_not_swallowed,
]
results = []

for test in tests:
try:
results.append(test())
except Exception as ex:
print(f'Test failed: {ex}')
traceback.print_exc()
results.append(False)

success = all(results)
print(f'Results: {sum(results)}/{len(results)} tests passed')
sys.exit(0 if success else 1)
Loading