From 2327e30c13fd7537de9c79ea58fc3db987013ea7 Mon Sep 17 00:00:00 2001 From: Chad Palmer Date: Mon, 27 Jul 2026 18:45:44 +0000 Subject: [PATCH] fix-docker-conflict - Fixed docker race conflict with cosmos. --- application/single_app/config.py | 24 +++- .../COSMOS_CONTAINER_STARTUP_CONFLICT_FIX.md | 38 ++++++ docs/explanation/release_notes.md | 9 ++ ...test_cosmos_container_conflict_recovery.py | 125 ++++++++++++++++++ 4 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 docs/explanation/fixes/COSMOS_CONTAINER_STARTUP_CONFLICT_FIX.md create mode 100644 functional_tests/test_cosmos_container_conflict_recovery.py diff --git a/application/single_app/config.py b/application/single_app/config.py index e8c8af0ab..9867a5433 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -96,7 +96,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.057" +VERSION = "0.250.058" SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') SESSION_COOKIE_HTTPONLY = os.getenv('SESSION_COOKIE_HTTPONLY', 'true').lower() != 'false' @@ -430,6 +430,28 @@ def get_redis_cache_infrastructure_endpoint(redis_hostname: str) -> str: cosmos_database_name = "SimpleChat" cosmos_database = cosmos_client.create_database_if_not_exists(cosmos_database_name) +_cosmos_create_container_if_not_exists = cosmos_database.create_container_if_not_exists + + +def _create_container_if_not_exists_with_conflict_recovery(*args, **kwargs): + """Return the existing container when concurrent startup creates it first.""" + try: + return _cosmos_create_container_if_not_exists(*args, **kwargs) + except exceptions.CosmosResourceExistsError: + container_id = kwargs.get('id') + if container_id is None and args: + container_id = args[0] + + if not container_id: + raise + + container = cosmos_database.get_container_client(container_id) + container.read() + return container + + +cosmos_database.create_container_if_not_exists = _create_container_if_not_exists_with_conflict_recovery + cosmos_conversations_container_name = "conversations" cosmos_conversations_container = cosmos_database.create_container_if_not_exists( id=cosmos_conversations_container_name, diff --git a/docs/explanation/fixes/COSMOS_CONTAINER_STARTUP_CONFLICT_FIX.md b/docs/explanation/fixes/COSMOS_CONTAINER_STARTUP_CONFLICT_FIX.md new file mode 100644 index 000000000..ac488bc9e --- /dev/null +++ b/docs/explanation/fixes/COSMOS_CONTAINER_STARTUP_CONFLICT_FIX.md @@ -0,0 +1,38 @@ +# Cosmos Container Startup Conflict Fix + +Fixed in version: **0.250.058** + +## Issue Description + +Local Docker startup could fail when gunicorn booted multiple workers and more than one worker imported `config.py` while Cosmos containers were still being initialized. The observed failure was a Cosmos `NotFound` during `create_container_if_not_exists`, followed by a `Conflict` because another worker created the same container first. + +## Root Cause + +`config.py` called the Azure Cosmos SDK `create_container_if_not_exists` helper directly for each application container at import time. The SDK helper performs a read-then-create flow, so concurrent workers can race when a container is missing: one worker creates the container after another worker has already observed `NotFound`, causing the second worker's create operation to receive `CosmosResourceExistsError` and crash during app import. + +## Technical Details + +Files modified: + +- `application/single_app/config.py` +- `functional_tests/test_cosmos_container_conflict_recovery.py` + +Code changes: + +- Added a local wrapper around `cosmos_database.create_container_if_not_exists`. +- Preserved the normal SDK helper behavior for successful reads and creates. +- On `CosmosResourceExistsError`, re-read the container by id and return the now-existing container. +- Left conflicts without a resolvable container id unchanged so unexpected errors are not hidden. +- Updated `VERSION` in `config.py` from `0.250.057` to `0.250.058`. + +## Validation + +Test results: + +- `python -m py_compile application/single_app/config.py` passed. +- `python functional_tests/test_cosmos_container_conflict_recovery.py` passed with 2/2 tests. + +User impact: + +- Local Docker gunicorn workers can now tolerate concurrent first-run Cosmos container creation instead of failing the whole app startup. +- Existing single-worker and already-provisioned Cosmos startup behavior remains unchanged. diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index b6ffca3ba..f62c13946 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,15 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](/explanation/features/) and [Fixes by Version](/explanation/fixes/). +### **(v0.250.058)** + +#### Bug Fixes + +* **Cosmos Container Startup Conflict Recovery** + * Fixed a local Docker startup failure where multiple gunicorn workers could race while creating first-run Cosmos containers, causing a `NotFound` followed by a `Conflict` during app import. + * Container initialization now re-reads and returns the existing container when another worker creates it first, preserving normal startup behavior for already-provisioned environments. + * (Ref: `config.py`, `test_cosmos_container_conflict_recovery.py`, `COSMOS_CONTAINER_STARTUP_CONFLICT_FIX.md`) + ### **(v0.250.057)** #### Bug Fixes diff --git a/functional_tests/test_cosmos_container_conflict_recovery.py b/functional_tests/test_cosmos_container_conflict_recovery.py new file mode 100644 index 000000000..5c55f4130 --- /dev/null +++ b/functional_tests/test_cosmos_container_conflict_recovery.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +""" +Functional test for Cosmos container startup conflict recovery. +Version: 0.250.058 +Implemented in: 0.250.058 + +This test ensures local Docker gunicorn workers recover when concurrent startup +creates the same Cosmos container between the SDK's read and create calls. +""" + +# test_cosmos_container_conflict_recovery.py +import ast +import os +import sys +import traceback +import types + + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +CONFIG_PATH = os.path.join(REPO_ROOT, 'application', 'single_app', 'config.py') + + +class FakeCosmosResourceExistsError(Exception): + """Stand-in for the Azure Cosmos conflict exception.""" + + +class FakeContainer: + def __init__(self, container_id): + self.container_id = container_id + self.read_count = 0 + + def read(self): + self.read_count += 1 + return {'id': self.container_id} + + +class FakeDatabase: + def __init__(self): + self.requested_container_ids = [] + self.containers = {} + + def get_container_client(self, container_id): + self.requested_container_ids.append(container_id) + container = FakeContainer(container_id) + self.containers[container_id] = container + return container + + +def load_conflict_recovery_function(original_create, database): + with open(CONFIG_PATH, 'r', encoding='utf-8') as config_file: + config_tree = ast.parse(config_file.read(), filename=CONFIG_PATH) + + helper_node = next( + node for node in config_tree.body + if isinstance(node, ast.FunctionDef) + and node.name == '_create_container_if_not_exists_with_conflict_recovery' + ) + + module = ast.Module(body=[helper_node], type_ignores=[]) + ast.fix_missing_locations(module) + namespace = { + '_cosmos_create_container_if_not_exists': original_create, + 'cosmos_database': database, + 'exceptions': types.SimpleNamespace( + CosmosResourceExistsError=FakeCosmosResourceExistsError, + ), + } + exec(compile(module, CONFIG_PATH, 'exec'), namespace) + return namespace['_create_container_if_not_exists_with_conflict_recovery'] + + +def test_conflict_recovery_returns_existing_container(): + print('Testing Cosmos container conflict recovery...') + database = FakeDatabase() + + def original_create(*args, **kwargs): + raise FakeCosmosResourceExistsError('container already exists') + + create_container = load_conflict_recovery_function(original_create, database) + + container = create_container(id='document_access_index', partition_key='ignored') + + assert container.container_id == 'document_access_index' + assert container.read_count == 1 + assert database.requested_container_ids == ['document_access_index'] + print('Test passed!') + return True + + +def test_conflict_without_container_id_is_not_swallowed(): + print('Testing Cosmos conflict without container id is re-raised...') + database = FakeDatabase() + + def original_create(*args, **kwargs): + raise FakeCosmosResourceExistsError('container already exists') + + create_container = load_conflict_recovery_function(original_create, database) + + try: + create_container(partition_key='ignored') + except FakeCosmosResourceExistsError: + print('Test passed!') + return True + + raise AssertionError('Expected conflict to be re-raised when container id is missing') + + +if __name__ == '__main__': + tests = [ + test_conflict_recovery_returns_existing_container, + test_conflict_without_container_id_is_not_swallowed, + ] + results = [] + + for test in tests: + try: + results.append(test()) + except Exception as ex: + print(f'Test failed: {ex}') + traceback.print_exc() + results.append(False) + + success = all(results) + print(f'Results: {sum(results)}/{len(results)} tests passed') + sys.exit(0 if success else 1)