diff --git a/backend/tests/test_kamal_deploy_config.py b/backend/tests/test_kamal_deploy_config.py index 6790548e..4d7e9280 100644 --- a/backend/tests/test_kamal_deploy_config.py +++ b/backend/tests/test_kamal_deploy_config.py @@ -53,6 +53,47 @@ def test_kamal_configs_do_not_deploy_flower_role(): assert "flower" not in accessory.get("roles", []) +def test_kamal_worker_commands_cap_concurrency_and_child_memory(): + deploy_paths = [ + "config/deploy.yml", + "config/deploy.dev1.yml", + "config/deploy.dev2.yml", + "config/deploy.prd.yml", + ] + + for path in deploy_paths: + cmd = _load_deploy_config(path)["servers"]["worker"]["cmd"] + assert "--concurrency=" in cmd + assert "--prefetch-multiplier=1" in cmd + assert "--max-tasks-per-child=" in cmd + assert "--max-memory-per-child=" in cmd + + +def test_prd_resource_limits_keep_long_uptime_headroom(): + prd_config = _load_deploy_config("config/deploy.prd.yml") + + servers = prd_config["servers"] + assert servers["web"]["options"]["memory"] == "6144m" + assert servers["worker"]["options"]["cpus"] == 2 + assert servers["worker"]["options"]["memory"] == "4096m" + assert servers["cron"]["options"]["cpus"] == 0.5 + assert servers["cron"]["options"]["memory"] == "1024m" + + prd_env = prd_config["env"]["clear"] + assert prd_env["CELERY_WORKER_CONCURRENCY"] == "3" + assert prd_env["CELERY_MAX_TASKS_PER_CHILD"] == "5000" + assert prd_env["CELERY_MAX_MEMORY_PER_CHILD_KB"] == "900000" + + +def test_prd_elasticsearch_heap_matches_current_single_host_budget(): + prd_config = _load_deploy_config("config/deploy.prd.yml") + + assert ( + prd_config["accessories"]["elasticsearch"]["env"]["clear"]["ES_JAVA_OPTS"] + == "-Xms2g -Xmx2g" + ) + + def test_prd_uses_canonical_geoportal_base_url(): prd_config = _load_deploy_config("config/deploy.prd.yml") diff --git a/config/deploy.dev1.yml b/config/deploy.dev1.yml index 7c340f9d..b70373d6 100644 --- a/config/deploy.dev1.yml +++ b/config/deploy.dev1.yml @@ -13,7 +13,7 @@ servers: worker: hosts: - lib-btaageoapi-dev-app-01.oit.umn.edu - cmd: bash -c "cd /app/backend && exec /opt/venv/bin/celery -A app.tasks.worker worker --loglevel=INFO" + cmd: bash -lc 'cd /app/backend && exec /opt/venv/bin/celery -A app.tasks.worker worker --loglevel=INFO --concurrency="${CELERY_WORKER_CONCURRENCY:-2}" --prefetch-multiplier=1 --max-tasks-per-child="${CELERY_MAX_TASKS_PER_CHILD:-5000}" --max-memory-per-child="${CELERY_MAX_MEMORY_PER_CHILD_KB:-900000}"' options: cpus: 0.75 memory: 1536m diff --git a/config/deploy.dev2.yml b/config/deploy.dev2.yml index c5820c05..c13bb6e8 100644 --- a/config/deploy.dev2.yml +++ b/config/deploy.dev2.yml @@ -13,7 +13,7 @@ servers: worker: hosts: - lib-geoportal-dev-web-01.oit.umn.edu - cmd: bash -c "cd /app/backend && exec /opt/venv/bin/celery -A app.tasks.worker worker --loglevel=INFO" + cmd: bash -lc 'cd /app/backend && exec /opt/venv/bin/celery -A app.tasks.worker worker --loglevel=INFO --concurrency="${CELERY_WORKER_CONCURRENCY:-2}" --prefetch-multiplier=1 --max-tasks-per-child="${CELERY_MAX_TASKS_PER_CHILD:-5000}" --max-memory-per-child="${CELERY_MAX_MEMORY_PER_CHILD_KB:-900000}"' options: cpus: 0.75 memory: 1536m diff --git a/config/deploy.prd.yml b/config/deploy.prd.yml index 41702cbb..3fe897f4 100644 --- a/config/deploy.prd.yml +++ b/config/deploy.prd.yml @@ -21,21 +21,23 @@ servers: cmd: bash -lc "/app/scripts/start_web_singlehost.sh" options: cpus: 8 - memory: 5120m + memory: 6144m worker: hosts: - lib-geoportal-prd-web-01.oit.umn.edu - cmd: bash -c "cd /app/backend && exec /opt/venv/bin/celery -A app.tasks.worker worker --loglevel=INFO" + cmd: bash -lc 'cd /app/backend && exec /opt/venv/bin/celery -A app.tasks.worker worker --loglevel=INFO --concurrency="${CELERY_WORKER_CONCURRENCY:-3}" --prefetch-multiplier=1 --max-tasks-per-child="${CELERY_MAX_TASKS_PER_CHILD:-5000}" --max-memory-per-child="${CELERY_MAX_MEMORY_PER_CHILD_KB:-900000}"' options: - cpus: 1.75 - memory: 2048m + cpus: 2 + memory: 4096m cron: hosts: - lib-geoportal-prd-web-01.oit.umn.edu cmd: bash -lc '/app/scripts/start_cron.sh' options: + cpus: 0.5 + memory: 1024m user: root proxy: @@ -112,6 +114,9 @@ env: SQLALCHEMY_SYNC_MAX_OVERFLOW: "0" SQLALCHEMY_SYNC_POOL_TIMEOUT: "5" WEB_UVICORN_WORKERS: "4" + CELERY_WORKER_CONCURRENCY: "3" + CELERY_MAX_TASKS_PER_CHILD: "5000" + CELERY_MAX_MEMORY_PER_CHILD_KB: "900000" WEB_INTERNAL_UVICORN_WORKERS: "6" WEB_SSR_WORKERS: "4" BACKUP_ENABLED: "<%= ENV.fetch('BACKUP_ENABLED', 'true') %>" @@ -158,7 +163,7 @@ accessories: elasticsearch: env: clear: - ES_JAVA_OPTS: "-Xms4g -Xmx4g" + ES_JAVA_OPTS: "-Xms2g -Xmx2g" secret: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY diff --git a/config/deploy.yml b/config/deploy.yml index ee43dd73..7a7591cb 100644 --- a/config/deploy.yml +++ b/config/deploy.yml @@ -54,7 +54,7 @@ servers: worker: hosts: - <%= ENV['KAMAL_HOST'] %> - cmd: bash -c "cd /app/backend && exec /opt/venv/bin/celery -A app.tasks.worker worker --loglevel=INFO" + cmd: bash -lc 'cd /app/backend && exec /opt/venv/bin/celery -A app.tasks.worker worker --loglevel=INFO --concurrency="${CELERY_WORKER_CONCURRENCY:-2}" --prefetch-multiplier=1 --max-tasks-per-child="${CELERY_MAX_TASKS_PER_CHILD:-5000}" --max-memory-per-child="${CELERY_MAX_MEMORY_PER_CHILD_KB:-900000}"' options: cpus: <%= ENV.fetch('KAMAL_WORKER_CPUS', '1') %> memory: <%= ENV.fetch('KAMAL_WORKER_MEMORY', '1024m') %> @@ -129,6 +129,9 @@ env: TURNSTILE_COOKIE_SECURE: "<%= ENV.fetch('TURNSTILE_COOKIE_SECURE', 'true') %>" SLACK_BOT_COMMAND: "<%= ENV.fetch('SLACK_BOT_COMMAND', '/btaa') %>" WEB_UVICORN_WORKERS: "<%= ENV.fetch('WEB_UVICORN_WORKERS', '2') %>" + CELERY_WORKER_CONCURRENCY: "<%= ENV.fetch('CELERY_WORKER_CONCURRENCY', '2') %>" + CELERY_MAX_TASKS_PER_CHILD: "<%= ENV.fetch('CELERY_MAX_TASKS_PER_CHILD', '5000') %>" + CELERY_MAX_MEMORY_PER_CHILD_KB: "<%= ENV.fetch('CELERY_MAX_MEMORY_PER_CHILD_KB', '900000') %>" # Loopback-only FastAPI pool used by SSR/BFF fetches. This keeps frontend # traffic from occupying the public /api worker queue used by QGIS, MCP, # and other external clients. Set to "0" to route SSR back to the public pool.