Harden production memory allocation

backend/tests/test_kamal_deploy_config.py

@@ -53,6 +53,47 @@ def test_kamal_configs_do_not_deploy_flower_role():
         assert "flower" not in accessory.get("roles", [])
 
 
+def test_kamal_worker_commands_cap_concurrency_and_child_memory():
+    deploy_paths = [
+        "config/deploy.yml",
+        "config/deploy.dev1.yml",
+        "config/deploy.dev2.yml",
+        "config/deploy.prd.yml",
+    ]
+
+    for path in deploy_paths:
+        cmd = _load_deploy_config(path)["servers"]["worker"]["cmd"]
+        assert "--concurrency=" in cmd
+        assert "--prefetch-multiplier=1" in cmd
+        assert "--max-tasks-per-child=" in cmd
+        assert "--max-memory-per-child=" in cmd
+
+
+def test_prd_resource_limits_keep_long_uptime_headroom():
+    prd_config = _load_deploy_config("config/deploy.prd.yml")
+
+    servers = prd_config["servers"]
+    assert servers["web"]["options"]["memory"] == "6144m"
+    assert servers["worker"]["options"]["cpus"] == 2
+    assert servers["worker"]["options"]["memory"] == "4096m"
+    assert servers["cron"]["options"]["cpus"] == 0.5
+    assert servers["cron"]["options"]["memory"] == "1024m"
+
+    prd_env = prd_config["env"]["clear"]
+    assert prd_env["CELERY_WORKER_CONCURRENCY"] == "3"
+    assert prd_env["CELERY_MAX_TASKS_PER_CHILD"] == "5000"
+    assert prd_env["CELERY_MAX_MEMORY_PER_CHILD_KB"] == "900000"
+
+
+def test_prd_elasticsearch_heap_matches_current_single_host_budget():
+    prd_config = _load_deploy_config("config/deploy.prd.yml")
+
+    assert (
+        prd_config["accessories"]["elasticsearch"]["env"]["clear"]["ES_JAVA_OPTS"]
+        == "-Xms2g -Xmx2g"
+    )
+
+
 def test_prd_uses_canonical_geoportal_base_url():
     prd_config = _load_deploy_config("config/deploy.prd.yml")
 

config/deploy.dev1.yml

@@ -13,7 +13,7 @@ servers:
   worker:
     hosts:
       - lib-btaageoapi-dev-app-01.oit.umn.edu
-    cmd: bash -c "cd /app/backend && exec /opt/venv/bin/celery -A app.tasks.worker worker --loglevel=INFO"
+    cmd: bash -lc 'cd /app/backend && exec /opt/venv/bin/celery -A app.tasks.worker worker --loglevel=INFO --concurrency="${CELERY_WORKER_CONCURRENCY:-2}" --prefetch-multiplier=1 --max-tasks-per-child="${CELERY_MAX_TASKS_PER_CHILD:-5000}" --max-memory-per-child="${CELERY_MAX_MEMORY_PER_CHILD_KB:-900000}"'
     options:
       cpus: 0.75
       memory: 1536m

config/deploy.dev2.yml

@@ -13,7 +13,7 @@ servers:
   worker:
     hosts:
       - lib-geoportal-dev-web-01.oit.umn.edu
-    cmd: bash -c "cd /app/backend && exec /opt/venv/bin/celery -A app.tasks.worker worker --loglevel=INFO"
+    cmd: bash -lc 'cd /app/backend && exec /opt/venv/bin/celery -A app.tasks.worker worker --loglevel=INFO --concurrency="${CELERY_WORKER_CONCURRENCY:-2}" --prefetch-multiplier=1 --max-tasks-per-child="${CELERY_MAX_TASKS_PER_CHILD:-5000}" --max-memory-per-child="${CELERY_MAX_MEMORY_PER_CHILD_KB:-900000}"'
     options:
       cpus: 0.75
       memory: 1536m

config/deploy.prd.yml

@@ -21,21 +21,23 @@ servers:
     cmd: bash -lc "/app/scripts/start_web_singlehost.sh"
     options:
       cpus: 8
-      memory: 5120m
+      memory: 6144m
 
   worker:
     hosts:
       - lib-geoportal-prd-web-01.oit.umn.edu
-    cmd: bash -c "cd /app/backend && exec /opt/venv/bin/celery -A app.tasks.worker worker --loglevel=INFO"
+    cmd: bash -lc 'cd /app/backend && exec /opt/venv/bin/celery -A app.tasks.worker worker --loglevel=INFO --concurrency="${CELERY_WORKER_CONCURRENCY:-3}" --prefetch-multiplier=1 --max-tasks-per-child="${CELERY_MAX_TASKS_PER_CHILD:-5000}" --max-memory-per-child="${CELERY_MAX_MEMORY_PER_CHILD_KB:-900000}"'
     options:
-      cpus: 1.75
-      memory: 2048m
+      cpus: 2
+      memory: 4096m
 
   cron:
     hosts:
       - lib-geoportal-prd-web-01.oit.umn.edu
     cmd: bash -lc '/app/scripts/start_cron.sh'
     options:
+      cpus: 0.5
+      memory: 1024m
       user: root
 
 proxy:
@@ -112,6 +114,9 @@ env:
     SQLALCHEMY_SYNC_MAX_OVERFLOW: "0"
     SQLALCHEMY_SYNC_POOL_TIMEOUT: "5"
     WEB_UVICORN_WORKERS: "4"
+    CELERY_WORKER_CONCURRENCY: "3"
+    CELERY_MAX_TASKS_PER_CHILD: "5000"
+    CELERY_MAX_MEMORY_PER_CHILD_KB: "900000"
     WEB_INTERNAL_UVICORN_WORKERS: "6"
     WEB_SSR_WORKERS: "4"
     BACKUP_ENABLED: "<%= ENV.fetch('BACKUP_ENABLED', 'true') %>"
@@ -158,7 +163,7 @@ accessories:
   elasticsearch:
     env:
       clear:
-        ES_JAVA_OPTS: "-Xms4g -Xmx4g"
+        ES_JAVA_OPTS: "-Xms2g -Xmx2g"
       secret:
         - AWS_ACCESS_KEY_ID
         - AWS_SECRET_ACCESS_KEY

config/deploy.yml

@@ -54,7 +54,7 @@ servers:
   worker:
     hosts:
       - <%= ENV['KAMAL_HOST'] %>
-    cmd: bash -c "cd /app/backend && exec /opt/venv/bin/celery -A app.tasks.worker worker --loglevel=INFO"
+    cmd: bash -lc 'cd /app/backend && exec /opt/venv/bin/celery -A app.tasks.worker worker --loglevel=INFO --concurrency="${CELERY_WORKER_CONCURRENCY:-2}" --prefetch-multiplier=1 --max-tasks-per-child="${CELERY_MAX_TASKS_PER_CHILD:-5000}" --max-memory-per-child="${CELERY_MAX_MEMORY_PER_CHILD_KB:-900000}"'
     options:
       cpus: <%= ENV.fetch('KAMAL_WORKER_CPUS', '1') %>
       memory: <%= ENV.fetch('KAMAL_WORKER_MEMORY', '1024m') %>
@@ -129,6 +129,9 @@ env:
     TURNSTILE_COOKIE_SECURE: "<%= ENV.fetch('TURNSTILE_COOKIE_SECURE', 'true') %>"
     SLACK_BOT_COMMAND:     "<%= ENV.fetch('SLACK_BOT_COMMAND', '/btaa') %>"
     WEB_UVICORN_WORKERS:   "<%= ENV.fetch('WEB_UVICORN_WORKERS', '2') %>"
+    CELERY_WORKER_CONCURRENCY: "<%= ENV.fetch('CELERY_WORKER_CONCURRENCY', '2') %>"
+    CELERY_MAX_TASKS_PER_CHILD: "<%= ENV.fetch('CELERY_MAX_TASKS_PER_CHILD', '5000') %>"
+    CELERY_MAX_MEMORY_PER_CHILD_KB: "<%= ENV.fetch('CELERY_MAX_MEMORY_PER_CHILD_KB', '900000') %>"
     # Loopback-only FastAPI pool used by SSR/BFF fetches. This keeps frontend
     # traffic from occupying the public /api worker queue used by QGIS, MCP,
     # and other external clients. Set to "0" to route SSR back to the public pool.