Resumable HTTP uploads

.github/dependabot.yml

@@ -7,9 +7,4 @@ updates:
     # Check for updates once a week
     schedule:
       interval: "weekly"
-    # Group actions version bumps into a single PR
-    groups:
-      actions-deps:
-        patterns:
-          - "*"
 

Dockerfile

@@ -1,9 +1,25 @@
-FROM python:3.7
-COPY ./ /tmp/simdb/
-RUN cd /tmp/simdb/ && \
-    pip3 install . && \
-    pip3 install flask flask_caching flask_cors flask_restx gunicorn psycopg2-binary && \
-    rm -rf /tmp/simdb
-
-ENTRYPOINT ["gunicorn", "--bind=0.0.0.0:5000", "--workers=1", "simdb.remote.wsgi:app"]
-EXPOSE 5000
+ARG PYVER=3.12
+FROM ghcr.io/astral-sh/uv:python${PYVER}-trixie-slim
+
+ENV UV_NO_DEV=1
+ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    libpq-dev \
+    libldap2-dev \
+    libsasl2-dev \
+    libmagic1 \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY uv.lock pyproject.toml alembic.ini ./
+COPY src/ ./src/
+COPY alembic/ ./alembic/
+RUN uv sync --locked --extra all
+
+ENV SIMDB_SITE_CONFIG_PATH=/app/config/simdb.cfg
+
+CMD ["uv", "run", "simdb_server"]
+

alembic/versions/b2c52ee8ff12_add_ingestion_status.py

@@ -0,0 +1,71 @@
+"""Add ingestion status
+
+Revision ID: b2c52ee8ff12
+Revises: 9e9a4a7cd639
+Create Date: 2026-05-11 16:16:03.768893
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "b2c52ee8ff12"
+down_revision: Union[str, Sequence[str], None] = "28bee3aa2429"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    conn = op.get_bind()
+    dialect = conn.dialect.name
+    if dialect == "postgresql":
+        op.execute(
+            "CREATE TYPE ingestionstatus AS ENUM ('QUEUED', 'COPYING', 'COPIED', "
+            "'VALIDATING', 'VALIDATED', 'COMPLETED', 'COPY_FAILED', "
+            "'VALIDATION_FAILED')"
+        )
+    with op.batch_alter_table("simulations", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "ingestion_status",
+                sa.Enum(
+                    "QUEUED",
+                    "COPYING",
+                    "COPIED",
+                    "VALIDATING",
+                    "VALIDATED",
+                    "COMPLETED",
+                    "COPY_FAILED",
+                    "VALIDATION_FAILED",
+                    name="ingestionstatus",
+                ),
+                nullable=True,
+            )
+        )
+        batch_op.add_column(sa.Column("ingestion_version", sa.Integer(), nullable=True))
+    op.execute(
+        "UPDATE simulations SET ingestion_status = 'COMPLETED' WHERE ingestion_status "
+        "IS NULL"
+    )
+    op.execute(
+        "UPDATE simulations SET ingestion_version = 0 WHERE ingestion_version IS NULL"
+    )
+    with op.batch_alter_table("simulations", schema=None) as batch_op:
+        batch_op.alter_column("ingestion_status", nullable=False)
+        batch_op.alter_column("ingestion_version", nullable=False)
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    with op.batch_alter_table("simulations", schema=None) as batch_op:
+        batch_op.drop_column("ingestion_version")
+        batch_op.drop_column("ingestion_status")
+    conn = op.get_bind()
+    dialect = conn.dialect.name
+    if dialect == "postgresql":
+        op.execute("DROP TYPE ingestionstatus")

config/simdb.cfg

@@ -0,0 +1,37 @@
+[flask]
+flask_env = development
+debug = True
+testing = True
+secret_key = CHANGE ME
+
+[authentication]
+type=none
+
+[server]
+upload_folder = /data/simdb/simulations
+port = 5000
+ssl_enabled = False
+authentication_type = None
+admin_password=CHANGE_ME
+imas_remote_host = localhost
+
+[database]
+type = postgres
+host = postgres
+port = 5432
+username = simdb
+password = simdb
+database = simdb
+
+[validation]
+path = ./validation
+auto_validate = True
+error_on_fail = True
+
+[celery]
+broker_url = redis://redis:6379/0
+result_backend = redis://redis:6379/0
+
+[partition]
+data = /data/simdb/partition
+http = /data/simdb/http

docker-compose-pyver.yml

@@ -0,0 +1,54 @@
+# This docker-compose-pyver.yml extends the base docker-compose.yml
+# to run three web services for different PYVER (3.11 and 3.13, 3.12 is the default)
+# but they all use the same redis and postgres service.
+# Run `docker compose -f docker-compose-workers.yml up` to start it up.
+include:
+    - docker-compose.yml
+
+services:
+  migrations-311:
+    extends:
+      service: migrations
+    build:
+      args:
+        PYVER: "3.11"
+
+  migrations-313:
+    extends:
+      service: migrations
+    build:
+      args:
+        PYVER: "3.13"
+
+  web-311:
+    extends:
+      service: web
+    build:
+      args:
+        PYVER: "3.11"
+    ports: !override
+      - "5001:5000"
+    depends_on: !override
+      # !override is necessary to remove the original migrations from depends_on
+      redis:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      migrations-311:
+        condition: service_completed_successfully
+
+  web-313:
+    extends:
+      service: web
+    build:
+      args:
+        PYVER: "3.13"
+    ports: !override
+      - "5003:5000"
+    depends_on: !override
+      redis:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      migrations-313:
+        condition: service_completed_successfully

docker-compose.yml

@@ -1,33 +1,96 @@
-version: "3.9"
 services:
-  db:
-    image: postgres
+  migrations:
+    build: .
     volumes:
-      - db_data:/var/lib/postgresql/data
+      - ./config:/app/config:ro
     environment:
-        POSTGRES_DB: simdb
-        POSTGRES_USER: simdb
-        POSTGRES_PASSWORD: simdb
-  simdb:
+      DATABASE_URL: postgresql+psycopg2://simdb:simdb@postgres:5432/simdb
+    depends_on:
+      - postgres
+    command: uv run alembic upgrade head
+    restart: "no"
+
+  web:
     build: .
-    environment:
-      SIMDB_DATABASE_TYPE: postgres
-      SIMDB_DATABASE_HOST: db
-      SIMDB_DATABASE_PORT: 5432
-      SIMDB_SERVER_UPLOAD_FOLDER: /simulations
+    ports:
+      - "5000:5000"
     volumes:
-      - simulations:/simulations
+      - ./validation:/app/validation:ro
+      - ./config:/app/config:ro
+      - ./tmp/partition_data:/data/simdb/partition:ro
+      - ./tmp/http:/data/simdb/http
+      - ./upload_folder:/data/simdb/simulations
     depends_on:
-      - db
-  nginx:
-    image: nginx
-    ports:
-      - 80:80
+      redis:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      migrations:
+        condition: service_completed_successfully
+    restart: unless-stopped
+
+  worker:
+    profiles: [with_workers]
+    # docker compose --profile with_workers enables this service
+    build: .
+    command: uv run simdb_worker
+    volumes:
+      - ./config:/app/config:ro
+      - ./tmp/partition_data:/data/simdb/partition:ro
+      - ./tmp/http:/data/simdb/http
+      - ./upload_folder:/data/simdb/simulations
+    depends_on:
+      redis:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      migrations:
+        condition: service_completed_successfully
+    restart: unless-stopped
+
+  beat:
+    profiles: [with_workers]
+    # docker compose --profile with_workers enables this service
+    build: .
+    command: uv run simdb_beat
+    volumes:
+      - ./config:/app/config:ro
     depends_on:
-      - simdb
+      redis:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      migrations:
+        condition: service_completed_successfully
+    restart: unless-stopped
+
+  redis:
+    image: redis:8-alpine
+    volumes:
+      - redis_data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: unless-stopped
+
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      - POSTGRES_USER=simdb
+      - POSTGRES_PASSWORD=simdb
+      - POSTGRES_DB=simdb
     volumes:
-      - ./docker/proxy_params:/etc/nginx/proxy_params
-      - ./docker/simdb.nginx:/etc/nginx/conf.d/default.conf
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U simdb"]
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: unless-stopped
+
 volumes:
-  db_data: {}
-  simulations: {}
+  redis_data:
+  postgres_data:
+

docs/Makefile

@@ -11,13 +11,7 @@ BUILDDIR      = _build
 help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
-.PHONY: help html Makefile
-
-# Copy source files from docs/ into sphinx/ before building (mirrors .readthedocs.yml)
-html: Makefile
-	cp ../*.md $(SOURCEDIR)/ 2>/dev/null || true
-	cp ../*.svg $(SOURCEDIR)/ 2>/dev/null || true
-	@$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+.PHONY: help Makefile
 
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).