diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..dfdb8b77 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..4905a9bb --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +ko_fi: fireform diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md index f7aa8994..252540b0 100644 --- a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +++ b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md @@ -35,3 +35,5 @@ Please describe the tests that you ran to verify your changes. Provide instructi - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged and published in downstream modules +- [ ] The **Lint** workflow check passes on this PR ✅ +- [ ] The **Tests** workflow check passes on this PR ✅ diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 39bd9d86..25804171 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -15,4 +15,4 @@ jobs: - name: Build Docker image run: | - docker build -t fireform:test . \ No newline at end of file + docker build -t fireform:test -f docker/prod/Dockerfile . \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 93d41fbb..129cb4ae 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,14 +2,13 @@ name: Lint on: push: - branches: [main] + branches: [main, development] # post-merge verification after PRs land pull_request: - branches: [main] + branches: [main, development, 'development-*'] # gates all PRs jobs: lint: runs-on: ubuntu-latest - continue-on-error: true steps: - uses: actions/checkout@v4 @@ -19,11 +18,12 @@ jobs: with: python-version: "3.11" + - name: Install uv + uses: astral-sh/setup-uv@v6 + - name: Install linter - run: | - python -m pip install --upgrade pip - pip install ruff + run: uv pip install --system ruff==0.16.1 - name: Run linter run: | - ruff check src/ --output-format=github \ No newline at end of file + ruff check app/ --output-format=github \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index d6d1f835..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Release Electron App - -on: - push: - tags: ["v*"] - -jobs: - build: - strategy: - matrix: - os: [macos-latest, ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} - defaults: - run: - working-directory: frontend - - permissions: - contents: write - - steps: - - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install Python dependencies - working-directory: . - shell: bash - run: | - python -m pip install --upgrade pip - if [ "$RUNNER_OS" == "macOS" ]; then - pip install pyinstaller - pip install -r requirements.txt - else - pip install torch --index-url https://download.pytorch.org/whl/cpu - pip install pyinstaller - pip install -r requirements.txt - fi - - - name: Build Python backend with PyInstaller - working-directory: . - shell: bash - run: | - pyinstaller --name api-backend --onefile api/main.py - mkdir -p frontend/bin - cp dist/api-backend* frontend/bin/ - - - uses: actions/setup-node@v4 - with: - node-version: 20 - - - run: npm ci - - - name: Build & publish - run: npx electron-builder --publish always - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0c90b98e..36fd3c10 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,14 +2,29 @@ name: Tests on: push: - branches: [main] + branches: [main, development] # post-merge verification after PRs land pull_request: - branches: [main] + branches: [main, development, 'development-*'] # gates all PRs jobs: test: runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: fireform + POSTGRES_PASSWORD: fireform + POSTGRES_DB: fireform + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U fireform" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: - uses: actions/checkout@v4 @@ -18,13 +33,22 @@ jobs: with: python-version: "3.11" + - name: Install uv + uses: astral-sh/setup-uv@v6 + - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + run: uv pip install --system -r requirements.txt - name: Run tests env: PYTHONPATH: . run: | - python -m pytest tests/ -v --tb=short \ No newline at end of file + python -m pytest tests/ -v --tb=short + + - name: Check for un-migrated model changes + env: + PYTHONPATH: . + DATABASE_URL: postgresql://fireform:fireform@localhost:5432/fireform + run: | + python -m alembic upgrade head + python -m alembic check \ No newline at end of file diff --git a/.gitignore b/.gitignore index b5a9a904..bdcbd83d 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,17 @@ src/inputs/*.pdf frontend/release/ # Local Claude Code instructions -CLAUDE.md \ No newline at end of file +CLAUDE.md + +*temp/ +*.env* +!*.env.example + +# Local benchmark output +reports/ + +# Benchmark run output +benchmark/results/ + +# Local VS Code workspace configuration +*.code-workspace diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 154daf8a..d30e9b93 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,25 +39,14 @@ If you have a great idea for FireForm, we'd love to hear it! Please open an issu 4. Make sure your code lints. 5. Issue that pull request! -## 🛠️ Local Development Setup - -FireForm uses Docker and Docker Compose for the backend to ensure a consistent environment. +**Issues are not formally assigned.** You are free to pick any open issue, work on it, and raise a PR directly. If multiple PRs address the same issue, the first one that actually fixes it generally gets preference. To avoid duplicating someone else's work, coordinate with other contributors on our [Discord](https://discord.gg/nBv5b6kF68) before starting. -### Prerequisites +## 💬 Community -- [Docker](https://docs.docker.com/get-docker/) -- [Docker Compose](https://docs.docker.com/compose/install/) -- `make` (optional, but recommended) -- [Node.js](https://nodejs.org/) 20+ (only needed for the desktop app) +Join our Discord server to ask questions, discuss issues, and coordinate work with other contributors: https://discord.gg/nBv5b6kF68 -### Desktop App Development - -The frontend is a vanilla HTML/CSS/JS app wrapped in Electron. To run it locally: +## 🛠️ Local Development Setup -```bash -cd frontend -npm install # one-time setup -npm start # launches the Electron desktop window -``` +See the [Setup Guide](docs/1.%20SETUP.md) for the full walkthrough: prerequisites, running the backend with Docker, testing endpoints via Swagger UI, day-to-day commands, and troubleshooting. -The backend (API + Ollama) must be running separately via Docker — see `make fireform`. +Before writing code, read the [Project Structure](docs/2.%20PROJECT_STRUCTURE.md) guide — it explains how the codebase is organized and where new code should go. diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 282eb251..00000000 --- a/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ - -FROM python:3.11-slim - -WORKDIR /app - -# Install system dependencies -RUN apt-get update && apt-get install -y \ - curl \ - libgl1 \ - libglib2.0-0 \ - libxcb1 \ - && rm -rf /var/lib/apt/lists/* - -# Copy and install Python dependencies -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - -# Copy application code -COPY . . - -# All imports use api.*, src.* which require the root to be on the path -ENV PYTHONPATH=/app - -# Expose FastAPI port -EXPOSE 8000 - -# Start the FastAPI server (not tail -f /dev/null which does nothing) -CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 00000000..c31dfb05 --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,103 @@ +# FireForm Governance + +This document describes how the FireForm project is governed: who does what, how decisions are made, and how contributors can grow within the community. + +## Roles + +FireForm uses three roles. They are intentionally simple. + +### Owners + +**Organization administrators — set direction, own accountability.** + +- Set the project's strategic direction and governance. +- Manage organization settings, repositories, teams, and permissions. +- Make major architectural and strategic decisions. +- Appoint and remove Maintainers. +- Oversee security and project sustainability. + +**GitHub permission:** Organization Owner (full administrative access). + +> Every Owner holds the GitHub *Organization Owner* role and therefore has full administrative access. Keep at least two active Owners at all times for continuity. + +--- + +### Maintainers + +**Technical leadership — keep the codebase healthy.** + +- Review and merge pull requests. +- Maintain code quality, tests, and releases. +- Manage issues and support contributors. +- Lead particular repositories or technical areas. +- Participate in technical planning. + +**GitHub permission:** `Maintain` or `Write`, depending on repository responsibilities. + +--- + +### Contributors + +**Community participation — anyone who helps FireForm improve.** + +- Contribute code, documentation, tests, and designs. +- Report bugs and suggest features. +- Participate in discussions and testing. +- Earn recognition and a path to become Maintainers. + +**GitHub permission:** Public contributors submit pull requests from forks without joining the organization. Additional access is granted only when specifically needed. + +--- + +## Permission Matrix + +| Action | Owner | Maintainer | Contributor | +| :--- | :---: | :---: | :---: | +| Manage organization settings | ✅ | ❌ | ❌ | +| Manage repository access | ✅ | Generally ❌ | ❌ | +| Review pull requests | ✅ | ✅ | Can participate | +| Merge pull requests | ✅ *(subject to rules)* | ✅ *(subject to rules)* | ❌ | +| Manage releases | ✅ | If authorized | ❌ | +| Submit pull requests | ✅ | ✅ | ✅ | +| Participate in discussions | ✅ | ✅ | ✅ | + +> Branch protection rules and required CI checks apply to everyone, including Owners. + +--- + +## Current Maintainers + +| Name | GitHub | Area | +| :--- | :--- | :--- | +| Marc Vergés Santiago | [@marcvergees](https://github.com/marcvergees) | Core backend, architecture | +| Vincent Harkins | [@vharkins1](https://github.com/vharkins1) | Core backend, infrastructure | + +--- + +## Promotion Path + +The hierarchy is simple, but promotion criteria should be explicit. + +**Contributor → Maintainer** +Sustained, high-quality contributions; good collaboration; demonstrated responsibility and reliability over time. Nominated by an existing Maintainer and approved by the Owners. + +**Maintainer → Owner** +Long-term trust, commitment to the project's future, and unanimous agreement from existing Owners. + +A person's role reflects their responsibilities. GitHub permissions should provide only the access appropriate to those responsibilities — with the exception that every Owner receives full organization administration by design. + +--- + +## Decision Making + +- **Day-to-day decisions** (code review, issue triage): Maintainers. +- **Technical direction** (architecture, major dependencies, releases): Maintainers with Owner visibility. +- **Governance and strategic decisions** (role changes, forks, project future): Owners. + +For significant decisions, open a [GitHub Discussion](https://github.com/fireform-core/FireForm/discussions) so the community can participate before a conclusion is reached. + +--- + +## Questions? + +Open a [Discussion](https://github.com/fireform-core/FireForm/discussions) or reach out on [Discord](https://discord.gg/nBv5b6kF68). diff --git a/Makefile b/Makefile index 026a721a..7e27b63d 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,14 @@ -.PHONY: help build up down logs shell exec pull-model test clean fireform logs-app logs-ollama logs-frontend super-clean +.PHONY: help init fireform fireform-native build up up-native native-banner down logs logs-app logs-ollama shell pull-model test clean super-clean status ready-banner sync docs -# The extraction model pulled into Ollama and used by src/llm.py. Override with -# `make pull-model OLLAMA_MODEL=...`. A 1.5B model keeps per-field fills fast. -OLLAMA_MODEL ?= qwen2.5:1.5b +# Bundled Ollama is behind a compose profile. COMPOSE includes it, so every +# standard target runs the full in-Docker stack as before. COMPOSE_NATIVE leaves +# it out, for an Ollama on the host or a remote server (see fireform-native). +COMPOSE = docker compose -f docker/dev/compose.yml --env-file docker/.env.dev --profile bundled-ollama +COMPOSE_NATIVE = docker compose -f docker/dev/compose.yml --env-file docker/.env.dev +ENV_DEV = docker/.env.dev + +# Read OLLAMA_MODEL from .env.dev at runtime; fall back to default if file absent. +OLLAMA_MODEL = $(shell grep -E '^OLLAMA_MODEL=' $(ENV_DEV) 2>/dev/null | cut -d= -f2 | tr -d '[:space:]' || echo qwen2.5:1.5b) help: @printf '%s\n' \ @@ -13,74 +19,134 @@ help: '/_/ /_//_/ \___/ /_/ \____/_/ /_/ /_/ /_/ ' \ '' @echo "" - @echo "Fireform Development Commands" + @echo "FireForm Development Commands" @echo "==============================" - @echo "make fireform - Build and start containers, then open a shell" + @echo "make init - First-time setup: check deps, create .env.dev, pick model" + @echo "make fireform - Build images, start containers, pull Ollama model" + @echo "make fireform-native - Same, without the Ollama container or model download" + @echo " (uses OLLAMA_HOST from .env.dev: host or remote server)" @echo "make build - Build Docker images" - @echo "make up - Start all containers" + @echo "make up - Start all containers (detached)" + @echo "make up-native - Start all containers except Ollama (detached)" @echo "make down - Stop all containers" - @echo "make logs - View container logs" - @echo "make logs-app - View API container logs" - @echo "make logs-frontend - View frontend container logs" - @echo "make logs-ollama - View Ollama container logs" - @echo "make shell - Open Python shell in app container" - @echo "make exec - Execute Python script in container" - @echo "make pull-model - Pull the extraction model ($(OLLAMA_MODEL)) into Ollama" - @echo "make test - Run tests" - @echo "make clean - Remove containers" - @echo "make super-clean - [CAUTION] Use carefully. Cleans up ALL stopped containers, networks, build cache..." - -# Fix #382 — pull-model is now part of the main setup flow -# The extraction model is pulled automatically before you need it -fireform: build up pull-model + @echo "make sync - Fast-install new requirements.txt deps into running app (no rebuild)" + @echo "make status - Show compact container health summary" + @echo "make logs - Stream all container logs" + @echo "make logs-app - Stream app container logs" + @echo "make logs-ollama - Stream Ollama container logs" + @echo "make logs-worker - Stream Celery worker logs" + @echo "make shell - Open shell in running app container" + @echo "make pull-model - Pull Ollama model from .env.dev ($(OLLAMA_MODEL))" + @echo "make test - Run test suite" + @echo "make docs - Serve interactive API docs from the OpenAPI contract" + @echo "make migrate - Run pending Alembic migrations" + @echo "make migration - Generate new migration (msg='description')" + @echo "make clean - Stop containers (preserves volumes)" + @echo "make super-clean - [CAUTION] Stop containers, delete volumes, prune Docker" + +init: + @chmod +x scripts/check-deps.sh scripts/init-env.sh scripts/select-model.sh + @sh scripts/check-deps.sh + @sh scripts/init-env.sh + @sh scripts/select-model.sh + @printf "Build containers and pull model now? [y/N] "; \ + read answer; \ + case "$$answer" in \ + [yY]*) $(MAKE) fireform ;; \ + *) echo "Run 'make fireform' when ready." ;; \ + esac + +fireform: + @$(COMPOSE) up -d --build + @if $(COMPOSE) exec -T ollama ollama list 2>/dev/null | grep -q "^$(OLLAMA_MODEL)"; then \ + echo " Model $(OLLAMA_MODEL) already pulled."; \ + else \ + echo " Pulling $(OLLAMA_MODEL)..."; \ + $(COMPOSE) exec -T ollama ollama pull $(OLLAMA_MODEL); \ + fi + @$(MAKE) --no-print-directory ready-banner + +# No Ollama container and no model pull: the app and worker use OLLAMA_HOST from +# .env.dev, which must point at an Ollama that already has the model. +fireform-native: + @$(COMPOSE_NATIVE) up -d --build + @$(MAKE) --no-print-directory native-banner + +build: + @$(COMPOSE) build + +up: + @$(COMPOSE) up -d + @$(MAKE) --no-print-directory ready-banner + +up-native: + @$(COMPOSE_NATIVE) up -d + @$(MAKE) --no-print-directory native-banner + +native-banner: @echo "" - @echo "✅ FireForm is ready!" - @echo " Frontend: http://localhost:5173" + @echo "Ollama is not in Docker. Using OLLAMA_HOST from $(ENV_DEV):" + @echo " $$(grep -E '^OLLAMA_HOST=' $(ENV_DEV) | cut -d= -f2)" + @echo "Make sure $(OLLAMA_MODEL) is pulled there. For Ollama on this machine:" + @echo " OLLAMA_HOST=0.0.0.0:11434 ollama serve (listen beyond localhost)" + @echo " ollama pull $(OLLAMA_MODEL)" + @echo " OLLAMA_HOST=http://host.docker.internal:11434 (in $(ENV_DEV))" + @$(MAKE) --no-print-directory ready-banner + +# Fast path for "I added a package": install the delta into the running container +# (no image rebuild, no 1.6GB layer re-export). uv installs only what's missing in +sync: + @$(COMPOSE) exec -T app sh -c "UV_TORCH_BACKEND=cpu uv pip install --system -r requirements.txt" + +status: + @$(COMPOSE) ps --format 'table {{.Service}}\t{{.Status}}' + +ready-banner: + @echo "" + @echo "FireForm is ready!" @echo " API: http://localhost:8000" @echo " API Docs: http://localhost:8000/docs" @echo "" @echo "Run 'make logs' to view live logs, 'make down' to stop." -build: - docker compose build - -up: - docker compose up -d - down: - docker compose down + @$(COMPOSE) down --remove-orphans logs: - docker compose logs -f + @$(COMPOSE) logs -f logs-app: - docker compose logs -f app + @$(COMPOSE) logs -f app logs-ollama: - docker compose logs -f ollama + @$(COMPOSE) logs -f ollama -logs-frontend: - docker compose logs -f frontend +logs-worker: + @$(COMPOSE) logs -f celery-worker shell: - docker compose exec app /bin/bash - -# Start the FastAPI server inside the running container -run: - docker compose exec app uvicorn api.main:app --host 0.0.0.0 --port 8000 --reload - -exec: - docker compose exec app python3 src/main.py + @$(COMPOSE) exec app /bin/sh pull-model: - docker compose exec ollama ollama pull $(OLLAMA_MODEL) + @$(COMPOSE) exec -T ollama ollama pull $(OLLAMA_MODEL) -# Fix — correct test directory (was src/test/ which doesn't exist) test: - docker compose exec app python3 -m pytest tests/ -v + @$(COMPOSE) exec -T app python3 -m pytest tests/ -v + +docs: + @echo "Serving Swagger UI from contracts/openapi.yaml at http://localhost:8088" + @npx --yes swagger-ui-watcher contracts/openapi.yaml --port 8088 + +migrate: + @$(COMPOSE) exec -T app alembic upgrade head + +migration: + @$(COMPOSE) exec -T app alembic revision --autogenerate -m "$(msg)" clean: - docker compose down -v + @$(COMPOSE) down + super-clean: - docker compose down -v - docker system prune + @echo "WARNING: this will delete all volumes (database, uploads, model weights)." + @$(COMPOSE) down -v + @docker system prune -f diff --git a/README.md b/README.md index 15a67a00..fd0f6b49 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ npm install npm start ``` -> **Note:** The desktop app is a thin Electron wrapper around the same web frontend. The backend (API + Ollama) still needs to be running — see the [Deployment Guide](https://github.com/fireform-core/FireForm/wiki/DEPLOYMENT). +> **Note:** The desktop app is a thin Electron wrapper around the same web frontend. The backend (API + Ollama) still needs to be running — see the [Deployment Guide](docs/DEPLOYMENT.md). ## 🏆 Acknowledgements and Contributors This project was built in 48 hours for the Reboot the Earth 2025 hackathon. Thank you to the United Nations and UC Santa Cruz for hosting this incredible event and inspiring us to build solutions for a better future. @@ -102,14 +102,74 @@ FireForm is designed from the ground up to ensure that all generated and collect ## 🔒 Privacy & Applicable Laws -FireForm is built on a local-first architecture, ensuring that all processing (including AI extraction) occurs locally on the user's hardware. We do not collect, process, or share any Personally Identifiable Information (PII) with third parties. By keeping data completely offline, FireForm natively assists deploying organizations in complying with strict data protection laws such as **HIPAA**, **CCPA**, and **GDPR**. +FireForm is built on a **local-first architecture**: all processing (including AI extraction) occurs locally on the operator's hardware and nothing is transmitted to external servers by default. However, operators should be aware of the following: -For complete details on our data handling, consent management procedures, and how the solution was designed to comply with HIPAA, CCPA, and GDPR, please review our public [Privacy Policy](https://fireform-core.github.io/FireForm/privacy.html). +- **PII is processed locally.** Incident descriptions, names, dates, and other personally identifiable information are extracted by the local LLM, written into filled PDF files, and persisted in the local database (`FormSubmission` records). These files and records remain on disk until explicitly deleted. +- **No external transmission.** Data does not leave the machine by default. Audio transcription (Whisper) and LLM inference (Ollama) are both local services. +- **Operator responsibility.** Because PII is stored locally, the security of that data depends entirely on the operator's host system, filesystem permissions, backup practices, and access controls. + +For complete details on data handling and compliance guidance, please review our public [Privacy Policy](https://fireform-core.github.io/FireForm/privacy.html). + +## 🗑️ Data Deletion & Retention + +FireForm provides API endpoints and an automated purge mechanism to manage stored PII. + +### Deleting individual records + +```bash +# Delete a single form submission (removes DB record + output PDF) +curl -X DELETE http://localhost:8000/api/v1/forms/{submission_id} \ + -H "X-API-Key: your-key" + +# Delete a template and all associated submissions +curl -X DELETE http://localhost:8000/api/v1/templates/{template_id} \ + -H "X-API-Key: your-key" +``` + +### Bulk purge + +```bash +# Purge all submissions older than 30 days +curl -X POST "http://localhost:8000/api/v1/forms/purge?days=30" \ + -H "X-API-Key: your-key" +``` + +### Automated retention + +Set `RETENTION_PERIOD_DAYS` in `docker/.env.dev` (default: `30`). Celery Beat will automatically run a purge job every night at 03:00 UTC: + +```env +RETENTION_PERIOD_DAYS=30 +``` + +To enable the scheduler: `celery -A app.core.celery beat -l info` + +## 🔑 Access Control + +Deletion and purge endpoints can be protected by setting an API key: + +```env +FIREFORM_API_KEY=your-strong-secret-key +``` + +When set, all `DELETE` and `POST /purge` requests must include one of: +- **Header:** `X-API-Key: your-strong-secret-key` +- **Bearer token:** `Authorization: Bearer your-strong-secret-key` + +Read-only endpoints (list, preview, fill) do **not** require authentication. + +## 🛡️ Operator Hardening Recommendations + +- **Filesystem permissions:** Restrict access to `data/inputs/` and `data/outputs/` to the application user only (`chmod 700`). +- **Backups:** Encrypt any backups containing output PDFs. +- **HTTPS:** Deploy behind a reverse proxy (e.g., nginx) with TLS enabled. +- **API key rotation:** Rotate `FIREFORM_API_KEY` regularly and never commit it to version control. +- **Retention policy:** Configure `RETENTION_PERIOD_DAYS` in line with applicable regulations (HIPAA, GDPR, CCPA). ## 🛡️ Do No Harm by Design FireForm is built to anticipate and prevent harm: -- **Data Privacy & Security (9A):** We handle sensitive incident data entirely offline. By never transmitting data to the cloud, we natively prevent online data breaches of PII. +- **Data Privacy & Security (9A):** We handle sensitive incident data entirely offline. By never transmitting data to the cloud, we natively prevent online data breaches of PII. Operators are responsible for securing local storage; see the Operator Hardening Recommendations above. - **Inappropriate Content (9B):** The application is an internal productivity tool without public social interactions or user-generated content hosting, completely mitigating the risks of public harassment or illegal content distribution. - **Protection from Harassment (9C):** For our open-source contributor community, we strictly enforce our [Code of Conduct](CODE_OF_CONDUCT.md) to ensure a safe, harassment-free environment for all contributors. diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 00000000..2788aaa8 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,37 @@ +[alembic] +script_location = alembic +prepend_sys_path = . +path_separator = os + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 00000000..fcf3750e --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,97 @@ +from logging.config import fileConfig + +import sqlalchemy as sa +from sqlalchemy import engine_from_config, pool +from sqlmodel import SQLModel + +from alembic import context +from app.core.config import DATABASE_URL +from app.models import ( # noqa: F401 + Extraction, + Form, + FormSubmission, + Incident, + Input, + Job, + Report, + Template, +) + +config = context.config + +if not config.get_main_option("sqlalchemy.url"): + config.set_main_option("sqlalchemy.url", DATABASE_URL) + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = SQLModel.metadata + + +def compare_type(context, inspected_column, metadata_column, inspected_type, metadata_type): + """Custom type comparison to handle SQLModel / PostgreSQL differences. + + SQLModel uses UTCDateTime (a TypeDecorator wrapping DateTime) for datetime fields, + while PostgreSQL reflects them as TIMESTAMP (or TIMESTAMP WITHOUT TIME ZONE). + Treat these as equivalent so Alembic autogenerate does not detect false-positive + type modification operations. + """ + meta_is_dt = ( + isinstance(metadata_type, (sa.DateTime, sa.TIMESTAMP)) + or isinstance(getattr(metadata_type, "impl", None), (sa.DateTime, sa.TIMESTAMP)) + or type(metadata_type).__name__ in ("UTCDateTime", "DateTime", "TIMESTAMP") + ) + insp_is_dt = ( + isinstance(inspected_type, (sa.DateTime, sa.TIMESTAMP)) + or isinstance(getattr(inspected_type, "impl", None), (sa.DateTime, sa.TIMESTAMP)) + or type(inspected_type).__name__ in ("UTCDateTime", "DateTime", "TIMESTAMP") + ) + if meta_is_dt and insp_is_dt: + return False + + return None + + +def run_migrations_offline(): + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + compare_type=compare_type, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + connectable = config.attributes.get("connection", None) + + if connectable is None: + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + compare_type=compare_type, + ) + with context.begin_transaction(): + context.run_migrations() + else: + context.configure( + connection=connectable, + target_metadata=target_metadata, + compare_type=compare_type, + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 00000000..6ce33510 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,27 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/001_initial_schema.py b/alembic/versions/001_initial_schema.py new file mode 100644 index 00000000..04e6a1fc --- /dev/null +++ b/alembic/versions/001_initial_schema.py @@ -0,0 +1,65 @@ +"""Initial schema — Template, FormSubmission, and Job tables. + +Revision ID: 001 +Revises: +Create Date: 2026-06-22 + +""" +from collections.abc import Sequence + +import sqlalchemy as sa +import sqlmodel + +from alembic import op + +revision: str = "001" +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_table( + "template", + sa.Column("id", sa.Integer, primary_key=True, autoincrement=True), + sa.Column("name", sqlmodel.sql.sqltypes.AutoString, nullable=False), + sa.Column("fields", sa.JSON, nullable=False), + sa.Column("pdf_path", sqlmodel.sql.sqltypes.AutoString, nullable=False), + sa.Column("created_at", sa.DateTime, nullable=False), + ) + + op.create_table( + "formsubmission", + sa.Column("id", sa.Integer, primary_key=True, autoincrement=True), + sa.Column("template_id", sa.Integer, sa.ForeignKey("template.id"), nullable=False), + sa.Column("input_text", sqlmodel.sql.sqltypes.AutoString, nullable=False), + sa.Column("output_pdf_path", sqlmodel.sql.sqltypes.AutoString, nullable=False), + sa.Column("created_at", sa.DateTime, nullable=False), + ) + + op.create_table( + "job", + sa.Column("id", sa.Integer, primary_key=True, autoincrement=True), + sa.Column("job_id", sqlmodel.sql.sqltypes.AutoString, nullable=False), + sa.Column("celery_task_id", sqlmodel.sql.sqltypes.AutoString, nullable=False), + sa.Column("job_type", sqlmodel.sql.sqltypes.AutoString, nullable=False), + sa.Column("template_id", sa.Integer, sa.ForeignKey("template.id"), nullable=True), + sa.Column("input_text", sqlmodel.sql.sqltypes.AutoString, nullable=True), + sa.Column("status", sqlmodel.sql.sqltypes.AutoString, nullable=False), + sa.Column("progress_percent", sa.Integer, nullable=False), + sa.Column("result_url", sqlmodel.sql.sqltypes.AutoString, nullable=True), + sa.Column("error", sa.JSON, nullable=True), + sa.Column("model", sqlmodel.sql.sqltypes.AutoString, nullable=True), + sa.Column("created_at", sa.DateTime, nullable=False), + sa.Column("updated_at", sa.DateTime, nullable=False), + ) + op.create_index("ix_job_job_id", "job", ["job_id"], unique=True) + op.create_index("ix_job_celery_task_id", "job", ["celery_task_id"], unique=False) + + +def downgrade() -> None: + op.drop_index("ix_job_celery_task_id", table_name="job") + op.drop_index("ix_job_job_id", table_name="job") + op.drop_table("job") + op.drop_table("formsubmission") + op.drop_table("template") diff --git a/alembic/versions/002_v1_models.py b/alembic/versions/002_v1_models.py new file mode 100644 index 00000000..c943e9e8 --- /dev/null +++ b/alembic/versions/002_v1_models.py @@ -0,0 +1,145 @@ +"""v1 models — Input, Extraction, Incident, Form, Report tables. + +Revision ID: 002 +Revises: 001 +Create Date: 2026-06-26 + +JSON columns use sa.JSON (Postgres json type, not jsonb) for consistency with +migration 001 and SQLite test-harness compatibility. A follow-up can switch +high-query blobs (especially incident_contract) to jsonb via with_variant() if +containment operators or GIN indexing are needed. + +Form.job_id and Report.job_id are plain UUID columns with no FK constraint. +The contract Job model (UUID-PK, enum-typed) is a separate decision; the FK +will be added when that model is resolved. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +import sqlmodel + +from alembic import op + +revision: str = "002" +down_revision: str | None = "001" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_table( + "inputs", + sa.Column("input_id", sa.Uuid(), primary_key=True), + sa.Column("input_type", sqlmodel.sql.sqltypes.AutoString, nullable=False), + sa.Column("status", sqlmodel.sql.sqltypes.AutoString, nullable=False), + sa.Column("transcript", sqlmodel.sql.sqltypes.AutoString, nullable=True), + sa.Column("original_filename", sqlmodel.sql.sqltypes.AutoString, nullable=True), + sa.Column("audio_duration_seconds", sa.Float, nullable=True), + sa.Column("character_count", sa.Integer, nullable=True), + sa.Column("word_count", sa.Integer, nullable=True), + sa.Column("station_id", sqlmodel.sql.sqltypes.AutoString, nullable=True), + sa.Column("responder_badge", sqlmodel.sql.sqltypes.AutoString, nullable=True), + sa.Column("incident_date_hint", sa.Date, nullable=True), + sa.Column("error_detail", sqlmodel.sql.sqltypes.AutoString, nullable=True), + sa.Column("created_at", sa.DateTime, nullable=False), + sa.Column("updated_at", sa.DateTime, nullable=False), + ) + + op.create_table( + "extractions", + sa.Column("extract_id", sa.Uuid(), primary_key=True), + sa.Column( + "input_id", + sa.Uuid(), + sa.ForeignKey("inputs.input_id"), + nullable=False, + ), + sa.Column("status", sqlmodel.sql.sqltypes.AutoString, nullable=False), + sa.Column("started_at", sa.DateTime, nullable=True), + sa.Column("completed_at", sa.DateTime, nullable=True), + sa.Column("model_used", sqlmodel.sql.sqltypes.AutoString, nullable=True), + sa.Column("processing_time_seconds", sa.Float, nullable=True), + sa.Column("incident_contract", sa.JSON, nullable=True), + sa.Column("corrections", sa.JSON, nullable=True), + sa.Column("error_type", sqlmodel.sql.sqltypes.AutoString, nullable=True), + sa.Column("error_detail", sqlmodel.sql.sqltypes.AutoString, nullable=True), + sa.Column("created_at", sa.DateTime, nullable=False), + sa.Column("updated_at", sa.DateTime, nullable=False), + ) + + op.create_table( + "incidents", + sa.Column("incident_id", sa.Uuid(), primary_key=True), + sa.Column( + "extract_id", + sa.Uuid(), + sa.ForeignKey("extractions.extract_id"), + nullable=False, + ), + sa.Column("incident_number", sqlmodel.sql.sqltypes.AutoString, nullable=True), + sa.Column("status", sqlmodel.sql.sqltypes.AutoString, nullable=False), + sa.Column("incident_name", sqlmodel.sql.sqltypes.AutoString, nullable=True), + sa.Column("incident_type", sqlmodel.sql.sqltypes.AutoString, nullable=True), + sa.Column("incident_date", sa.Date, nullable=True), + sa.Column("tags", sa.JSON, nullable=True), + sa.Column("notes", sqlmodel.sql.sqltypes.AutoString, nullable=True), + sa.Column("created_at", sa.DateTime, nullable=False), + sa.Column("updated_at", sa.DateTime, nullable=False), + sa.Column("deleted_at", sa.DateTime, nullable=True), + ) + + op.create_table( + "forms", + sa.Column("form_id", sa.Uuid(), primary_key=True), + sa.Column("form_type", sqlmodel.sql.sqltypes.AutoString, nullable=False), + sa.Column("status", sqlmodel.sql.sqltypes.AutoString, nullable=False), + sa.Column( + "extract_id", + sa.Uuid(), + sa.ForeignKey("extractions.extract_id"), + nullable=False, + ), + sa.Column( + "incident_id", + sa.Uuid(), + sa.ForeignKey("incidents.incident_id"), + nullable=True, + ), + sa.Column("job_id", sa.Uuid(), nullable=True), # no FK — see module docstring + sa.Column("completed_at", sa.DateTime, nullable=True), + sa.Column("pdf_ready", sa.Boolean, nullable=False), + sa.Column("json_ready", sa.Boolean, nullable=False), + sa.Column("field_mapping_summary", sa.JSON, nullable=True), + sa.Column("pdf_path", sqlmodel.sql.sqltypes.AutoString, nullable=True), + sa.Column("json_data", sa.JSON, nullable=True), + sa.Column("created_at", sa.DateTime, nullable=False), + sa.Column("updated_at", sa.DateTime, nullable=False), + ) + + op.create_table( + "reports", + sa.Column("report_id", sa.Uuid(), primary_key=True), + sa.Column("period_type", sqlmodel.sql.sqltypes.AutoString, nullable=False), + sa.Column("period_label", sqlmodel.sql.sqltypes.AutoString, nullable=True), + sa.Column("year", sa.Integer, nullable=False), + sa.Column("month", sa.Integer, nullable=True), + sa.Column("quarter", sa.Integer, nullable=True), + sa.Column("output_format", sqlmodel.sql.sqltypes.AutoString, nullable=False), + sa.Column("status", sqlmodel.sql.sqltypes.AutoString, nullable=False), + sa.Column("generated_at", sa.DateTime, nullable=True), + sa.Column("summary", sa.JSON, nullable=True), + sa.Column("job_id", sa.Uuid(), nullable=True), # no FK — see module docstring + sa.Column("pdf_path", sqlmodel.sql.sqltypes.AutoString, nullable=True), + sa.Column("json_data", sa.JSON, nullable=True), + sa.Column("created_at", sa.DateTime, nullable=False), + sa.Column("updated_at", sa.DateTime, nullable=False), + ) + + +def downgrade() -> None: + op.drop_table("reports") + op.drop_table("forms") + op.drop_table("incidents") + op.drop_table("extractions") + op.drop_table("inputs") diff --git a/alembic/versions/003_formsubmission_input_id_fk.py b/alembic/versions/003_formsubmission_input_id_fk.py new file mode 100644 index 00000000..303b0a51 --- /dev/null +++ b/alembic/versions/003_formsubmission_input_id_fk.py @@ -0,0 +1,42 @@ +"""formsubmission.input_id FK to inputs. + +Revision ID: 003 +Revises: 002 +Create Date: 2026-08-07 + +Adds a nullable input_id FK on formsubmission -> inputs.input_id so submissions +can link to the Input they were filled from, instead of only duplicating the +transcript into input_text. input_text is kept for now (read by the +/forms/submissions and analytics endpoints); dropping it is a deferred +follow-up once those readers move to the FK. +""" + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + +revision: str = "003" +down_revision: str | None = "002" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # batch_alter_table: SQLite can't ALTER a table to add a FK constraint in + # place (no ALTER-constraint support), so this goes through its + # copy-and-move strategy. On Postgres it emits a plain ALTER TABLE. + with op.batch_alter_table("formsubmission") as batch_op: + batch_op.add_column(sa.Column("input_id", sa.Uuid(), nullable=True)) + batch_op.create_foreign_key( + "fk_formsubmission_input_id_inputs", + "inputs", + ["input_id"], + ["input_id"], + ) + + +def downgrade() -> None: + with op.batch_alter_table("formsubmission") as batch_op: + batch_op.drop_constraint("fk_formsubmission_input_id_inputs", type_="foreignkey") + batch_op.drop_column("input_id") diff --git a/api/db/database.py b/api/db/database.py deleted file mode 100644 index 97e6e622..00000000 --- a/api/db/database.py +++ /dev/null @@ -1,20 +0,0 @@ -import os -from sqlmodel import create_engine, Session - -# Define path to the database in the user's home directory -HOME_DIR = os.path.expanduser("~") -APP_DIR = os.path.join(HOME_DIR, ".fireform") -os.makedirs(APP_DIR, exist_ok=True) -DB_PATH = os.path.join(APP_DIR, "fireform.db") - -DATABASE_URL = f"sqlite:///{DB_PATH}" - -engine = create_engine( - DATABASE_URL, - echo=True, - connect_args={"check_same_thread": False}, -) - -def get_session(): - with Session(engine) as session: - yield session \ No newline at end of file diff --git a/api/db/init_db.py b/api/db/init_db.py deleted file mode 100644 index ea77cb62..00000000 --- a/api/db/init_db.py +++ /dev/null @@ -1,47 +0,0 @@ -import json -import datetime -from sqlmodel import SQLModel, Session, select -from api.db.database import engine -from api.db import models -from api.db.models import Template - -def seed_db(): - with Session(engine) as session: - # Check if we already have templates - statement = select(Template) - try: - results = session.exec(statement).first() - except Exception: - # Table might not exist yet if called at a weird time - results = None - - if not results: - print("Seeding database with default template...") - fields = { - "Employee's name": "string", - "Employee's job title": "string", - "Employee's department supervisor": "string", - "Employee's phone number": "string", - "Employee's email": "string", - "Signature": "string", - "Date": "string" - } - - # Using ID 2 as agreed to avoid any ID 1 corruption - default_template = Template( - id=2, - name="Manual Test Template", - fields=fields, - pdf_path="src/inputs/file_template_manual.pdf", - created_at=datetime.datetime.now() - ) - session.add(default_template) - session.commit() - print("Database seeded successfully.") - -def init_db(): - SQLModel.metadata.create_all(engine) - seed_db() - -if __name__ == "__main__": - init_db() \ No newline at end of file diff --git a/api/db/models.py b/api/db/models.py deleted file mode 100644 index 8f826556..00000000 --- a/api/db/models.py +++ /dev/null @@ -1,19 +0,0 @@ -from sqlmodel import SQLModel, Field -from sqlalchemy import Column, JSON -from datetime import datetime, timezone - - -class Template(SQLModel, table=True): - id: int | None = Field(default=None, primary_key=True) - name: str - fields: dict = Field(sa_column=Column(JSON)) - pdf_path: str - created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) - - -class FormSubmission(SQLModel, table=True): - id: int | None = Field(default=None, primary_key=True) - template_id: int = Field(foreign_key="template.id") - input_text: str - output_pdf_path: str - created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) \ No newline at end of file diff --git a/api/db/repositories.py b/api/db/repositories.py deleted file mode 100644 index a9ac3cfa..00000000 --- a/api/db/repositories.py +++ /dev/null @@ -1,24 +0,0 @@ -from sqlmodel import Session, select -from api.db.models import Template, FormSubmission - -# Templates -def create_template(session: Session, template: Template) -> Template: - session.add(template) - session.commit() - session.refresh(template) - return template - -def get_template(session: Session, template_id: int) -> Template | None: - return session.get(Template, template_id) - - -def list_templates(session: Session) -> list[Template]: - statement = select(Template).order_by(Template.created_at.desc(), Template.id.desc()) - return list(session.exec(statement)) - -# Forms -def create_form(session: Session, form: FormSubmission) -> FormSubmission: - session.add(form) - session.commit() - session.refresh(form) - return form diff --git a/api/deps.py b/api/deps.py deleted file mode 100644 index c2a0b748..00000000 --- a/api/deps.py +++ /dev/null @@ -1,4 +0,0 @@ -from api.db.database import get_session - -def get_db(): - yield from get_session() \ No newline at end of file diff --git a/api/errors/base.py b/api/errors/base.py deleted file mode 100644 index 1f81a082..00000000 --- a/api/errors/base.py +++ /dev/null @@ -1,4 +0,0 @@ -class AppError(Exception): - def __init__(self, message: str, status_code: int = 400): - self.message = message - self.status_code = status_code \ No newline at end of file diff --git a/api/errors/handlers.py b/api/errors/handlers.py deleted file mode 100644 index 903e744b..00000000 --- a/api/errors/handlers.py +++ /dev/null @@ -1,11 +0,0 @@ -from fastapi import Request -from fastapi.responses import JSONResponse -from api.errors.base import AppError - -def register_exception_handlers(app): - @app.exception_handler(AppError) - async def app_error_handler(request: Request, exc: AppError): - return JSONResponse( - status_code=exc.status_code, - content={"error": exc.message}, - ) \ No newline at end of file diff --git a/api/main.py b/api/main.py deleted file mode 100644 index 0c30bc5e..00000000 --- a/api/main.py +++ /dev/null @@ -1,42 +0,0 @@ -from contextlib import asynccontextmanager -import os - -# Disable CUDA to prevent PyTorch from trying to find NVIDIA drivers on Mac Silicon / Docker -os.environ["CUDA_VISIBLE_DEVICES"] = "" - -from fastapi import FastAPI -from api.routes import templates, forms -from api.db.init_db import init_db -from api.errors.handlers import register_exception_handlers -from fastapi.middleware.cors import CORSMiddleware -from api.routes import forms, templates - -@asynccontextmanager -async def lifespan(app: FastAPI): - # Startup: Initialize the database and seed it if necessary - print("Initializing database...") - init_db() - yield - # Shutdown logic goes here if needed - -app = FastAPI(lifespan=lifespan) - -register_exception_handlers(app) - -default_origins = "http://127.0.0.1:5173,http://localhost:5173" -allowed_origins = [ - origin.strip() - for origin in os.getenv("FRONTEND_ORIGINS", default_origins).split(",") - if origin.strip() -] - -app.add_middleware( - CORSMiddleware, - allow_origins=allowed_origins, - allow_credentials=False, - allow_methods=["*"], - allow_headers=["*"], -) - -app.include_router(templates.router) -app.include_router(forms.router) diff --git a/api/routes/__init__.py b/api/routes/__init__.py deleted file mode 100644 index e8fe8f45..00000000 --- a/api/routes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import templates, forms diff --git a/api/routes/forms.py b/api/routes/forms.py deleted file mode 100644 index 74069eaf..00000000 --- a/api/routes/forms.py +++ /dev/null @@ -1,107 +0,0 @@ -import os - -import requests -from fastapi import APIRouter, Depends, File, UploadFile -from sqlmodel import Session -from api.deps import get_db -from api.schemas.forms import ( - FormFill, - FormFillResponse, - ModelsResponse, - TranscriptionResponse, -) -from api.db.repositories import create_form, get_template -from api.db.models import FormSubmission -from api.errors.base import AppError -from src.controller import Controller - -router = APIRouter(prefix="/forms", tags=["forms"]) - - -@router.post("/fill", response_model=FormFillResponse) -def fill_form(form: FormFill, db: Session = Depends(get_db)): - - fetched_template = get_template(db, form.template_id) - if not fetched_template: - raise AppError("Template not found", status_code=404) - - controller = Controller() - try: - path = controller.fill_form( - user_input=form.input_text, - fields=fetched_template.fields, - pdf_form_path=fetched_template.pdf_path, - model=form.model, - ) - - # `model` is a runtime override, not a column — keep it out of the DB row. - submission = FormSubmission( - **form.model_dump(exclude={"model"}), output_pdf_path=path - ) - return create_form(db, submission) - except Exception as e: - raise AppError(str(e), status_code=500) - - -@router.get("/models", response_model=ModelsResponse) -def list_models(): - """List the Whisper-independent extraction models available in the local - Ollama instance, plus the configured default. Used by the Fill Form UI's - model picker. Falls back to just the default if Ollama is unreachable.""" - default_model = os.getenv("OLLAMA_MODEL", "qwen2.5:1.5b") - ollama_host = os.getenv("OLLAMA_HOST", "http://localhost:11434").rstrip("/") - - models: list[str] = [] - try: - response = requests.get(f"{ollama_host}/api/tags", timeout=10) - response.raise_for_status() - models = [m["name"] for m in response.json().get("models", []) if m.get("name")] - except requests.exceptions.RequestException: - models = [] - - # Always surface the configured default, even if Ollama hasn't pulled it yet. - if default_model not in models: - models.insert(0, default_model) - - return ModelsResponse(models=models, default=default_model) - - -@router.post("/transcribe", response_model=TranscriptionResponse) -def transcribe(audio: UploadFile = File(...)): - """Forward recorded audio to the local Whisper ASR sidecar and return text. - - Mirrors the Ollama wiring: WHISPER_HOST points at the whisper service - (http://whisper:9000 inside Docker, http://localhost:9000 otherwise). The - audio is streamed straight through to the local STT service and never - persisted — no PII leaves the machine. - """ - whisper_host = os.getenv("WHISPER_HOST", "http://localhost:9000").rstrip("/") - whisper_url = f"{whisper_host}/asr" - - files = { - "audio_file": ( - audio.filename or "audio.wav", - audio.file.read(), - audio.content_type or "audio/wav", - ) - } - params = {"task": "transcribe", "output": "json", "encode": "true"} - - try: - response = requests.post(whisper_url, params=params, files=files, timeout=120) - response.raise_for_status() - except requests.exceptions.ConnectionError: - raise AppError( - f"Could not connect to the speech-to-text service at {whisper_url}. " - "Please ensure the whisper service is running.", - status_code=503, - ) - except requests.exceptions.RequestException as e: - raise AppError(f"Transcription failed: {e}", status_code=502) - - try: - text = (response.json().get("text") or "").strip() - except ValueError: - text = response.text.strip() - - return TranscriptionResponse(text=text) diff --git a/api/routes/templates.py b/api/routes/templates.py deleted file mode 100644 index 64db2e05..00000000 --- a/api/routes/templates.py +++ /dev/null @@ -1,198 +0,0 @@ -import re -from datetime import datetime, timezone -from pathlib import Path - -from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile -from fastapi.responses import FileResponse -from sqlmodel import Session -from api.deps import get_db -from api.schemas.templates import ( - TemplateCreate, - TemplateResponse, - TemplateUploadResponse, - MakeFillableRequest, - MakeFillableResponse, -) -from api.db.repositories import create_template, list_templates -from api.db.models import Template -from src.controller import Controller - -router = APIRouter(prefix="/templates", tags=["templates"]) -PROJECT_ROOT = Path(__file__).resolve().parents[2] -DEFAULT_TEMPLATE_DIR = "src/inputs" - - -def _resolve_target_directory(directory: str) -> Path: - dir_value = (directory or DEFAULT_TEMPLATE_DIR).strip() - if not dir_value: - raise HTTPException(status_code=400, detail="Directory is required.") - - candidate = Path(dir_value) - if not candidate.is_absolute(): - candidate = (PROJECT_ROOT / candidate).resolve() - else: - candidate = candidate.resolve() - - if candidate != PROJECT_ROOT and PROJECT_ROOT not in candidate.parents: - raise HTTPException(status_code=400, detail="Directory must be inside the project.") - - return candidate - - -def _resolve_project_file(file_path: str) -> Path: - raw_path = (file_path or "").strip() - if not raw_path: - raise HTTPException(status_code=400, detail="Path is required.") - - candidate = Path(raw_path) - if not candidate.is_absolute(): - candidate = (PROJECT_ROOT / candidate).resolve() - else: - candidate = candidate.resolve() - - if candidate != PROJECT_ROOT and PROJECT_ROOT not in candidate.parents: - raise HTTPException(status_code=400, detail="Path must be inside the project.") - - return candidate - - -@router.post("/upload", response_model=TemplateUploadResponse) -async def upload_template_pdf( - file: UploadFile = File(...), - directory: str = Form(DEFAULT_TEMPLATE_DIR), -): - filename = Path(file.filename or "").name - if not filename: - raise HTTPException(status_code=400, detail="A PDF filename is required.") - - if not filename.lower().endswith(".pdf"): - raise HTTPException(status_code=400, detail="Only PDF files are supported.") - - target_dir = _resolve_target_directory(directory) - target_dir.mkdir(parents=True, exist_ok=True) - - target_path = target_dir / filename - if target_path.exists(): - timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") - target_path = target_dir / f"{target_path.stem}_{timestamp}{target_path.suffix}" - - content = await file.read() - with target_path.open("wb") as output_file: - output_file.write(content) - - relative_path = target_path.relative_to(PROJECT_ROOT).as_posix() - extracted = _extract_pdf_fields(relative_path) - return TemplateUploadResponse( - filename=target_path.name, - pdf_path=relative_path, - field_count=None if extracted is None else len(extracted), - fields=extracted or [], - ) - - -# PDF field-type codes -> the type values the frontend field builder uses. -_FIELD_TYPE_BY_FT = {"/Tx": "string", "/Btn": "checkbox", "/Ch": "list", "/Sig": "signature"} - - -def _pdf_text(value) -> str: - """Decode a pdfrw string (field name / tooltip) to plain text.""" - if value is None: - return "" - if hasattr(value, "to_unicode"): - return value.to_unicode().strip() - return str(value).strip() - - -def _humanize(name: str) -> str: - """Turn a raw field name into a readable description (JobTitle -> Job Title).""" - text = re.sub(r"_+", " ", name) - text = re.sub(r"(?<=[a-z])(?=[A-Z])", " ", text) - return re.sub(r"\s+", " ", text).strip() - - -def _extract_pdf_fields(pdf_path: str) -> list[dict] | None: - """Fillable widgets in the same order Filler.fill_form writes them - (top-to-bottom, left-to-right per page), so seeded rows line up with the - fill order. Returns None if the PDF can't be read.""" - try: - from pdfrw import PdfReader - candidate = Path(pdf_path) - if not candidate.is_absolute(): - candidate = (PROJECT_ROOT / candidate).resolve() - pdf = PdfReader(str(candidate)) - fields: list[dict] = [] - for page in pdf.pages: - widgets = [a for a in (page.Annots or []) if a.Subtype == "/Widget" and a.T] - widgets.sort(key=lambda a: (-float(a.Rect[1]), float(a.Rect[0]))) - for annot in widgets: - name = _pdf_text(annot.T) - fields.append({ - "name": name, - "description": _pdf_text(annot.TU) or _humanize(name), - "type": _FIELD_TYPE_BY_FT.get(str(annot.FT), "string"), - }) - return fields - except Exception: - return None - - -def _count_pdf_widgets(pdf_path: str) -> int | None: - """Number of fillable widgets in a PDF, or None if unreadable.""" - fields = _extract_pdf_fields(pdf_path) - return None if fields is None else len(fields) - - -@router.get("", response_model=list[TemplateResponse]) -def get_templates(db: Session = Depends(get_db)): - return list_templates(db) - - -@router.get("/preview") -def preview_template_pdf(path: str = Query(..., description="Project-relative PDF path")): - resolved_path = _resolve_project_file(path) - - if not resolved_path.exists() or not resolved_path.is_file(): - raise HTTPException(status_code=404, detail="PDF file not found.") - - if resolved_path.suffix.lower() != ".pdf": - raise HTTPException(status_code=400, detail="Only PDF files can be previewed.") - - return FileResponse( - resolved_path, - media_type="application/pdf", - filename=resolved_path.name, - content_disposition_type="inline", - ) - - -@router.post("/create", response_model=TemplateResponse) -def create(template: TemplateCreate, db: Session = Depends(get_db)): - tpl = Template(**template.model_dump()) - created = create_template(db, tpl) - return TemplateResponse( - id=created.id, - name=created.name, - pdf_path=created.pdf_path, - fields=created.fields, - field_count=_count_pdf_widgets(created.pdf_path), - ) - - -@router.post("/make-fillable", response_model=MakeFillableResponse) -def make_fillable(req: MakeFillableRequest): - # Validate the path stays inside the project root. - resolved = _resolve_project_file(req.pdf_path) - if not resolved.exists() or not resolved.is_file(): - raise HTTPException(status_code=404, detail="PDF file not found.") - - controller = Controller() - new_absolute = controller.prepare_fillable(str(resolved)) - new_path = Path(new_absolute) - if not new_path.is_absolute(): - new_path = (PROJECT_ROOT / new_path).resolve() - relative_path = new_path.relative_to(PROJECT_ROOT).as_posix() - - return MakeFillableResponse( - pdf_path=relative_path, - field_count=_count_pdf_widgets(relative_path), - ) diff --git a/api/schemas/common.py b/api/schemas/common.py deleted file mode 100644 index 578cd2c0..00000000 --- a/api/schemas/common.py +++ /dev/null @@ -1,14 +0,0 @@ -from pydantic import BaseModel -from typing import Any, Optional - -class SuccessResponse(BaseModel): - success: bool = True - data: Any - -class ErrorDetail(BaseModel): - code: str - message: str - -class ErrorResponse(BaseModel): - success: bool = False - error: ErrorDetail \ No newline at end of file diff --git a/api/schemas/forms.py b/api/schemas/forms.py deleted file mode 100644 index ca996515..00000000 --- a/api/schemas/forms.py +++ /dev/null @@ -1,33 +0,0 @@ -from pydantic import BaseModel, field_validator - -class FormFill(BaseModel): - template_id: int - input_text: str - # Optional Ollama model override for this fill; falls back to OLLAMA_MODEL. - # Not persisted (no DB column) — excluded before building FormSubmission. - model: str | None = None - - @field_validator("input_text") - def validate_input_text(cls, value): - if not value or not value.strip(): - raise ValueError("Input text cannot be empty") - return value - - -class FormFillResponse(BaseModel): - id: int - template_id: int - input_text: str - output_pdf_path: str - - class Config: - from_attributes = True - - -class TranscriptionResponse(BaseModel): - text: str - - -class ModelsResponse(BaseModel): - models: list[str] - default: str \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..de9379be --- /dev/null +++ b/app/__init__.py @@ -0,0 +1,8 @@ +"""FireForm backend application package.""" + +import os + +# Force CPU before any service module imports torch / rfdetr. Prevents PyTorch +# from probing for NVIDIA drivers on Mac Silicon and inside Docker. Runs here so +# it is guaranteed to execute before `app.main` imports the service layer. +os.environ["CUDA_VISIBLE_DEVICES"] = "" diff --git a/api/__init__.py b/app/api/__init__.py similarity index 100% rename from api/__init__.py rename to app/api/__init__.py diff --git a/app/api/deps.py b/app/api/deps.py new file mode 100644 index 00000000..7850dab0 --- /dev/null +++ b/app/api/deps.py @@ -0,0 +1,22 @@ +from fastapi import Header, Request + +from app.core.config import FIREFORM_API_KEY +from app.core.errors.base import AppError +from app.db.database import get_session + + +def get_db(): + yield from get_session() + +def verify_api_key(request: Request, x_api_key: str | None = Header(None)): + if not FIREFORM_API_KEY: + return + + provided_key = x_api_key + if not provided_key: + auth_header = request.headers.get("Authorization") + if auth_header and auth_header.startswith("Bearer "): + provided_key = auth_header[len("Bearer "):] + + if not provided_key or provided_key != FIREFORM_API_KEY: + raise AppError("Unauthorized", status_code=401, error_code="UNAUTHORIZED") \ No newline at end of file diff --git a/app/api/router.py b/app/api/router.py new file mode 100644 index 00000000..cc59847a --- /dev/null +++ b/app/api/router.py @@ -0,0 +1,13 @@ +from fastapi import APIRouter + +from app.api.routes import forms, input, jobs, system, templates, weather, zipcode +from app.core.config import API_PREFIX + +api_router = APIRouter() +api_router.include_router(templates.router, prefix=API_PREFIX) +api_router.include_router(forms.router, prefix=API_PREFIX) +api_router.include_router(system.router, prefix=API_PREFIX) +api_router.include_router(jobs.router, prefix=API_PREFIX) +api_router.include_router(weather.router, prefix=API_PREFIX) +api_router.include_router(zipcode.router, prefix=API_PREFIX) +api_router.include_router(input.router, prefix=API_PREFIX) \ No newline at end of file diff --git a/app/api/routes/__init__.py b/app/api/routes/__init__.py new file mode 100644 index 00000000..d99c8894 --- /dev/null +++ b/app/api/routes/__init__.py @@ -0,0 +1,3 @@ +from . import forms, templates + +__all__ = ["forms", "templates"] diff --git a/app/api/routes/forms.py b/app/api/routes/forms.py new file mode 100644 index 00000000..805d18df --- /dev/null +++ b/app/api/routes/forms.py @@ -0,0 +1,130 @@ +import requests +from fastapi import APIRouter, Depends, File, UploadFile, Query +from sqlmodel import Session + +from app.api.deps import get_db, verify_api_key +from app.api.schemas.forms import ( + FormFill, + FormFillResponse, + ModelsResponse, + TranscriptionResponse, +) +from app.core import paths +from app.core.config import OLLAMA_HOST, OLLAMA_MODEL, RETENTION_PERIOD_DAYS +from app.services.whisper import call_whisper_asr +from app.core.errors.base import AppError +from app.db.repositories import get_template, get_form_submission, delete_form_submission +from app.services.form import FormService + +router = APIRouter(prefix="/forms", tags=["forms"]) + + +@router.post("/fill", response_model=FormFillResponse) +def fill_form(form: FormFill, db: Session = Depends(get_db)): + fetched_template = get_template(db, form.template_id) + if not fetched_template: + raise AppError("Template not found", status_code=404, error_code="TEMPLATE_NOT_FOUND") + + svc = FormService() + try: + submission = svc.fill_form( + db, + template=fetched_template, + input_id=form.input_id, + input_text=form.input_text, + model=form.model, + ) + return FormFillResponse( + id=submission.id, + template_id=submission.template_id, + input_text=submission.input_text, + output_pdf_path=submission.output_pdf_path, + ) + except AppError: + raise + except Exception as e: + raise AppError(str(e), status_code=500, error_code="FORM_FILL_ERROR") + + + +@router.get("/models", response_model=ModelsResponse) +def list_models(): + """List the Whisper-independent extraction models available in the local + Ollama instance, plus the configured default. Used by the Fill Form UI's + model picker. Falls back to just the default if Ollama is unreachable.""" + default_model = OLLAMA_MODEL + + models: list[str] = [] + try: + response = requests.get(f"{OLLAMA_HOST}/api/tags", timeout=10) + response.raise_for_status() + models = [m["name"] for m in response.json().get("models", []) if m.get("name")] + except requests.exceptions.RequestException: + models = [] + + # Always surface the configured default, even if Ollama hasn't pulled it yet. + if default_model not in models: + models.insert(0, default_model) + + return ModelsResponse(models=models, default=default_model) + + +@router.post("/transcribe", response_model=TranscriptionResponse) +def transcribe(audio: UploadFile = File(...)): + """Forward recorded audio to the local Whisper ASR sidecar and return text. + + Mirrors the Ollama wiring: WHISPER_HOST points at the whisper service + (http://whisper:9000 inside Docker, http://localhost:9000 otherwise). The + audio is streamed straight through to the local STT service and never + persisted — no PII leaves the machine. + """ + try: + text = call_whisper_asr( + audio.file.read(), + audio.filename or "audio.wav", + audio.content_type or "audio/wav", + ) + except ConnectionError as exc: + raise AppError(str(exc), status_code=503, error_code="STT_UNAVAILABLE") + except RuntimeError as exc: + raise AppError(str(exc), status_code=502, error_code="TRANSCRIPTION_FAILED") + + return TranscriptionResponse(text=text) + + +@router.delete("/{submission_id}", dependencies=[Depends(verify_api_key)]) +def delete_submission_endpoint(submission_id: int, db: Session = Depends(get_db)): + sub = get_form_submission(db, submission_id) + if not sub: + raise AppError("Submission not found", status_code=404, error_code="SUBMISSION_NOT_FOUND") + + if sub.output_pdf_path: + try: + resolved_out = paths._resolve_project_file(sub.output_pdf_path) + if resolved_out.exists() and resolved_out.is_file(): + resolved_out.unlink() + except Exception: + pass + + delete_form_submission(db, sub) + return {"status": "success", "message": "Submission and associated output file deleted"} + + +@router.post("/purge", dependencies=[Depends(verify_api_key)]) +def purge_submissions_endpoint( + days: int | None = Query(default=None, ge=1, le=3650, description="Retention window in days (must be >= 1)"), + db: Session = Depends(get_db), +): + retention_days = days if days is not None else RETENTION_PERIOD_DAYS + purged_count = FormService().purge_submissions(db, retention_days) + return {"status": "success", "purged_count": purged_count, "retention_days_used": retention_days} + + +@router.get("/submissions") +def get_submissions(db: Session = Depends(get_db)): + return FormService().list_submissions(db) + + +@router.get("/submissions/analytics") +def get_submissions_analytics(db: Session = Depends(get_db)): + return FormService().get_analytics(db) diff --git a/app/api/routes/input.py b/app/api/routes/input.py new file mode 100644 index 00000000..d0afce64 --- /dev/null +++ b/app/api/routes/input.py @@ -0,0 +1,173 @@ +from datetime import date +from pathlib import Path +from uuid import UUID + +from fastapi import APIRouter, Depends, File, Form, UploadFile +from fastapi.exceptions import RequestValidationError +from sqlmodel import Session + +from app.api.deps import get_db +from app.api.schemas.enums import InputStatus +from app.api.schemas.input import ( + InputRecordResponse, + TextInputRequest, + TextInputResponse, + VoiceInputResponse, +) +from app.core.config import ( + ALLOWED_AUDIO_EXTENSIONS, + AUDIO_CONTENT_TYPES, + ESTIMATED_TRANSCRIPTION_SECONDS, + INPUT_POLL_INTERVAL_SECONDS, +) +from app.core.errors.base import AppError +from app.db.repositories import create_input +from app.db.repositories import get_input as repo_get_input +from app.services.input import InputService +from app.services.whisper import check_whisper_available + +router = APIRouter(prefix="/input", tags=["input"]) + +_MAX_AUDIO_BYTES = 500 * 1024 * 1024 # 500 MB + + +@router.post("/voice", response_model=VoiceInputResponse, status_code=201) +def submit_voice_input( + audio_file: UploadFile = File(...), + station_id: str | None = Form(default=None), + responder_badge: str | None = Form(default=None), + incident_date_hint: date | None = Form(default=None), + db: Session = Depends(get_db), +): + # 415 — format check (HTTP concern, before any I/O) + filename = audio_file.filename or "" + ext = Path(filename).suffix.lstrip(".").lower() + if not ext or ext not in ALLOWED_AUDIO_EXTENSIONS: + raise AppError( + "Audio format not supported. Accepted formats: wav, mp3, m4a, ogg, webm", + status_code=415, + error_code="UNSUPPORTED_FORMAT", + detail={"accepted_formats": list(AUDIO_CONTENT_TYPES)}, + ) + + # 413 — size check: fast path via Content-Length (no read), fallback via len(bytes) + if audio_file.size is not None and audio_file.size > _MAX_AUDIO_BYTES: + raise AppError( + "Audio file exceeds maximum size of 500MB", + status_code=413, + error_code="FILE_TOO_LARGE", + detail={"max_size_bytes": _MAX_AUDIO_BYTES, "received_size_bytes": audio_file.size}, + ) + content = audio_file.file.read() + if len(content) > _MAX_AUDIO_BYTES: + raise AppError( + "Audio file exceeds maximum size of 500MB", + status_code=413, + error_code="FILE_TOO_LARGE", + detail={"max_size_bytes": _MAX_AUDIO_BYTES, "received_size_bytes": len(content)}, + ) + + # 503 — Whisper availability (HTTP concern) + if not check_whisper_available(): + raise AppError( + "Whisper transcription service is not available", + status_code=503, + error_code="STT_UNAVAILABLE", + ) + + svc = InputService() + record, job = svc.process_voice_upload( + session=db, + audio_content=content, + ext=ext, + filename=filename, + station_id=station_id, + responder_badge=responder_badge, + incident_date_hint=incident_date_hint, + ) + + return VoiceInputResponse( + input_id=record.input_id, + status=record.status, + input_type=record.input_type, + job_id=job.job_id, + poll_url=f"/api/v1/input/{record.input_id}", + estimated_processing_seconds=ESTIMATED_TRANSCRIPTION_SECONDS, + created_at=record.created_at, + ) + + +@router.post("/text", response_model=TextInputResponse, status_code=201) +def submit_text_input(body: TextInputRequest, db: Session = Depends(get_db)): + if len(body.narrative) > 50_000: + raise AppError( + "Narrative exceeds maximum length of 50,000 characters", + status_code=413, + error_code="NARRATIVE_TOO_LONG", + detail={"max_characters": 50_000, "received_characters": len(body.narrative)}, + ) + + svc = InputService() + try: + record = svc.build_text_input( + narrative=body.narrative, + station_id=body.station_id, + responder_badge=body.responder_badge, + incident_date_hint=body.incident_date_hint, + ) + except ValueError as exc: + raise RequestValidationError( + errors=[ + { + "loc": ("body", "narrative"), + "msg": str(exc), + "input": body.narrative, + "type": "value_error", + } + ] + ) + + record = create_input(db, record) + + return TextInputResponse( + input_id=record.input_id, + status=record.status, + input_type=record.input_type, + character_count=record.character_count, + word_count=record.word_count, + created_at=record.created_at, + ) + + +@router.get("/{input_id}", response_model=InputRecordResponse) +def get_input(input_id: UUID, db: Session = Depends(get_db)): + record = repo_get_input(db, input_id) + if record is None: + raise AppError( + f"Input with ID {input_id} not found", + status_code=404, + error_code="INPUT_NOT_FOUND", + ) + + retry_after = ( + INPUT_POLL_INTERVAL_SECONDS + if record.status in (InputStatus.queued, InputStatus.transcribing) + else None + ) + return InputRecordResponse( + input_id=record.input_id, + input_type=record.input_type, + status=record.status, + transcript=record.transcript, + original_filename=record.original_filename, + audio_duration_seconds=record.audio_duration_seconds, + character_count=record.character_count, + word_count=record.word_count, + station_id=record.station_id, + responder_badge=record.responder_badge, + incident_date_hint=record.incident_date_hint, + error_detail=record.error_detail, + retry_after_seconds=retry_after, + created_at=record.created_at, + updated_at=record.updated_at, + ) diff --git a/app/api/routes/jobs.py b/app/api/routes/jobs.py new file mode 100644 index 00000000..4a4f3826 --- /dev/null +++ b/app/api/routes/jobs.py @@ -0,0 +1,63 @@ +from fastapi import APIRouter, Depends +from sqlmodel import Session + +from app.api.deps import get_db +from app.api.schemas.forms import ( + AsyncFormFill, + AsyncFormFillResponse, + AsyncJobSubmitResponse, + JobResponse, +) +from app.core.errors.base import AppError +from app.db.repositories import create_job, get_job_by_uuid, get_template +from app.models import Job +from app.services.input import InputService +from app.tasks.fill import fill_form_task + +router = APIRouter(tags=["jobs"]) + + +@router.get("/jobs/{job_id}", response_model=JobResponse) +def get_job_status(job_id: str, db: Session = Depends(get_db)): + job = get_job_by_uuid(db, job_id) + if not job: + raise AppError("Job not found", status_code=404) + return JobResponse( + job_id=job.job_id, + job_type=job.job_type, + status=job.status, + progress_percent=job.progress_percent, + result_url=job.result_url, + error=job.error, + created_at=job.created_at.isoformat() if job.created_at else None, + updated_at=job.updated_at.isoformat() if job.updated_at else None, + ) + + +@router.post("/forms/jobs", response_model=AsyncFormFillResponse) +def submit_async_form_fill(form: AsyncFormFill, db: Session = Depends(get_db)): + for tid in form.template_ids: + if not get_template(db, tid): + raise AppError(f"Template {tid} not found", status_code=404) + + transcript = InputService().resolve_transcript(db, form.input_id) + + jobs: list[AsyncJobSubmitResponse] = [] + for tid in form.template_ids: + result = fill_form_task.delay(tid, transcript, str(form.input_id), form.model) + job = Job( + celery_task_id=result.id, + job_type="form_generation", + template_id=tid, + input_text=transcript, + status="queued", + model=form.model, + ) + job = create_job(db, job) + jobs.append(AsyncJobSubmitResponse( + job_id=job.job_id, + status=job.status, + poll_url=f"/api/v1/jobs/{job.job_id}", + )) + + return AsyncFormFillResponse(jobs=jobs) diff --git a/app/api/routes/system.py b/app/api/routes/system.py new file mode 100644 index 00000000..eadd1d9f --- /dev/null +++ b/app/api/routes/system.py @@ -0,0 +1,181 @@ +"""System endpoints: GET /api/v1/health, /schema/incident, /schema/incident/versions. + +Health contract: contracts/path/system.yaml + contracts/schemas/system.yaml +""" + +import shutil +import time + +import requests +from fastapi import APIRouter +from fastapi.responses import JSONResponse +from sqlalchemy import text + +from app.api.schemas.system import ( + ComponentHealth, + HealthComponents, + HealthStatus, + ModelInfo, +) +from app.core.config import APP_VERSION, DATA_DIR, OLLAMA_HOST, WHISPER_HOST +from app.db.database import engine + +router = APIRouter(tags=["system"]) + +_START_TIME = time.monotonic() + +# Tune-able thresholds. _SLOW_MS is patched in tests to trigger degraded +# without mocking wall-clock time. +_PROBE_TIMEOUT = 5 +_SLOW_MS = 5_000 + + +def _check_database() -> ComponentHealth: + t0 = time.monotonic() + try: + with engine.connect() as conn: + conn.execute(text("SELECT 1")) + elapsed = int((time.monotonic() - t0) * 1000) + return ComponentHealth(status="healthy", response_time_ms=elapsed) + except Exception as exc: + return ComponentHealth(status="unhealthy", detail=str(exc)) + + +def _check_ollama() -> ComponentHealth: + t0 = time.monotonic() + try: + tags_resp = requests.get(f"{OLLAMA_HOST}/api/tags", timeout=_PROBE_TIMEOUT) + tags_resp.raise_for_status() + elapsed = int((time.monotonic() - t0) * 1000) + + tags_data = tags_resp.json() + + ollama_version: str | None = None + try: + ver_resp = requests.get(f"{OLLAMA_HOST}/api/version", timeout=_PROBE_TIMEOUT) + ver_resp.raise_for_status() + ollama_version = ver_resp.json().get("version") + except Exception: + pass + + running_names: set[str] = set() + model_loaded: str | None = None + try: + ps_resp = requests.get(f"{OLLAMA_HOST}/api/ps", timeout=_PROBE_TIMEOUT) + ps_resp.raise_for_status() + running = ps_resp.json().get("models", []) + running_names = {m.get("name", "") for m in running} + if running: + model_loaded = running[0].get("name") + except Exception: + pass + + raw_models = tags_data.get("models", []) + models_available: list[ModelInfo] = [] + for m in raw_models: + name = m.get("name", "") + size_gb = round(m.get("size", 0) / (1024 ** 3), 2) + quantization = m.get("details", {}).get("quantization_level") + models_available.append( + ModelInfo(name=name, size_gb=size_gb, quantization=quantization, loaded=name in running_names) + ) + + status = "degraded" if elapsed > _SLOW_MS else "healthy" + + return ComponentHealth( + status=status, + response_time_ms=elapsed, + model_loaded=model_loaded, + ollama_version=ollama_version, + models_available=models_available if models_available else None, + # current_load is always None — Ollama exposes no queue-depth API; + # populating it would require fabricating data. + current_load=None, + ) + except requests.exceptions.RequestException as exc: + return ComponentHealth(status="unhealthy", detail=str(exc)) + + +def _check_whisper() -> ComponentHealth: + # The onerahmet/openai-whisper-asr-webservice image does not expose /health. + # Both compose files probe /docs (a FastAPI auto-endpoint always present). + t0 = time.monotonic() + try: + resp = requests.get(f"{WHISPER_HOST}/docs", timeout=_PROBE_TIMEOUT) + resp.raise_for_status() + elapsed = int((time.monotonic() - t0) * 1000) + return ComponentHealth(status="healthy", response_time_ms=elapsed) + except requests.exceptions.RequestException as exc: + return ComponentHealth(status="unhealthy", detail=str(exc)) + + +def _check_storage() -> ComponentHealth: + try: + usage = shutil.disk_usage(DATA_DIR) + disk_free_gb = round(usage.free / (1024 ** 3), 2) + return ComponentHealth(status="healthy", disk_free_gb=disk_free_gb) + except OSError as exc: + return ComponentHealth(status="unhealthy", detail=str(exc)) + + +@router.get( + "/health", + responses={ + 200: {"model": HealthStatus, "description": "System healthy or degraded"}, + 503: {"model": HealthStatus, "description": "System unhealthy, cannot serve requests"}, + }, + summary="System health check", +) +def get_health(): + database = _check_database() + ollama = _check_ollama() + whisper = _check_whisper() + storage = _check_storage() + + components = HealthComponents( + database=database, ollama=ollama, whisper=whisper, storage=storage + ) + + statuses = {database.status, ollama.status, whisper.status, storage.status} + + if database.status == "unhealthy": + overall = "unhealthy" + elif "unhealthy" in statuses or "degraded" in statuses: + overall = "degraded" + else: + overall = "healthy" + + body = HealthStatus( + status=overall, + version=APP_VERSION, + uptime_seconds=int(time.monotonic() - _START_TIME), + components=components, + ) + + http_status = 503 if overall == "unhealthy" else 200 + return JSONResponse( + content=body.model_dump(exclude_none=True), + status_code=http_status, + ) + + +@router.get("/schema/incident", summary="Get canonical incident JSON Schema") +def get_schema_incident(): + return JSONResponse( + status_code=501, + content={ + "error_code": "NOT_IMPLEMENTED", + "message": "Incident schema not yet finalised — see issue #555", + }, + ) + + +@router.get("/schema/incident/versions", summary="Get incident schema version history") +def get_schema_versions(): + return JSONResponse( + status_code=501, + content={ + "error_code": "NOT_IMPLEMENTED", + "message": "Schema version history not yet available — see issue #555", + }, + ) diff --git a/app/api/routes/templates.py b/app/api/routes/templates.py new file mode 100644 index 00000000..797c3470 --- /dev/null +++ b/app/api/routes/templates.py @@ -0,0 +1,83 @@ +from pathlib import Path + +from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile +from fastapi.responses import FileResponse +from sqlmodel import Session + +from app.api.deps import get_db, verify_api_key +from app.api.schemas.templates import ( + MakeFillableRequest, + MakeFillableResponse, + TemplateCreate, + TemplateResponse, + TemplateUploadResponse, +) +from app.core.config import DEFAULT_TEMPLATE_DIR +from app.db.repositories import get_template +from app.services.template import TemplateService + +router = APIRouter(prefix="/templates", tags=["templates"]) + + +@router.post("/upload", response_model=TemplateUploadResponse) +async def upload_template_pdf( + file: UploadFile = File(...), + directory: str = Form(DEFAULT_TEMPLATE_DIR), +): + filename = Path(file.filename or "").name + if not filename: + raise HTTPException(status_code=400, detail="A PDF filename is required.") + + if not filename.lower().endswith(".pdf"): + raise HTTPException(status_code=400, detail="Only PDF files are supported.") + + content = await file.read() + return TemplateService().save_uploaded_pdf(directory, filename, content) + + +@router.get("", response_model=list[TemplateResponse]) +def get_templates(db: Session = Depends(get_db)): + return TemplateService().list_templates(db) + + +@router.get("/preview") +def preview_template_pdf(path: str = Query(..., description="Project-relative PDF path")): + resolved_path = TemplateService().resolve_pdf_path(path) + + if not resolved_path.exists() or not resolved_path.is_file(): + raise HTTPException(status_code=404, detail="PDF file not found.") + + if resolved_path.suffix.lower() != ".pdf": + raise HTTPException(status_code=400, detail="Only PDF files can be previewed.") + + return FileResponse( + resolved_path, + media_type="application/pdf", + filename=resolved_path.name, + content_disposition_type="inline", + ) + + +@router.post("/create", response_model=TemplateResponse) +def create(template: TemplateCreate, db: Session = Depends(get_db)): + return TemplateService().create_template(db, template) + + +@router.post("/make-fillable", response_model=MakeFillableResponse) +def make_fillable(req: MakeFillableRequest): + svc = TemplateService() + resolved = svc.resolve_pdf_path(req.pdf_path) + if not resolved.exists() or not resolved.is_file(): + raise HTTPException(status_code=404, detail="PDF file not found.") + + return svc.make_fillable(str(resolved)) + + +@router.delete("/{template_id}", dependencies=[Depends(verify_api_key)]) +def delete_template_endpoint(template_id: int, db: Session = Depends(get_db)): + template = get_template(db, template_id) + if not template: + raise HTTPException(status_code=404, detail="Template not found") + + TemplateService().delete_template(db, template) + return {"status": "success", "message": "Template and all associated data deleted"} diff --git a/app/api/routes/weather.py b/app/api/routes/weather.py new file mode 100644 index 00000000..11735996 --- /dev/null +++ b/app/api/routes/weather.py @@ -0,0 +1,42 @@ +from fastapi import APIRouter + +from app.core.errors.base import AppError +from app.services.external_apis.weather_api import WeatherAPI + +router = APIRouter(prefix="/weather", tags=["weather"]) + +ALL_HOURLY_FIELDS = [ + "temperature_2m", + "relative_humidity_2m", + "rain", + "apparent_temperature", + "precipitation_probability", + "precipitation", + "wind_direction_10m", + "wind_speed_10m", + "soil_temperature_0cm", + "uv_index", +] + + +@router.get("/forecast") +def get_weather_forecast( + latitude: float, + longitude: float, + start_date: str, + end_date: str, + fields: str | None = None, +): + """ + Fetch hourly weather forecast data for the given coordinates and date range. + + Coordinates must be supplied as **decimal degrees** — not degree/minute/second notation. E.g. -122.4194, 37.7749. + Dates must use the **YYYY-MM-DD** format (ISO 8601). + """ + weather_api = WeatherAPI() + try: + requested = [f.strip() for f in fields.split(",") if f.strip()] if fields else [] + weather_data = weather_api.get_weather(latitude, longitude, start_date, end_date, hourly_fields=requested) + return weather_data + except Exception as e: + raise AppError(str(e), status_code=500) \ No newline at end of file diff --git a/app/api/routes/zipcode.py b/app/api/routes/zipcode.py new file mode 100644 index 00000000..e2d7cdbd --- /dev/null +++ b/app/api/routes/zipcode.py @@ -0,0 +1,29 @@ +from fastapi import APIRouter + +from app.core.errors.base import AppError +from app.services.external_apis.zipcode_api import ZipCodeAPI + +router = APIRouter(prefix="/zipcode", tags=["zipcode"]) + + +@router.get("/lookup-address") +def lookup_address(address: str): + """ + Resolve a free-form address to one or more geographic locations including + postal code, latitude, longitude, place name, state, county, and country. + + Returns a list of matching results ordered by relevance (most relevant first). + Each result is guaranteed to include `latitude` and `longitude`; `postal_code` + is present when the geocoding service can determine it for the given address. + + Example: + address = "1600 Amphitheatre Parkway, Mountain View, CA, US" + """ + zipcode_api = ZipCodeAPI() + try: + return zipcode_api.lookup_address(address) + except TimeoutError as e: + raise AppError(str(e), status_code=504) + except Exception as e: + raise AppError(str(e), status_code=500) + diff --git a/api/db/__init__.py b/app/api/schemas/__init__.py similarity index 100% rename from api/db/__init__.py rename to app/api/schemas/__init__.py diff --git a/app/api/schemas/common.py b/app/api/schemas/common.py new file mode 100644 index 00000000..8d29d318 --- /dev/null +++ b/app/api/schemas/common.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, ConfigDict + +from app.api.schemas.enums import JobStatus, JobType + + +class ValidationErrorItem(BaseModel): + model_config = ConfigDict() + field: str | None = None + issue: str | None = None + value: Any = None + + +class ErrorResponse(BaseModel): + model_config = ConfigDict() + error_code: str + message: str + detail: dict[str, Any] | None = None + retry_after_seconds: int | None = None + validation_errors: list[ValidationErrorItem] | None = None + + +class Pagination(BaseModel): + model_config = ConfigDict() + total: int + page: int + per_page: int + total_pages: int + has_next: bool + has_prev: bool + + +class AsyncJobResponse(BaseModel): + model_config = ConfigDict() + job_id: str + job_type: JobType | None = None + status: JobStatus + estimated_seconds: int | None = None + poll_url: str | None = None + + +class Job(BaseModel): + model_config = ConfigDict() + job_id: str + job_type: JobType + status: JobStatus + progress_percent: int | None = None + result_url: str | None = None + error: ErrorResponse | None = None + created_at: datetime | None = None + updated_at: datetime | None = None diff --git a/app/api/schemas/enums.py b/app/api/schemas/enums.py new file mode 100644 index 00000000..b6335389 --- /dev/null +++ b/app/api/schemas/enums.py @@ -0,0 +1,116 @@ +from enum import Enum + + +class InputType(str, Enum): + voice = "voice" + text = "text" + + +class InputStatus(str, Enum): + queued = "queued" + transcribing = "transcribing" + ready = "ready" + failed = "failed" + + +class ExtractionStatus(str, Enum): + processing = "processing" + completed = "completed" + failed = "failed" + needs_review = "needs_review" + + +class FormStatus(str, Enum): + queued = "queued" + generating = "generating" + completed = "completed" + failed = "failed" + + +class ReportStatus(str, Enum): + draft = "draft" + under_review = "under_review" + approved = "approved" + submitted = "submitted" + + +class JobStatus(str, Enum): + queued = "queued" + processing = "processing" + completed = "completed" + failed = "failed" + + +class JobType(str, Enum): + transcription = "transcription" + extraction = "extraction" + form_generation = "form_generation" + batch_form_generation = "batch_form_generation" + report_generation = "report_generation" + + +class FormType(str, Enum): + neris = "neris" + nemsis_epcr = "nemsis_epcr" + nibrs = "nibrs" + nfirs_basic = "nfirs_basic" + nfirs_fire = "nfirs_fire" + nfirs_structure = "nfirs_structure" + nfirs_wildland = "nfirs_wildland" + nfirs_ems = "nfirs_ems" + nfirs_hazmat = "nfirs_hazmat" + nfirs_apparatus = "nfirs_apparatus" + nfirs_personnel = "nfirs_personnel" + nfirs_arson = "nfirs_arson" + nfirs_casualty_civilian = "nfirs_casualty_civilian" + nfirs_casualty_responder = "nfirs_casualty_responder" + cal_fire_ics209 = "cal_fire_ics209" + osha_301 = "osha_301" + un_ssirs = "un_ssirs" + state_georgia = "state_georgia" + state_california = "state_california" + state_new_york = "state_new_york" + + +class IncidentCategory(str, Enum): + fire = "fire" + ems = "ems" + rescue = "rescue" + hazardous_conditions = "hazardous_conditions" + service_call = "service_call" + good_intent = "good_intent" + false_alarm = "false_alarm" + law_enforcement = "law_enforcement" + + +class CauseCertainty(str, Enum): + confirmed = "confirmed" + probable = "probable" + suspected = "suspected" + undetermined = "undetermined" + + +class InjurySeverity(str, Enum): + minor = "minor" + moderate = "moderate" + severe = "severe" + fatal = "fatal" + + +class RateOfSpread(str, Enum): + slow = "slow" + moderate = "moderate" + rapid = "rapid" + extreme = "extreme" + + +class PeriodType(str, Enum): + monthly = "monthly" + quarterly = "quarterly" + annual = "annual" + + +class OutputFormat(str, Enum): + pdf = "pdf" + json = "json" + both = "both" diff --git a/app/api/schemas/forms.py b/app/api/schemas/forms.py new file mode 100644 index 00000000..00db3a14 --- /dev/null +++ b/app/api/schemas/forms.py @@ -0,0 +1,68 @@ +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, field_validator + + +class FormFill(BaseModel): + template_id: int + input_text: str | None = None + input_id: UUID | None = None + model: str | None = None + + +class FormFillResponse(BaseModel): + id: int + template_id: int + input_text: str + output_pdf_path: str + values: dict | None = None + + model_config = ConfigDict(from_attributes=True) + + +class TranscriptionResponse(BaseModel): + text: str + + +class ModelsResponse(BaseModel): + models: list[str] + default: str + + +class AsyncFormFill(BaseModel): + template_ids: list[int] + input_id: UUID + model: str | None = None + + @field_validator("template_ids") + def validate_template_ids(cls, value): + if not value: + raise ValueError("template_ids cannot be empty") + return value + + +class JobResponse(BaseModel): + job_id: str + job_type: str + status: str + progress_percent: int = 0 + result_url: str | None = None + error: dict | None = None + created_at: str | None = None + updated_at: str | None = None + + class Config: + from_attributes = True + + +class AsyncJobSubmitResponse(BaseModel): + job_id: str + status: str + poll_url: str + + class Config: + from_attributes = True + + +class AsyncFormFillResponse(BaseModel): + jobs: list[AsyncJobSubmitResponse] \ No newline at end of file diff --git a/app/api/schemas/input.py b/app/api/schemas/input.py new file mode 100644 index 00000000..9d8b908c --- /dev/null +++ b/app/api/schemas/input.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from datetime import date, datetime +from uuid import UUID + +from pydantic import BaseModel, Field + +from app.api.schemas.enums import InputStatus, InputType + + +class VoiceInputResponse(BaseModel): + input_id: UUID + status: InputStatus + input_type: InputType + estimated_processing_seconds: int | None = None + created_at: datetime | None = None + job_id: str | None = None + poll_url: str | None = None + + +class TextInputRequest(BaseModel): + narrative: str = Field(min_length=20) + station_id: str | None = None + responder_badge: str | None = None + incident_date_hint: date | None = None + + +class TextInputResponse(BaseModel): + input_id: UUID + status: InputStatus + input_type: InputType + character_count: int | None = None + word_count: int | None = None + created_at: datetime | None = None + + +class InputRecordResponse(BaseModel): + input_id: UUID + input_type: InputType + status: InputStatus + transcript: str | None = None + original_filename: str | None = None + audio_duration_seconds: float | None = None + character_count: int | None = None + word_count: int | None = None + station_id: str | None = None + responder_badge: str | None = None + incident_date_hint: date | None = None + error_detail: str | None = None + retry_after_seconds: int | None = None + created_at: datetime | None = None + updated_at: datetime | None = None diff --git a/app/api/schemas/system.py b/app/api/schemas/system.py new file mode 100644 index 00000000..bafa5cfc --- /dev/null +++ b/app/api/schemas/system.py @@ -0,0 +1,45 @@ +from pydantic import BaseModel + + +class ModelInfo(BaseModel): + name: str + size_gb: float + quantization: str | None = None + loaded: bool + + +class CurrentLoad(BaseModel): + active_requests: int + queued_requests: int + + +class ComponentHealth(BaseModel): + status: str + response_time_ms: int | None = None + detail: str | None = None + model_loaded: str | None = None + ollama_version: str | None = None + models_available: list[ModelInfo] | None = None + current_load: CurrentLoad | None = None + disk_free_gb: float | None = None + + +class HealthComponents(BaseModel): + database: ComponentHealth + ollama: ComponentHealth + whisper: ComponentHealth + storage: ComponentHealth + + +class HealthStatus(BaseModel): + status: str + version: str + uptime_seconds: int | None = None + components: HealthComponents | None = None + + +class SchemaVersion(BaseModel): + version: str + released_at: str + changelog: str | None = None + breaking_changes: bool | None = None diff --git a/api/schemas/templates.py b/app/api/schemas/templates.py similarity index 69% rename from api/schemas/templates.py rename to app/api/schemas/templates.py index df39832b..e730aa72 100644 --- a/api/schemas/templates.py +++ b/app/api/schemas/templates.py @@ -1,9 +1,10 @@ -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict + class TemplateCreate(BaseModel): name: str pdf_path: str - fields: dict + fields: dict = {} class MakeFillableRequest(BaseModel): @@ -14,6 +15,7 @@ class MakeFillableResponse(BaseModel): pdf_path: str field_count: int | None = None + class TemplateResponse(BaseModel): id: int name: str @@ -21,14 +23,13 @@ class TemplateResponse(BaseModel): fields: dict field_count: int | None = None - class Config: - from_attributes = True + model_config = ConfigDict(from_attributes=True) class ExtractedField(BaseModel): name: str - description: str - type: str + description: str = "" + type: str = "string" class TemplateUploadResponse(BaseModel): @@ -36,3 +37,6 @@ class TemplateUploadResponse(BaseModel): pdf_path: str field_count: int | None = None fields: list[ExtractedField] = [] + schema_data: dict | None = None + tables: list[dict] | None = None + diff --git a/api/errors/__init__.py b/app/core/__init__.py similarity index 100% rename from api/errors/__init__.py rename to app/core/__init__.py diff --git a/app/core/celery.py b/app/core/celery.py new file mode 100644 index 00000000..8e7d9bba --- /dev/null +++ b/app/core/celery.py @@ -0,0 +1,29 @@ +from celery import Celery +from app.core.config import CELERY_BROKER_URL, CELERY_RESULT_BACKEND +# Optional Celery Beat schedule — runs purge_old_submissions once a day. +# Enable by running: celery -A app.core.celery beat +from celery.schedules import crontab + +celery_app = Celery( + "fireform", + broker=CELERY_BROKER_URL, + backend=CELERY_RESULT_BACKEND, +) + +celery_app.conf.update( + task_serializer="json", + result_serializer="json", + accept_content=["json"], + task_track_started=True, + result_expires=86400, +) + +celery_app.conf.include = ["app.tasks.fill", "app.tasks.purge", "app.tasks.transcribe"] + + +celery_app.conf.beat_schedule = { + "daily-submission-purge": { + "task": "purge_old_submissions", + "schedule": crontab(hour=3, minute=0), # 03:00 UTC daily + }, +} diff --git a/app/core/config.py b/app/core/config.py new file mode 100644 index 00000000..3e153cf2 --- /dev/null +++ b/app/core/config.py @@ -0,0 +1,88 @@ +"""Central configuration. + +Single source of truth for paths, the database URL, external service hosts and +CORS. Read environment once here so the rest of the app imports settings instead +of calling os.getenv() in scattered places. +""" + +import os +from pathlib import Path + +# Repo root. config.py lives at app/core/config.py -> parents[2] is the repo root. +BASE_DIR = Path(__file__).resolve().parents[2] + +# --- App metadata --------------------------------------------------------- +APP_TITLE = "FireForm API" +APP_VERSION = "1.1.0" + +# --- Runtime data paths --------------------------------------------------- +# Uploaded templates and generated PDFs. Project-relative paths the API echoes +# back to the client are resolved against BASE_DIR (the "inside the project" +# guard in the templates routes). Override the data dir with FIREFORM_DATA_DIR. +DATA_DIR = Path(os.getenv("FIREFORM_DATA_DIR", BASE_DIR / "data")).resolve() + +# Directory new uploads land in, as a project-relative string (was "src/inputs" +# before the restructure). Override with FIREFORM_TEMPLATE_DIR. +DEFAULT_TEMPLATE_DIR = os.getenv("FIREFORM_TEMPLATE_DIR", "data/inputs") + +# --- Database ------------------------------------------------------------- +DATABASE_URL = os.getenv( + "DATABASE_URL", + "postgresql://fireform:fireform@localhost:5432/fireform", +) +DB_ECHO = os.getenv("FIREFORM_DB_ECHO", "true").lower() == "true" + +# --- External services ---------------------------------------------------- +OLLAMA_HOST = os.getenv("OLLAMA_HOST", "http://localhost:11434").rstrip("/") +OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen2.5:1.5b") +OLLAMA_TIMEOUT = int(os.getenv("OLLAMA_TIMEOUT", "300")) +WHISPER_HOST = os.getenv("WHISPER_HOST", "http://localhost:9000").rstrip("/") + +# --- Celery / Redis ------------------------------------------------------- +CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0") +CELERY_RESULT_BACKEND = os.getenv("CELERY_RESULT_BACKEND", "redis://localhost:6379/0") + +# --- CORS ----------------------------------------------------------------- +_DEFAULT_ORIGINS = "http://127.0.0.1:5173,http://localhost:5173" +ALLOWED_ORIGINS = [ + origin.strip() + for origin in os.getenv("FRONTEND_ORIGINS", _DEFAULT_ORIGINS).split(",") + if origin.strip() +] + +# --- Error handling ------------------------------------------------------- +# Advisory Retry-After sent to clients on 503 responses. This is a client +# hint, not a measured backpressure value — the app has no queue-depth signal. +RETRY_AFTER_SECONDS = 30 + +# Polling hint returned by GET /input/{id} when a voice input is still queued +# or transcribing. Value matches the contract example (contracts/path/input.yaml). +INPUT_POLL_INTERVAL_SECONDS = 5 + +# --- API Versioning ------------------------------------------------------- +API_PREFIX = "/api/v1" + +# --- Security & Access Control --------------------------------------------- +FIREFORM_API_KEY = os.getenv("FIREFORM_API_KEY", "") + +# --- Data Retention -------------------------------------------------------- +RETENTION_PERIOD_DAYS = int(os.getenv("RETENTION_PERIOD_DAYS", "30")) + +# --- Audio storage -------------------------------------------------------- +# Voice input audio files land here: {AUDIO_DIR}/{input_id}.{ext} +AUDIO_DIR = DATA_DIR / "audio" + +# Advisory estimate returned in VoiceInputResponse.estimated_processing_seconds. +ESTIMATED_TRANSCRIPTION_SECONDS = int(os.getenv("ESTIMATED_TRANSCRIPTION_SECONDS", "30")) + +# Canonical audio format mapping — single source of truth for both the route +# (membership check, 415 detail list) and the task (content-type lookup). +# Dict insertion order gives the stable list shown in error responses. +AUDIO_CONTENT_TYPES: dict[str, str] = { + "wav": "audio/wav", + "mp3": "audio/mpeg", + "m4a": "audio/m4a", + "ogg": "audio/ogg", + "webm": "audio/webm", +} +ALLOWED_AUDIO_EXTENSIONS: frozenset[str] = frozenset(AUDIO_CONTENT_TYPES) \ No newline at end of file diff --git a/api/schemas/__init__.py b/app/core/errors/__init__.py similarity index 100% rename from api/schemas/__init__.py rename to app/core/errors/__init__.py diff --git a/app/core/errors/base.py b/app/core/errors/base.py new file mode 100644 index 00000000..d317736a --- /dev/null +++ b/app/core/errors/base.py @@ -0,0 +1,23 @@ +def _default_error_code(status_code: int) -> str: + return { + 400: "BAD_REQUEST", + 404: "NOT_FOUND", + 422: "VALIDATION_ERROR", + 500: "INTERNAL_ERROR", + 502: "BAD_GATEWAY", + 503: "SERVICE_UNAVAILABLE", + }.get(status_code, "ERROR") + + +class AppError(Exception): + def __init__( + self, + message: str, + status_code: int = 400, + error_code: str | None = None, + detail: dict | None = None, + ): + self.message = message + self.status_code = status_code + self.error_code = error_code or _default_error_code(status_code) + self.detail = detail diff --git a/app/core/errors/handlers.py b/app/core/errors/handlers.py new file mode 100644 index 00000000..ce8988ad --- /dev/null +++ b/app/core/errors/handlers.py @@ -0,0 +1,37 @@ +from fastapi import Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse + +from app.core.config import RETRY_AFTER_SECONDS +from app.core.errors.base import AppError + + +def register_exception_handlers(app): + @app.exception_handler(AppError) + async def app_error_handler(request: Request, exc: AppError): + body: dict = {"error_code": exc.error_code, "message": exc.message} + if exc.detail is not None: + body["detail"] = exc.detail + if exc.status_code == 503: + body["retry_after_seconds"] = RETRY_AFTER_SECONDS + return JSONResponse(status_code=exc.status_code, content=body) + + @app.exception_handler(RequestValidationError) + async def validation_error_handler(request: Request, exc: RequestValidationError): + validation_errors = [] + for error in exc.errors(): + loc = error.get("loc", ()) + field = ".".join(str(x) for x in loc if x != "body") + validation_errors.append({ + "field": field or None, + "issue": error.get("msg"), + "value": error.get("input"), + }) + return JSONResponse( + status_code=422, + content={ + "error_code": "VALIDATION_ERROR", + "message": "Request validation failed", + "validation_errors": validation_errors, + }, + ) diff --git a/app/core/lifespan.py b/app/core/lifespan.py new file mode 100644 index 00000000..7f842739 --- /dev/null +++ b/app/core/lifespan.py @@ -0,0 +1,17 @@ +"""Application lifespan: startup and shutdown hooks.""" + +import logging +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from app.db.init_db import init_db + +logger = logging.getLogger(__name__) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + logger.info("Initializing database...") + init_db() + yield diff --git a/app/core/logging.py b/app/core/logging.py new file mode 100644 index 00000000..48d28a8a --- /dev/null +++ b/app/core/logging.py @@ -0,0 +1,14 @@ +"""Logging setup. Call setup_logging() once at app startup.""" + +import logging + + +def setup_logging(level: str = "INFO") -> None: + logging.basicConfig( + level=level, + format="%(asctime)s %(levelname)-8s %(name)s: %(message)s", + ) + + +def get_logger(name: str) -> logging.Logger: + return logging.getLogger(name) diff --git a/app/core/paths.py b/app/core/paths.py new file mode 100644 index 00000000..ddc51412 --- /dev/null +++ b/app/core/paths.py @@ -0,0 +1,41 @@ +from pathlib import Path + +from fastapi import HTTPException + +from app.core.config import BASE_DIR, DEFAULT_TEMPLATE_DIR + +PROJECT_ROOT = BASE_DIR + + +def _resolve_target_directory(directory: str) -> Path: + dir_value = (directory or DEFAULT_TEMPLATE_DIR).strip() + if not dir_value: + raise HTTPException(status_code=400, detail="Directory is required.") + + candidate = Path(dir_value) + if not candidate.is_absolute(): + candidate = (PROJECT_ROOT / candidate).resolve() + else: + candidate = candidate.resolve() + + if candidate != PROJECT_ROOT and PROJECT_ROOT not in candidate.parents: + raise HTTPException(status_code=400, detail="Directory must be inside the project.") + + return candidate + + +def _resolve_project_file(file_path: str) -> Path: + raw_path = (file_path or "").strip() + if not raw_path: + raise HTTPException(status_code=400, detail="Path is required.") + + candidate = Path(raw_path) + if not candidate.is_absolute(): + candidate = (PROJECT_ROOT / candidate).resolve() + else: + candidate = candidate.resolve() + + if candidate != PROJECT_ROOT and PROJECT_ROOT not in candidate.parents: + raise HTTPException(status_code=400, detail="Path must be inside the project.") + + return candidate diff --git a/app/core/pdf_utils.py b/app/core/pdf_utils.py new file mode 100644 index 00000000..bd6ae203 --- /dev/null +++ b/app/core/pdf_utils.py @@ -0,0 +1,54 @@ +import re +from pathlib import Path + +from app.core import paths + +# PDF field-type codes -> the type values the frontend field builder uses. +_FIELD_TYPE_BY_FT = {"/Tx": "string", "/Btn": "checkbox", "/Ch": "list", "/Sig": "signature"} + + +def _pdf_text(value) -> str: + """Decode a pdfrw string (field name / tooltip) to plain text.""" + if value is None: + return "" + if hasattr(value, "to_unicode"): + return value.to_unicode().strip() + return str(value).strip() + + +def _humanize(name: str) -> str: + """Turn a raw field name into a readable description (JobTitle -> Job Title).""" + text = re.sub(r"_+", " ", name) + text = re.sub(r"(?<=[a-z])(?=[A-Z])", " ", text) + return re.sub(r"\s+", " ", text).strip() + + +def _extract_pdf_fields(pdf_path: str) -> list[dict] | None: + """Fillable widgets in page order (top-to-bottom, left-to-right per page). + Returns None if the PDF can't be read.""" + try: + from pdfrw import PdfReader + candidate = Path(pdf_path) + if not candidate.is_absolute(): + candidate = (paths.PROJECT_ROOT / candidate).resolve() + pdf = PdfReader(str(candidate)) + fields: list[dict] = [] + for page in pdf.pages: + widgets = [a for a in (page.Annots or []) if a.Subtype == "/Widget" and a.T] + widgets.sort(key=lambda a: (-float(a.Rect[1]), float(a.Rect[0]))) + for annot in widgets: + name = _pdf_text(annot.T) + fields.append({ + "name": name, + "description": _pdf_text(annot.TU) or _humanize(name), + "type": _FIELD_TYPE_BY_FT.get(str(annot.FT), "string"), + }) + return fields + except Exception: + return None + + +def _count_pdf_widgets(pdf_path: str) -> int | None: + """Number of fillable widgets in a PDF, or None if unreadable.""" + fields = _extract_pdf_fields(pdf_path) + return None if fields is None else len(fields) diff --git a/src/__init__.py b/app/db/__init__.py similarity index 100% rename from src/__init__.py rename to app/db/__init__.py diff --git a/app/db/database.py b/app/db/database.py new file mode 100644 index 00000000..08f95293 --- /dev/null +++ b/app/db/database.py @@ -0,0 +1,10 @@ +from sqlmodel import Session, create_engine + +from app.core.config import DATABASE_URL, DB_ECHO + +engine = create_engine(DATABASE_URL, echo=DB_ECHO) + + +def get_session(): + with Session(engine) as session: + yield session diff --git a/app/db/init_db.py b/app/db/init_db.py new file mode 100644 index 00000000..34dafe6b --- /dev/null +++ b/app/db/init_db.py @@ -0,0 +1,67 @@ +import datetime +import logging +from pathlib import Path + +from alembic.config import Config +from sqlmodel import Session, select + +from alembic import command +from app.core.config import DEFAULT_TEMPLATE_DIR +from app.db.database import engine +from app.models import FormSubmission, Template # noqa: F401 + +logger = logging.getLogger(__name__) + +ALEMBIC_INI = str(Path(__file__).resolve().parents[2] / "alembic.ini") + + +def _alembic_cfg() -> Config: + cfg = Config(ALEMBIC_INI) + return cfg + + +def run_migrations(): + logger.info("Running Alembic migrations...") + command.upgrade(_alembic_cfg(), "head") + logger.info("Migrations complete.") + + +def seed_db(): + with Session(engine) as session: + statement = select(Template) + try: + results = session.exec(statement).first() + except Exception: + results = None + + if not results: + logger.info("Seeding database with default template...") + fields = { + "Employee's name": "string", + "Employee's job title": "string", + "Employee's department supervisor": "string", + "Employee's phone number": "string", + "Employee's email": "string", + "Signature": "string", + "Date": "string", + } + + default_template = Template( + id=2, + name="Manual Test Template", + fields=fields, + pdf_path=f"{DEFAULT_TEMPLATE_DIR}/file_template_manual.pdf", + created_at=datetime.datetime.now(datetime.timezone.utc), + ) + session.add(default_template) + session.commit() + logger.info("Database seeded successfully.") + + +def init_db(): + run_migrations() + seed_db() + + +if __name__ == "__main__": + init_db() diff --git a/app/db/repositories.py b/app/db/repositories.py new file mode 100644 index 00000000..bf490e87 --- /dev/null +++ b/app/db/repositories.py @@ -0,0 +1,123 @@ +from datetime import datetime +from uuid import UUID + +from sqlmodel import Session, select + +from app.models import FormSubmission, Input, Job, Template + + +# Templates +def create_template(session: Session, template: Template) -> Template: + session.add(template) + session.commit() + session.refresh(template) + return template + +def get_template(session: Session, template_id: int) -> Template | None: + return session.get(Template, template_id) + + +def list_templates(session: Session) -> list[Template]: + statement = select(Template).order_by(Template.created_at.desc(), Template.id.desc()) + return list(session.exec(statement)) + +# Forms +def create_form(session: Session, form: FormSubmission) -> FormSubmission: + session.add(form) + session.commit() + session.refresh(form) + return form + + +def get_submissions(session: Session) -> list[tuple[FormSubmission, str | None]]: + statement = ( + select(FormSubmission, Template.name) + .join(Template, FormSubmission.template_id == Template.id, isouter=True) + .order_by(FormSubmission.created_at.desc(), FormSubmission.id.desc()) + ) + return list(session.exec(statement).all()) + + +def get_submissions_with_template(session: Session) -> list[tuple[FormSubmission, str | None]]: + statement = select(FormSubmission, Template.name).join( + Template, FormSubmission.template_id == Template.id, isouter=True + ) + return list(session.exec(statement).all()) + + +def get_submissions_before(session: Session, cutoff: datetime) -> list[FormSubmission]: + statement = select(FormSubmission).where(FormSubmission.created_at < cutoff) + return list(session.exec(statement)) + + +# Jobs +def create_job(session: Session, job: Job) -> Job: + session.add(job) + session.commit() + session.refresh(job) + return job + + +def get_job(session: Session, job_id: int) -> Job | None: + return session.get(Job, job_id) + + +def get_job_by_uuid(session: Session, job_uuid: str) -> Job | None: + statement = select(Job).where(Job.job_id == job_uuid) + return session.exec(statement).first() + + +def get_job_by_celery_id(session: Session, celery_task_id: str) -> Job | None: + statement = select(Job).where(Job.celery_task_id == celery_task_id) + return session.exec(statement).first() + + +def update_job(session: Session, job: Job) -> Job: + session.add(job) + session.commit() + session.refresh(job) + return job + + +def delete_template(session: Session, template: Template) -> None: + session.delete(template) + session.commit() + + +def get_submissions_by_template(session: Session, template_id: int) -> list[FormSubmission]: + statement = select(FormSubmission).where(FormSubmission.template_id == template_id) + return list(session.exec(statement)) + + +def get_jobs_by_template(session: Session, template_id: int) -> list[Job]: + statement = select(Job).where(Job.template_id == template_id) + return list(session.exec(statement)) + + +def get_form_submission(session: Session, submission_id: int) -> FormSubmission | None: + return session.get(FormSubmission, submission_id) + + +def delete_form_submission(session: Session, submission: FormSubmission) -> None: + session.delete(submission) + session.commit() + + +# Inputs +def create_input(session: Session, input_obj: Input) -> Input: + session.add(input_obj) + session.commit() + session.refresh(input_obj) + return input_obj + + +def get_input(session: Session, input_id: UUID) -> Input | None: + return session.get(Input, input_id) + + +def update_input(session: Session, input_obj: Input) -> Input: + session.add(input_obj) + session.commit() + session.refresh(input_obj) + return input_obj + diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..3d3bdad8 --- /dev/null +++ b/app/main.py @@ -0,0 +1,34 @@ +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware + +from app.api.router import api_router +from app.core import config +from app.core.errors.handlers import register_exception_handlers +from app.core.lifespan import lifespan +from app.core.logging import setup_logging + + +def create_app() -> FastAPI: + setup_logging() + + app = FastAPI( + title=config.APP_TITLE, + version=config.APP_VERSION, + lifespan=lifespan, + ) + + app.add_middleware( + CORSMiddleware, + allow_origins=config.ALLOWED_ORIGINS, + allow_credentials=False, + allow_methods=["*"], + allow_headers=["*"], + ) + + register_exception_handlers(app) + app.include_router(api_router) + + return app + + +app = create_app() diff --git a/app/models/__init__.py b/app/models/__init__.py new file mode 100644 index 00000000..9e736ec7 --- /dev/null +++ b/app/models/__init__.py @@ -0,0 +1,23 @@ +"""ORM models. Import from here: `from app.models import Template`.""" + +from app.models.models import ( + Extraction, + Form, + FormSubmission, + Incident, + Input, + Job, + Report, + Template, +) + +__all__ = [ + "Extraction", + "Form", + "FormSubmission", + "Incident", + "Input", + "Job", + "Report", + "Template", +] diff --git a/app/models/models.py b/app/models/models.py new file mode 100644 index 00000000..0b63a0c1 --- /dev/null +++ b/app/models/models.py @@ -0,0 +1,168 @@ +import uuid as uuid_mod +from datetime import date, datetime, timezone +from uuid import UUID, uuid4 + +from sqlalchemy import JSON, Column +from sqlmodel import Field, SQLModel +from sqlmodel.sql.sqltypes import AutoString + +from app.api.schemas.enums import ( + ExtractionStatus, + FormStatus, + FormType, + InputStatus, + InputType, + JobStatus, + OutputFormat, + PeriodType, + ReportStatus, +) + + +class Template(SQLModel, table=True): + id: int | None = Field(default=None, primary_key=True) + name: str + fields: dict = Field(sa_column=Column(JSON, nullable=False)) + pdf_path: str + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + +class FormSubmission(SQLModel, table=True): + id: int | None = Field(default=None, primary_key=True) + template_id: int = Field(foreign_key="template.id") + input_id: UUID | None = Field(default=None, foreign_key="inputs.input_id") + input_text: str + output_pdf_path: str + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + +class Job(SQLModel, table=True): + id: int | None = Field(default=None, primary_key=True) + job_id: str = Field(default_factory=lambda: str(uuid_mod.uuid4()), index=True, unique=True) + celery_task_id: str = Field(index=True) + job_type: str = Field(default="form_generation") + template_id: int | None = Field(default=None, foreign_key="template.id") + input_text: str | None = None + status: str = Field(default="queued") + progress_percent: int = Field(default=0) + result_url: str | None = None + error: dict | None = Field(default=None, sa_column=Column(JSON)) + model: str | None = None + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + +# --------------------------------------------------------------------------- +# v1 contract models +# --------------------------------------------------------------------------- + +class Input(SQLModel, table=True): + __tablename__ = "inputs" + + input_id: UUID = Field(default_factory=uuid4, primary_key=True) + # sa_column required on all str-Enum fields: without it SQLModel emits + # sa.Enum(native_enum=True) which creates a Postgres ENUM type — hard to + # migrate and inconsistent with the VARCHAR approach used in migration 001. + input_type: InputType = Field(sa_column=Column(AutoString, nullable=False)) + status: InputStatus = Field( + default=InputStatus.queued, sa_column=Column(AutoString, nullable=False) + ) + transcript: str | None = None + original_filename: str | None = None + audio_duration_seconds: float | None = None + character_count: int | None = None + word_count: int | None = None + station_id: str | None = None + responder_badge: str | None = None + incident_date_hint: date | None = None + error_detail: str | None = None + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + +class Extraction(SQLModel, table=True): + __tablename__ = "extractions" + + extract_id: UUID = Field(default_factory=uuid4, primary_key=True) + input_id: UUID = Field(foreign_key="inputs.input_id") + status: ExtractionStatus = Field( + default=ExtractionStatus.processing, sa_column=Column(AutoString, nullable=False) + ) + started_at: datetime | None = None + completed_at: datetime | None = None + model_used: str | None = None + processing_time_seconds: float | None = None + # Full IncidentContract superset blob; stores partial result while processing, + # final canonical JSON when status=completed. + incident_contract: dict | None = Field(default=None, sa_column=Column(JSON)) + # Audit trail of manual corrections applied via PATCH /extract/{id}. + corrections: list | None = Field(default=None, sa_column=Column(JSON)) + error_type: str | None = None + error_detail: str | None = None + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + +class Incident(SQLModel, table=True): + __tablename__ = "incidents" + + incident_id: UUID = Field(default_factory=uuid4, primary_key=True) + extract_id: UUID = Field(foreign_key="extractions.extract_id") + incident_number: str | None = None + status: ReportStatus = Field( + default=ReportStatus.draft, sa_column=Column(AutoString, nullable=False) + ) + incident_name: str | None = None + incident_type: str | None = None + incident_date: date | None = None + tags: list | None = Field(default=None, sa_column=Column(JSON)) + notes: str | None = None + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + deleted_at: datetime | None = None + + +class Form(SQLModel, table=True): + __tablename__ = "forms" + + form_id: UUID = Field(default_factory=uuid4, primary_key=True) + form_type: FormType = Field(sa_column=Column(AutoString, nullable=False)) + status: FormStatus = Field( + default=FormStatus.queued, sa_column=Column(AutoString, nullable=False) + ) + extract_id: UUID = Field(foreign_key="extractions.extract_id") + incident_id: UUID | None = Field(default=None, foreign_key="incidents.incident_id") + # Plain UUID, no FK constraint — pending contract Job model resolution (#544 decision A). + job_id: UUID | None = None + completed_at: datetime | None = None + pdf_ready: bool = Field(default=False) + json_ready: bool = Field(default=False) + field_mapping_summary: dict | None = Field(default=None, sa_column=Column(JSON)) + pdf_path: str | None = None + json_data: dict | None = Field(default=None, sa_column=Column(JSON)) + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + +class Report(SQLModel, table=True): + __tablename__ = "reports" + + report_id: UUID = Field(default_factory=uuid4, primary_key=True) + period_type: PeriodType = Field(sa_column=Column(AutoString, nullable=False)) + period_label: str | None = None + year: int + month: int | None = None + quarter: int | None = None + # Named output_format (not format) to avoid shadowing the Python builtin. + output_format: OutputFormat = Field(sa_column=Column(AutoString, nullable=False)) + status: JobStatus = Field( + default=JobStatus.queued, sa_column=Column(AutoString, nullable=False) + ) + generated_at: datetime | None = None + summary: dict | None = Field(default=None, sa_column=Column(JSON)) + # Plain UUID, no FK constraint — pending contract Job model resolution (#544 decision A). + job_id: UUID | None = None + pdf_path: str | None = None + json_data: dict | None = Field(default=None, sa_column=Column(JSON)) + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) \ No newline at end of file diff --git a/app/services/__init__.py b/app/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/services/external_apis/weather_api.py b/app/services/external_apis/weather_api.py new file mode 100644 index 00000000..8f0e0ab2 --- /dev/null +++ b/app/services/external_apis/weather_api.py @@ -0,0 +1,83 @@ +import openmeteo_requests +import pandas as pd +import requests_cache +from retry_requests import retry + +# All supported hourly fields exposed by this service +ALL_HOURLY_FIELDS = [ + "temperature_2m", + "relative_humidity_2m", + "rain", + "apparent_temperature", + "precipitation_probability", + "precipitation", + "wind_direction_10m", + "wind_speed_10m", + "soil_temperature_0cm", + "uv_index", +] + +class WeatherAPI: + def __init__(self): + pass + + def get_weather(self, latitude: float, longitude: float, start_date: str, end_date: str, hourly_fields: list[str] | None = None): + """ + Get weather data for a given latitude and longitude. + + Args: + latitude (float): The latitude of the location. + longitude (float): The longitude of the location. + hourly_fields (list[str] | None): Subset of hourly variables to request. + Defaults to ALL_HOURLY_FIELDS when None or empty. + + Returns: + dict: A dictionary containing the weather data. + """ + requested = [f for f in (hourly_fields or []) if f in ALL_HOURLY_FIELDS] + if not requested: + requested = list(ALL_HOURLY_FIELDS) + + # Setup the Open-Meteo API client with cache and retry on error + cache_session = requests_cache.CachedSession('.cache', expire_after=3600) + retry_session = retry(cache_session, retries=5, backoff_factor=0.2) + openmeteo = openmeteo_requests.Client(session=retry_session) + + url = "https://api.open-meteo.com/v1/forecast" + params = { + "latitude": latitude, + "longitude": longitude, + "hourly": requested, + "start_date": start_date, + "end_date": end_date, + } + responses = openmeteo.weather_api(url, params=params) + + response = responses[0] + + hourly = response.Hourly() + + hourly_data: dict = { + "date": pd.date_range( + start=pd.to_datetime(hourly.Time(), unit="s", utc=True), + end=pd.to_datetime(hourly.TimeEnd(), unit="s", utc=True), + freq=pd.Timedelta(seconds=hourly.Interval()), + inclusive="left", + ) + } + + for i, field in enumerate(requested): + hourly_data[field] = hourly.Variables(i).ValuesAsNumpy().tolist() + + # Convert pandas Timestamp objects to ISO strings for JSON serialization + hourly_data["date"] = [d.isoformat() for d in hourly_data["date"]] + + return { + "latitude": response.Latitude(), + "longitude": response.Longitude(), + "elevation": response.Elevation(), + "timezone": response.Timezone(), + "timezone_abbreviation": response.TimezoneAbbreviation(), + "utc_offset_seconds": response.UtcOffsetSeconds(), + "hourly": hourly_data, + } \ No newline at end of file diff --git a/app/services/external_apis/zipcode_api.py b/app/services/external_apis/zipcode_api.py new file mode 100644 index 00000000..094b6d5d --- /dev/null +++ b/app/services/external_apis/zipcode_api.py @@ -0,0 +1,72 @@ +from geopy.exc import GeocoderServiceError, GeocoderTimedOut +from geopy.geocoders import Nominatim + +from app.core.logging import get_logger + +logger = get_logger(__name__) + +_GEOLOCATOR = Nominatim(user_agent="fireform", timeout=10) + + +class ZipCodeAPI: + def __init__(self): + pass + + def lookup_address(self, address: str) -> list[dict]: + """ + Look up an address string via OpenStreetMap Nominatim and return a list + of matching locations with their postal codes, coordinates, and names. + + Args: + address: Full address string, e.g. "1600 Amphitheatre Parkway, Mountain View, CA, US" + + Returns: + A list of dicts, each containing: + - postal_code (str | None) + - place_name (str) + - latitude (float) + - longitude (float) + - display_name (str) + - country (str | None) + - state (str | None) + - county (str | None) + """ + try: + locations = _GEOLOCATOR.geocode( + address, + exactly_one=False, + addressdetails=True, + limit=10, + ) + except GeocoderTimedOut: + logger.error("Nominatim geocoding timed out for address: %s", address) + raise TimeoutError(f"Geocoding service timed out for address: '{address}'") + except GeocoderServiceError as exc: + logger.error("Nominatim geocoding service error: %s", exc) + raise RuntimeError(f"Geocoding service error: {exc}") + + if not locations: + logger.info("No results found for address: %s", address) + return [] + + results = [] + for loc in locations: + addr = loc.raw.get("address", {}) + results.append({ + "postal_code": addr.get("postcode"), + "place_name": addr.get("city") + or addr.get("town") + or addr.get("village") + or addr.get("hamlet") + or addr.get("municipality") + or "", + "latitude": loc.latitude, + "longitude": loc.longitude, + "display_name": loc.address, + "country": addr.get("country"), + "state": addr.get("state"), + "county": addr.get("county"), + }) + logger.debug("Geocoded result: %s", results[-1]) + + return results \ No newline at end of file diff --git a/app/services/external_apis_coordinator.py b/app/services/external_apis_coordinator.py new file mode 100644 index 00000000..1e7654f1 --- /dev/null +++ b/app/services/external_apis_coordinator.py @@ -0,0 +1,14 @@ +from app.services.external_apis.weather_api import WeatherAPI +from app.services.external_apis.zipcode_api import ZipCodeAPI + + +class ExternalAPIsCoordinator: + def __init__(self): + self.weather_api = WeatherAPI() + self.zipcode_api = ZipCodeAPI() + + def get_weather(self, latitude: float, longitude: float, start_date: str, end_date: str, hourly_fields: list[str] | None = None): + return self.weather_api.get_weather(latitude, longitude, start_date, end_date, hourly_fields) + + def lookup_address(self, address: str) -> list[dict]: + return self.zipcode_api.lookup_address(address) \ No newline at end of file diff --git a/app/services/form.py b/app/services/form.py new file mode 100644 index 00000000..bd5a7a79 --- /dev/null +++ b/app/services/form.py @@ -0,0 +1,165 @@ +import re +from collections import Counter +from datetime import datetime, timedelta, timezone +from uuid import UUID + +from sqlmodel import Session + +from app.core import paths +from app.core.errors.base import AppError +from app.db.repositories import ( + create_form, + delete_form_submission, + get_submissions, + get_submissions_before, + get_submissions_with_template, +) +from app.models import FormSubmission, Template +from app.services.form_filler import filler +from app.services.input import InputService + +_STOPWORDS = { + "the", "and", "a", "of", "to", "in", "is", "that", "it", "was", "for", "on", + "as", "with", "by", "at", "an", "be", "this", "are", "from", "or", "have", + "has", "had", "but", "not", "he", "she", "they", "we", "i", "you", "my", "his", + "her", "their", "our", "me", "him", "them", "us", "about", "there", "their", + "were", "been", "would", "could", "should", "will", "can", "no", "yes", "any", + "so", "very", "patient", "presents", "with", "reported", "history", "shows", + "left", "right", "pain", "due", "after", "before", "emergency", "department", + "medical", "clinical" +} + + +class FormService: + def __init__(self): + self.input_service = InputService() + + def fill_form( + self, + session: Session, + template: Template, + input_id: UUID | None = None, + input_text: str | None = None, + model: str | None = None, + ) -> FormSubmission: + if input_id is not None and not input_text: + transcript = self.input_service.resolve_transcript(session, input_id) + elif input_text: + transcript = input_text + if input_id is None: + from app.api.schemas.enums import InputStatus, InputType + from app.models import Input + input_record = Input( + input_type=InputType.text, + status=InputStatus.ready, + transcript=input_text, + character_count=len(input_text), + word_count=len(input_text.split()), + ) + session.add(input_record) + session.commit() + session.refresh(input_record) + input_id = input_record.input_id + else: + raise AppError("Either input_id or input_text is required", status_code=422, error_code="VALIDATION_ERROR") + return self.fill_and_persist(session, template, transcript, input_id, model) + + def fill_and_persist( + self, + session: Session, + template: Template, + transcript: str, + input_id: UUID, + model: str | None = None, + ) -> FormSubmission: + resolved_pdf = paths._resolve_project_file(template.pdf_path) + if not resolved_pdf.exists(): + raise FileNotFoundError(f"PDF template not found at {template.pdf_path}") + + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + out_name = f"{resolved_pdf.stem}_{timestamp}_filled.pdf" + out_target = resolved_pdf.parent / out_name + + filler.fill( + pdf_path=str(resolved_pdf), + narrative=transcript, + out_path=str(out_target), + model=model, + ) + + relative_out = out_target.relative_to(paths.PROJECT_ROOT).as_posix() + + submission = FormSubmission( + template_id=template.id, + input_id=input_id, + input_text=transcript, + output_pdf_path=relative_out, + ) + return create_form(session, submission) + + + def list_submissions(self, session: Session) -> list[dict]: + results = get_submissions(session) + return [ + { + "id": sub.id, + "template_id": sub.template_id, + "template_name": name or "Unknown Template", + "input_text": sub.input_text, + "output_pdf_path": sub.output_pdf_path, + "created_at": sub.created_at.isoformat() if sub.created_at else None, + } + for sub, name in results + ] + + def get_analytics(self, session: Session) -> dict: + results = get_submissions_with_template(session) + + total_submissions = len(results) + + template_counts = Counter() + daily_counts = Counter() + words = [] + + for sub, name in results: + template_name = name or "Unknown Template" + template_counts[template_name] += 1 + + if sub.created_at: + date_str = sub.created_at.strftime("%Y-%m-%d") + daily_counts[date_str] += 1 + + if sub.input_text: + found_words = re.findall(r"\b[a-zA-Z]{3,15}\b", sub.input_text.lower()) + for w in found_words: + if w not in _STOPWORDS: + words.append(w) + + sorted_daily = [{"date": k, "count": v} for k, v in sorted(daily_counts.items())] + sorted_templates = [{"template_name": k, "count": v} for k, v in template_counts.most_common()] + common_terms = [{"word": k, "count": v} for k, v in Counter(words).most_common(12)] + + return { + "total_submissions": total_submissions, + "by_template": sorted_templates, + "by_date": sorted_daily, + "common_terms": common_terms, + } + + def purge_submissions(self, session: Session, retention_days: int) -> int: + cutoff_date = datetime.now(timezone.utc) - timedelta(days=retention_days) + submissions = get_submissions_before(session, cutoff_date) + + purged_count = 0 + for sub in submissions: + if sub.output_pdf_path: + try: + resolved_out = paths._resolve_project_file(sub.output_pdf_path) + if resolved_out.exists() and resolved_out.is_file(): + resolved_out.unlink() + except Exception: + pass + delete_form_submission(session, sub) + purged_count += 1 + + return purged_count diff --git a/app/services/form_filler/__init__.py b/app/services/form_filler/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/services/form_filler/filler.py b/app/services/form_filler/filler.py new file mode 100644 index 00000000..75b88e88 --- /dev/null +++ b/app/services/form_filler/filler.py @@ -0,0 +1,123 @@ +"""Narrative + fillable PDF -> filled PDF. + + result = filler.fill("ics_201.pdf", narrative, "ics_201_filled.pdf") + +Ollama host, model and timeout come from app.core.config (OLLAMA_HOST, +OLLAMA_MODEL, OLLAMA_TIMEOUT environment variables). + +fill() builds the schema (template.py), sends prompt.txt + narrative + schema +to Ollama, maps the answer back to widget names and writes the filled PDF. +It returns every intermediate step so callers (the benchmark) can save them. +""" + +import json +import os + +import requests +from pypdf import PdfWriter + +from app.core.config import OLLAMA_HOST, OLLAMA_MODEL, OLLAMA_TIMEOUT + +from . import template +from .geometry import CHECKBOX, RADIOBUTTON, SIGNATURE, drop_null_parents, on_state, page_widgets + +OPTIONS = { + "num_ctx": 32768, + # Backstop for runaway generation; the largest real output (ICS 209) is ~8k tokens + "num_predict": 12288, + "temperature": 0, + "seed": 42, + "repeat_penalty": 1.0, + "repeat_last_n": 0, +} + +with open(os.path.join(os.path.dirname(__file__), "prompt.txt"), "r") as f: + PROMPT = f.read() + + +def fill(pdf_path, narrative, out_path=None, model=None): + """Fill pdf_path from the narrative. Writes the PDF when out_path is given. + + Returns {schema, tables, groups, prompt, response_text, values}: the + schema sent as Ollama's format, the exact prompt, the raw model output, + and the flat {widget name: value} dict used to fill the PDF. + """ + schema, tables, groups = template.create_template(pdf_path) + strip_empty_descriptions(schema["properties"]) + + prompt = build_prompt(narrative, schema) + response_text = call_model(prompt, schema, model or OLLAMA_MODEL) + + # Tables come back as arrays of rows; map them to widget names + values = template.expand_output(json.loads(response_text), tables, groups) + + if out_path: + write_pdf(pdf_path, values, out_path) + + return { + "schema": schema, + "tables": tables, + "groups": groups, + "prompt": prompt, + "response_text": response_text, + "values": values, + } + + +def strip_empty_descriptions(props): + for prop in props.values(): + if prop.get("description") == "": + del prop["description"] + if "items" in prop: + strip_empty_descriptions(prop["items"]["properties"]) + return props + + +def build_prompt(narrative, schema): + user_content = ( + "\n" + + narrative + + "\n\n\n\n" + + json.dumps(schema["properties"], indent=2, ensure_ascii=False) + + "\n" + ) + return PROMPT.rstrip() + "\n\n" + user_content + + +def call_model(prompt, schema, model): + """Ollama with the schema as `format`, so the answer always parses into it.""" + payload = { + "model": model, + "prompt": prompt, + "format": schema, + "stream": False, + "think": False, + "options": OPTIONS, + } + response = requests.post( + OLLAMA_HOST + "/api/generate", + json=payload, + timeout=OLLAMA_TIMEOUT, + ) + response.raise_for_status() + return response.json().get("response", "") + + +def write_pdf(pdf_path, values, out_path): + new_doc = PdfWriter(clone_from=pdf_path) + + # pypdf sets checkboxes by appearance-state name ("/Yes"), not "Yes" + for page in new_doc.pages: + drop_null_parents(page) + page_values = {} + for widget in page_widgets(page): + if widget.field_type == SIGNATURE: + continue + value = values.get(widget.field_name, "") + if widget.field_type in (CHECKBOX, RADIOBUTTON): + value = on_state(widget) if value == "Yes" else "/Off" + page_values[widget.field_name] = "" if value is None else str(value) + if page_values: + new_doc.update_page_form_field_values(page, page_values) + + new_doc.write(out_path) diff --git a/app/services/form_filler/geometry.py b/app/services/form_filler/geometry.py new file mode 100644 index 00000000..0a2301b7 --- /dev/null +++ b/app/services/form_filler/geometry.py @@ -0,0 +1,744 @@ +"""Read a fillable PDF: its widgets (pypdf) and the grid it is printed on (pdfplumber). + +A blank government form is a drawn grid, so the row and the column of a +fillable box are a property of the paper, not of the widget's name. extract() +reads the ruling lines, decides which runs of grid rows are logical tables, +and returns every widget with where it sits and what text names it. + + widget = {page, rect, field_name, field_type, tooltip, + label_left, label_above, label_inside, label_right, + block, # "10. Resource Summary" + table_id, table_caption, column_header, row_index} +""" +import os +import re +from collections import namedtuple + +import pdfplumber +from pypdf import PdfReader +from pypdf.generic import DictionaryObject + +# ------------------------------------------------------------- pypdf widgets +# Each widget annotation as (field_name, field_type, rect, annotation). + +Widget = namedtuple("Widget", ["field_name", "field_type", "rect", "annotation"]) + +# Widget type codes +UNKNOWN, BUTTON, CHECKBOX, COMBOBOX, LISTBOX, RADIOBUTTON, SIGNATURE, TEXT = range(8) + +# /Ff bits (PDF 32000-1, tables 226 and 230) +FF_RADIO = 1 << 15 +FF_PUSHBUTTON = 1 << 16 +FF_COMBO = 1 << 17 + + +def _parent(node): + parent = node.get("/Parent") + parent = parent.get_object() if parent is not None else None + return parent if isinstance(parent, DictionaryObject) else None + + +def _inherited(annotation, key): + """/FT and /Ff can live on a parent field instead of the widget itself.""" + node = annotation + while node is not None: + if key in node: + return node[key] + node = _parent(node) + return None + + +def _field_name(annotation): + """Fully qualified name: parent /T values joined with '.'.""" + parts = [] + node = annotation + while node is not None: + if "/T" in node: + parts.append(str(node["/T"])) + node = _parent(node) + return ".".join(reversed(parts)) + + +def _field_type(annotation): + ft = _inherited(annotation, "/FT") + flags = int(_inherited(annotation, "/Ff") or 0) + if ft == "/Tx": + return TEXT + if ft == "/Sig": + return SIGNATURE + if ft == "/Ch": + return COMBOBOX if flags & FF_COMBO else LISTBOX + if ft == "/Btn": + if flags & FF_PUSHBUTTON: + return BUTTON + if flags & FF_RADIO: + return RADIOBUTTON + return CHECKBOX + return UNKNOWN + + +def page_widgets(page): + """Form widgets on a pypdf page, in /Annots order.""" + widgets = [] + for annot in page.get("/Annots") or []: + annotation = annot.get_object() + if annotation.get("/Subtype") != "/Widget": + continue + widgets.append(Widget( + _field_name(annotation), + _field_type(annotation), + [float(v) for v in annotation["/Rect"]], + annotation, + )) + return widgets + + +def drop_null_parents(page): + """ics_209 has widgets with /Parent null. A null entry means the same as a + missing one, but pypdf's form filler crashes on it, so remove it.""" + for widget in page_widgets(page): + node = widget.annotation + while node is not None: + if "/Parent" in node and _parent(node) is None: + del node["/Parent"] + node = _parent(node) + + +def on_state(widget): + """Name of the 'checked' appearance state, e.g. '/Yes'.""" + ap = widget.annotation.get("/AP") + states = ap.get_object().get("/N") if ap is not None else None + states = states.get_object() if states is not None else {} + return next((s for s in states if s != "/Off"), "/Yes") + + +# --------------------------------------------------------------- geometry + +class Rect: + """Axis-aligned rectangle. Top-left origin, like pdfplumber.""" + + def __init__(self, x0, y0=None, x1=None, y1=None): + if y0 is None: + x0, y0, x1, y1 = x0 + self.x0, self.y0, self.x1, self.y1 = float(x0), float(y0), float(x1), float(y1) + + @property + def width(self): + return self.x1 - self.x0 + + @property + def height(self): + return self.y1 - self.y0 + + def get_area(self): + return self.width * self.height + + def __iter__(self): + return iter((self.x0, self.y0, self.x1, self.y1)) + + def __repr__(self): + return f"Rect({self.x0}, {self.y0}, {self.x1}, {self.y1})" + + +# Ruling lines only, like the old find_tables(strategy="lines") +TABLE_SETTINGS = {"vertical_strategy": "lines", "horizontal_strategy": "lines"} + +# ------------------------------------------------------------------ primitives + +CAPTION_RE = re.compile(r"^\s*(\d{1,2})\.\s*\S") +JUNK_TOOLTIP_RE = re.compile( + r"^(undefined|check\s*box|checkbox|text\s*\d*|text\s*field|button|field)\s*\d*$", re.I) +ROW_TAIL_RE = re.compile(r"[,\s_]*\(?\s*row\s*[_\s]*\d+\s*\)?\s*$", re.I) +INDEX_TAIL_RE = re.compile(r"[\s_\-.#]*\[?\d+\]?$") + + +def is_signature(w): + return w.field_type == 6 or "signature" in w.field_name.lower() + + +def clean(text, limit=90): + text = re.sub(r"[-]", " ", str(text)) # dingbat checkbox glyphs + text = re.sub(r"\s+", " ", text).strip(" :.-_•*") + return text[:limit] + + +def norm(text): + """Comparable form of a naming signal: no case, no punctuation, no row tail.""" + t = re.sub(r"[-]", " ", str(text or "")).lower() + t = ROW_TAIL_RE.sub("", t) + t = INDEX_TAIL_RE.sub("", t) + return re.sub(r"[^a-z0-9]+", " ", t).strip() + + +def rect_text(words, rect, limit=90): + """Printed words whose centre falls inside rect, in reading order.""" + hits = [w for w in words + if rect.x0 <= (w[0] + w[2]) / 2 <= rect.x1 + and rect.y0 <= (w[1] + w[3]) / 2 <= rect.y1] + hits.sort(key=lambda w: (round(w[1] / 3), w[0])) + return clean(" ".join(h[4] for h in hits), limit) + + +# ------------------------------------------------------------- printed labels +# The text left of, above, inside and right of each widget. The right-hand +# label matters because the only thing that tells a "Yes" checkbox from its +# "No" twin is the word to its right, and that pair is a whole column on ICS 206. + +def _blocked(a, b, others): + lo_x, hi_x = min(a.x0, b.x0), max(a.x1, b.x1) + lo_y, hi_y = min(a.y0, b.y0), max(a.y1, b.y1) + for o in others: + if o.x1 <= lo_x + 1 or o.x0 >= hi_x - 1: + continue + if o.y1 <= lo_y + 1 or o.y0 >= hi_y - 1: + continue + return True + return False + + +def labels_for(rect, words, others): + above, left, right, inside = [], [], [], [] + for wx0, wy0, wx1, wy1, txt in words: + wr = Rect(wx0, wy0, wx1, wy1) + ox = min(wx1, rect.x1) - max(wx0, rect.x0) + oy = min(wy1, rect.y1) - max(wy0, rect.y0) + if ox > 1 and oy > 1 and rect.width > 4: + inside.append((wx0, txt)) + elif ox > 0 and -2 <= rect.y0 - wy1 < 30 and not _blocked(rect, wr, others): + above.append((round(rect.y0 - wy1), wx0, txt)) + elif oy > 1 and wx1 <= rect.x0 + 2 and rect.x0 - wx1 < 200 \ + and not _blocked(rect, wr, others): + left.append((rect.x0 - wx1, txt)) + elif oy > 1 and wx0 >= rect.x1 - 2 and wx0 - rect.x1 < 60 \ + and not _blocked(rect, wr, others): + right.append((wx0 - rect.x1, txt)) + above.sort(key=lambda t: (t[0], t[1])) + left.sort(key=lambda t: -t[0]) + right.sort() + inside.sort() + return { + "label_above": clean(" ".join(t[2] for t in above), 80), + "label_left": clean(" ".join(t[1] for t in left[-14:]), 80), + "label_right": clean(" ".join(t[1] for t in right[:4]), 40), + "label_inside": clean(" ".join(t[1] for t in inside), 60), + } + + +BLOCK_RE = re.compile(r"^\s*(\d{1,2})[.\s]\s*([A-Z][^\n]{2,80})") + + +def caption_cells(grids, lines): + """Grid cells that open a numbered section. Which block a widget belongs to + is then pure geometry -- the lowest caption cell above it that its own x + range touches -- which is what keeps "7. Operations Section" in ICS 203's + right page column from claiming the left column's boxes.""" + out = [] + for g in grids: + for row in g["rows"]: + for cell, _ in [c for r in [row] + row["subs"] for c in r["cells"]]: + head = next((ln for ln in lines + if cell.y0 - 2 <= ln["rect"].y0 <= cell.y1 + and cell.x0 - 2 <= ln["rect"].x0 <= cell.x1 + and BLOCK_RE.match(ln["text"])), None) + if head: + out.append((cell, clean(head["text"], 110))) + return out + + +def block_for(rect, captions, lines): + best = None + for cell, text in captions: + if cell.y0 > rect.y0 + 2 or min(cell.x1, rect.x1) - max(cell.x0, rect.x0) <= 0: + continue + if best is None or cell.y0 > best[0].y0: + best = (cell, text) + if best: + return best[1] + above = [ln for ln in lines if ln["rect"].y0 <= rect.y0 + 2 and BLOCK_RE.match(ln["text"])] + return clean(above[-1]["text"], 110) if above else "" + + +def page_words(page): + """(x0, y0, x1, y1, text) words, as MuPDF's get_text("words") gives them. + + pdfplumber sizes each word by its own font, so a bold "4." sits lower than + the "Name:" beside it and rect_text() reads the row out of order. Words on + one pdfminer text line share that line's top and bottom instead. Rotated + headers ("Arrived", "# of Persons") are printed bottom-to-top. + """ + lines = page.objects.get("textlinehorizontal", []) + out = [] + for w in page.extract_words(char_dir_rotated="btt"): + top, bottom = w["top"], w["bottom"] + cy = (top + bottom) / 2 + line = next((ln for ln in lines + if ln["x0"] - 1 <= w["x0"] and w["x1"] <= ln["x1"] + 1 + and ln["top"] <= cy <= ln["bottom"]), None) if w["upright"] else None + if line: + top, bottom = line["top"], line["bottom"] + out.append((w["x0"], top, w["x1"], bottom, w["text"])) + return out + + +def page_lines(page): + """pdfminer's layout lines (pdfplumber opened with laparams), which split + side-by-side columns the way MuPDF's text lines do.""" + out = [] + for line in page.objects.get("textlinehorizontal", []): + txt = line["text"] + if txt.strip(): + out.append({"rect": Rect(line["x0"], line["top"], line["x1"], line["bottom"]), + "text": txt.strip()}) + out.sort(key=lambda item: (round(item["rect"].y0, 1), item["rect"].x0)) + return out + + +# --------------------------------------------------------------------- the grid + +Y_TOL = 1.5 + + +def page_grids(page): + """Ruling-line grids, each as coarse rows with their nested sub-rows. + + find_tables reports a row for every horizontal rule, so a cell that is + subdivided (the Yes/No pair inside "Paramedics on Site?") shows up as extra + rows nested inside the real one. A row wholly inside another is a sub-row: + it is not a table row, but its cell edges tell us how the parent's columns + are subdivided. + """ + grids = [] + for t in page.find_tables(TABLE_SETTINGS): + rows = [] + for ri, row in enumerate(t.rows): + cells = [(Rect(c), ci) for ci, c in enumerate(row.cells) if c] + if cells: + rows.append({"idx": ri, "y0": row.bbox[1], "y1": row.bbox[3], + "cells": cells, "subs": []}) + if not rows: + continue + coarse = [] + for r in sorted(rows, key=lambda r: (r["y0"], -(r["y1"] - r["y0"]))): + host = next((c for c in coarse + if c["y0"] <= r["y0"] + Y_TOL and r["y1"] <= c["y1"] + Y_TOL + and (c["y1"] - c["y0"]) > (r["y1"] - r["y0"]) + Y_TOL), None) + if host: + host["subs"].append(r) + else: + coarse.append(r) + grids.append({"bbox": Rect(t.bbox), "rows": coarse}) + return grids + + +def page_bands(grid, words): + """[(x0, x1)] page columns. Two-column layouts (ICS 203, 207) put unrelated + blocks side by side; a cut only counts when it is a cell edge on most rows + AND two or more numbered captions start exactly there.""" + rows = grid["rows"] + gx0, gx1 = grid["bbox"].x0, grid["bbox"].x1 + edges = {} + for r in rows: + for x in {round(c.x0) for c, _ in r["cells"]}: + edges[x] = edges.get(x, 0) + 1 + for x, n in sorted(edges.items(), key=lambda kv: -kv[1]): + if n < 0.5 * len(rows) or not (gx0 + 0.3 * (gx1 - gx0) < x < gx0 + 0.7 * (gx1 - gx0)): + continue + caps = 0 + for r in rows: + for c, _ in r["cells"]: + if abs(c.x0 - x) < 2 and CAPTION_RE.match(rect_text(words, c)): + caps += 1 + if caps >= 2: + return [(gx0 - 1, x), (x, gx1 + 1)] + return [(gx0 - 1, gx1 + 1)] + + +def classify(row, band, words, widgets): + """What a grid row is, seen from inside one page column band.""" + cells = [c for c, _ in row["cells"] if band[0] <= (c.x0 + c.x1) / 2 <= band[1]] + if not cells: + return None + cells.sort(key=lambda c: c.x0) + texts = [rect_text(words, c) for c in cells] + wid = [w for w in widgets + if row["y0"] - Y_TOL <= (w["rect"].y0 + w["rect"].y1) / 2 <= row["y1"] + Y_TOL + and band[0] <= (w["rect"].x0 + w["rect"].x1) / 2 <= band[1]] + caption = next((t for t in texts if CAPTION_RE.match(t)), "") + if wid: + kind = "body" + elif sum(1 for t in texts if t) >= 2 and len(cells) >= 2: + kind = "header" + elif caption: + kind = "caption" + elif any(texts): + kind = "label" + else: + kind = "blank" + return {"row": row, "cells": cells, "texts": texts, "widgets": wid, + "caption": caption, "kind": kind, + "label": texts[0] if texts and not CAPTION_RE.match(texts[0]) else ""} + + +def header_bands(rec, words): + """Column bands of a header row, split by anything its sub-rows subdivide. + + ICS 204 prints "Resource Identifier | Leader" on a second rule inside the + "5. Resources Assigned:" cell; ICS 206 prints "Air | Ground" under + "Travel Time". Both are sub-rows, and both are the real column names. + """ + bands = [] + for cell, text in zip(rec["cells"], rec["texts"]): + kids = [c for sub in rec["row"]["subs"] for c, _ in sub["cells"] + if cell.x0 - 1 <= c.x0 and c.x1 <= cell.x1 + 1 and c.width < cell.width - 2] + cuts = sorted({round(c.x0, 1) for c in kids} | {round(c.x1, 1) for c in kids} + | {round(cell.x0, 1), round(cell.x1, 1)}) + parent = "" if CAPTION_RE.match(text) else text + if len(cuts) <= 2: + bands.append({"x0": cell.x0, "x1": cell.x1, "text": parent}) + continue + for a, b in zip(cuts, cuts[1:]): + if b - a < 4: + continue + kid = next((c for c in kids if abs(c.x0 - a) < 1 and abs(c.x1 - b) < 1), None) + sub = rect_text(words, kid) if kid is not None else "" + joined = parent if sub in ("", parent) else f"{parent} {sub}".strip() + bands.append({"x0": a, "x1": b, "text": clean(joined, 70)}) + return [b for b in bands if b["x1"] - b["x0"] >= 4] + + +def fits(rec, bands): + for w in rec["widgets"]: + cx = (w["rect"].x0 + w["rect"].x1) / 2 + if not any(b["x0"] - 2 <= cx <= b["x1"] + 2 for b in bands): + return False + return True + + +# ------------------------------------------------------------- logical tables + +MIN_TABLE_ROWS = 2 +MIN_REPEAT_ROWS = 3 +X_TOL = 9.0 + + +def split_subcolumns(bands, body): + """A header band that holds more than one widget per row is really several + columns: "Paramedics on Site?" is a Yes box beside a No box, "Burn Center" + is a Yes box above a No box, "Trauma Center" is a checkbox plus a Level. + Slot them by their position within the row (top to bottom, then left to + right) and name each slot from the word printed next to it.""" + out = [] + for band in bands: + per_row = [] + for r in body: + here = [w for w in r["widgets"] + if band["x0"] - 2 <= (w["rect"].x0 + w["rect"].x1) / 2 <= band["x1"] + 2] + here.sort(key=lambda w: (round(w["rect"].y0 / 6), w["rect"].x0)) + per_row.append(here) + k = max((len(p) for p in per_row), default=0) + if k <= 1: + out.append(dict(band, members=[p[0] if p else None for p in per_row])) + continue + if not all(len(p) in (0, k) for p in per_row): # ragged: fall back to x + slots = cluster_x([w for p in per_row for w in p]) + index = {id(w): i for i, s in enumerate(slots) for w in s} + per_row = [[next((w for w in p if index[id(w)] == i), None) + for i in range(len(slots))] for p in per_row] + k = len(slots) + else: + per_row = [p + [None] * (k - len(p)) for p in per_row] + for i in range(k): + members = [p[i] for p in per_row] + live = [m for m in members if m] + side = most_common([m["label_right"] for m in live]) or \ + most_common([m["label_left"] for m in live]) + text = clean(f"{band['text']} {side}".strip(), 70) or f"{band['text']} {i + 1}" + out.append({"x0": live[0]["rect"].x0, "x1": live[0]["rect"].x1, + "text": text, "members": members}) + return dedupe_headers([c for c in out if any(c["members"])]) + + +def dedupe_headers(cols): + """Two sub-columns of one band can pick up the same neighbouring word + ("Trauma Center: [x] Yes Level: ____"). Fall back to what the cells + themselves are called, which is the only thing left that separates them.""" + seen = {} + for c in cols: + seen.setdefault(c["text"], []).append(c) + for text, group in seen.items(): + if len(group) == 1: + continue + for i, c in enumerate(group): + own = shared_text([m for m in c["members"] if m]) + c["text"] = clean(own, 70) if own else f"{text} {i + 1}" + return cols + + +def shared_text(widgets): + """What the widgets of one column are called, with the row number removed: + "Trauma Center - Level (Row 1)" -> "Trauma Center - Level".""" + for key in ("tooltip", "field_name"): + vals = [] + for w in widgets: + v = w[key] + if key == "tooltip" and (not v or JUNK_TOOLTIP_RE.match(v)): + continue + vals.append(clean(INDEX_TAIL_RE.sub("", ROW_TAIL_RE.sub("", v)), 70)) + best = most_common(vals) + if best and len(vals) >= max(1, len(widgets) - 1) and vals.count(best) > len(vals) / 2: + return best + return "" + + +def most_common(values): + values = [v for v in values if v] + return max(set(values), key=values.count) if values else "" + + +def cluster_x(widgets): + """Group widgets into stacks that share a left edge.""" + stacks = [] + for w in sorted(widgets, key=lambda w: (w["rect"].x0, w["rect"].y0)): + for s in stacks: + if abs(s[0]["rect"].x0 - w["rect"].x0) <= X_TOL: + s.append(w) + break + else: + stacks.append([w]) + return stacks + + +def ruled_tables(recs, words): + """header row + the run of body rows underneath it that fits its columns.""" + tables, i = [], 0 + while i < len(recs): + if recs[i]["kind"] != "header": + i += 1 + continue + bands = header_bands(recs[i], words) + body, j = [], i + 1 + while j < len(recs): + r = recs[j] + if r["kind"] != "body" or r["caption"] or not fits(r, bands): + break + body.append(r) + j += 1 + if len(body) >= MIN_TABLE_ROWS and len(bands) >= 1: + cols = split_subcolumns(bands, body) + if sum(1 for c in cols if any(c["members"])) >= 1: + tables.append({"kind": "ruled", "caption": recs[i]["caption"], + "columns": cols, "rows": len(body), + "y0": recs[i]["row"]["y0"], "y1": body[-1]["row"]["y1"]}) + i = j + continue + i += 1 + return tables + + +def band_signature(rec): + return [(w["rect"].x0 + w["rect"].x1) / 2 for w in + sorted(rec["widgets"], key=lambda w: w["rect"].x0)] + + +def same_columns(a, b): + return len(a) == len(b) and all(abs(x - y) <= X_TOL for x, y in zip(a, b)) + + +def repeat_tables(recs, used): + """Runs of consecutive body rows that carry the same row label and the same + widget columns. This is how ICS 203 writes its Division/Group lists: no + header row, just the words "Division/Group" repeated down the left edge, + five rows to a branch.""" + tables, i = [], 0 + while i < len(recs): + r = recs[i] + if r["kind"] != "body" or r["caption"] or any(taken(w, used) for w in r["widgets"]): + i += 1 + continue + sig, lab = band_signature(r), norm(r["label"]) + j = i + 1 + while j < len(recs) and recs[j]["kind"] == "body" and not recs[j]["caption"] \ + and same_columns(band_signature(recs[j]), sig) and norm(recs[j]["label"]) == lab \ + and not any(taken(w, used) for w in recs[j]["widgets"]): + j += 1 + run = recs[i:j] + if len(run) >= MIN_REPEAT_ROWS and len(sig) >= 1: + cols = [] + for k in range(len(sig)): + members = [sorted(x["widgets"], key=lambda w: w["rect"].x0)[k] for x in run] + cols.append({"x0": members[0]["rect"].x0, "x1": members[0]["rect"].x1, + "text": r["label"] or clean(shared_text(members), 70), + "members": members}) + tables.append({"kind": "repeat", "caption": "", "columns": dedupe_headers(cols), + "rows": len(run), "y0": run[0]["row"]["y0"], + "y1": run[-1]["row"]["y1"]}) + i = j + continue + i += 1 + return tables + + +def loose_tables(widgets, used): + """Tables drawn without rules, inside one big merged cell: ICS 204 block 8, + ICS 202's "Other Attachments". Stacks of 3+ widgets at the same x whose + rows line up across stacks. A stack whose cells carry DIFFERENT printed + labels is not a column -- that is an org chart or a checklist (ICS 207's + unit leaders, ICS 202's "ICS 203 / ICS 204 / ..."), where each box means + something of its own.""" + cols = [] + free = [w for w in widgets if not taken(w, used)] + for block in dict.fromkeys(w["block"] for w in free): + # one numbered block at a time: an unruled table never spans two + for s in cluster_x([w for w in free if w["block"] == block]): + s.sort(key=lambda w: w["rect"].y0) + if len(s) >= MIN_REPEAT_ROWS and len({identity(w) for w in s}) == 1: + cols.append(s) + + tables = [] + for col in sorted(cols, key=lambda c: c[0]["rect"].x0): + for t in tables: + if len(t["columns"][0]["members"]) != len(col): + continue + if all(abs(a["rect"].y0 - b["rect"].y0) < 6 + for a, b in zip(t["columns"][0]["members"], col)): + t["columns"].append({"x0": col[0]["rect"].x0, "x1": col[0]["rect"].x1, + "text": col_text(col), "members": col}) + break + else: + tables.append({"kind": "loose", "caption": col[0]["block"], + "columns": [{"x0": col[0]["rect"].x0, "x1": col[0]["rect"].x1, + "text": col_text(col), "members": col}], + "rows": len(col), + "y0": col[0]["rect"].y0, "y1": col[-1]["rect"].y1}) + return tables + + +def row_lists(widgets, used): + """A row of three or more identical, evenly spaced boxes with nothing to + tell them apart is a list, not three fields: ICS 207 prints four blank + boxes side by side for "however many operations elements you have". The + schema wants one repeated key, so the run becomes a one-column table.""" + tables = [] + free = [w for w in widgets if not taken(w, used)] + for block in dict.fromkeys(w["block"] for w in free): + here = sorted((w for w in free if w["block"] == block), + key=lambda w: (round(w["rect"].y0 / 6), w["rect"].x0)) + run = [] + for w in here + [None]: + same = run and w is not None and abs(w["rect"].y0 - run[-1]["rect"].y0) < 6 \ + and abs(w["rect"].width - run[-1]["rect"].width) < 2 \ + and identity(w) == identity(run[-1]) + if same: + run.append(w) + continue + if len(run) >= MIN_REPEAT_ROWS: + tables.append({"kind": "row_list", "caption": block, + "columns": [{"x0": run[0]["rect"].x0, "x1": run[-1]["rect"].x1, + "text": col_text(run) or block, "members": list(run)}], + "rows": len(run), "y0": run[0]["rect"].y0, + "y1": run[-1]["rect"].y1}) + run = [w] if w is not None else [] + return tables + + +def identity(w): + """What distinguishes this box from its neighbours, ignoring row numbers.""" + return norm(w["tooltip"] if not JUNK_TOOLTIP_RE.match(w["tooltip"] or "") else "") \ + or norm(w["label_left"] or w["label_right"] or w["label_above"]) \ + or norm(w["field_name"]) + + +def col_text(col): + """An unruled column has no printed heading of its own, so the name the + cells share is the best description of it.""" + return shared_text(col) or most_common([w["label_above"] for w in col]) \ + or most_common([w["label_left"] for w in col]) or "" + + + +# -------------------------------------------------------------------- extract + +def extract(pdf_path, use_tooltips=True): + doc = pdfplumber.open(pdf_path, laparams={}) + reader = PdfReader(pdf_path) + all_widgets, all_tables = [], [] + for pno, page in enumerate(doc.pages): + words = page_words(page) + lines = page_lines(page) + raw = [w for w in page_widgets(reader.pages[pno]) if not is_signature(w)] + # pypdf rects are bottom-left origin; flip to pdfplumber's top-left + top = float(reader.pages[pno].cropbox.top) + left = float(reader.pages[pno].cropbox.left) + rects = [Rect(x0 - left, top - y1, x1 - left, top - y0) + for x0, y0, x1, y1 in (w.rect for w in raw)] + grids = page_grids(page) + captions = caption_cells(grids, lines) + widgets = [] + for i, (w, rect) in enumerate(zip(raw, rects)): + tip = str(w.annotation.get("/TU", "")) + rec = {"page": pno + 1, "rect": [round(v, 1) for v in rect], + "field_name": w.field_name, "field_type": w.field_type, + "tooltip": clean(tip, 90) if use_tooltips else "", + "block": block_for(rect, captions, lines), + "table_id": None, "table_caption": "", "column_header": "", + "row_index": None} + rec.update(labels_for(rect, words, [r for j, r in enumerate(rects) if j != i])) + rec["_rect"] = rect + rec["rect"] = rect + widgets.append(rec) + + page_tables = [] + used = set() + for gi, grid in enumerate(grids): + for band in page_bands(grid, words): + recs = [r for r in (classify(row, band, words, widgets) + for row in grid["rows"]) if r] + found = ruled_tables(recs, words) + mark(found, used) + rep = repeat_tables(recs, used) + mark(rep, used) + page_tables += found + rep + loose = loose_tables(widgets, used) + mark(loose, used) + lists = row_lists(widgets, used) + mark(lists, used) + page_tables += loose + lists + + for t in page_tables: + t["page"] = pno + 1 + all_tables += page_tables + all_widgets += widgets + doc.close() + + for ti, t in enumerate(sorted(all_tables, key=lambda t: (t["page"], t["y0"], t["columns"][0]["x0"]))): + t["id"] = ti + if not t["caption"]: + t["caption"] = most_common([m["block"] for c in t["columns"] + for m in c["members"] if m]) + for ci, col in enumerate(t["columns"]): + for ri, m in enumerate(col["members"]): + if m is None: + continue + m["table_id"] = ti + m["table_caption"] = t["caption"] + m["column_header"] = col["text"] + m["row_index"] = ri + 1 + all_tables.sort(key=lambda t: t["id"]) + + return {"form": os.path.basename(pdf_path)[:-4], + "widgets": all_widgets, "tables": all_tables} + + +def taken(widget, used): + return id(widget) in used + + +def mark(tables, used): + for t in tables: + for c in t["columns"]: + for m in c["members"]: + if m is not None: + used.add(id(m)) + + + diff --git a/app/services/form_filler/prompt.txt b/app/services/form_filler/prompt.txt new file mode 100644 index 00000000..c49fba61 --- /dev/null +++ b/app/services/form_filler/prompt.txt @@ -0,0 +1,17 @@ +You are a FEMA ICS form-filling engine. Extract data from the incident narrative and produce one JSON object matching the supplied schema exactly. + + +1. EVERY key in the schema MUST appear in your output. Use "" for text fields with no supporting evidence and "Off" for unsupported checkboxes. +2. Extract values ONLY from the narrative. Never invent facts, guess, or copy text from schema descriptions. +3. Descriptions explain what each field means — they are instructions to you, not output data. Never echo a description back as a value. +4. Repeated fields (ending in _2, _3, _4, etc.) often expect the same header information on each page. If the narrative provides a value for the base field, apply it to numbered variants too unless context says otherwise. +5. Dates: MM/DD/YYYY. Times: HH:MM (24-hour, no AM/PM). Date-times: MM/DD/YYYY HH:MM. +6. Checkboxes: return exactly "Yes" when the narrative supports that option, otherwise exactly "Off". +7. Tables/arrays: one row per distinct item mentioned in the narrative. Fill every column for each row. Never merge separate items into one row. Omit only unused trailing rows. Never return an empty array when the narrative contains matching data. Column names inside an array are independent from top-level fields with the same name — fill both. +8. Long text fields (summaries, narratives, objectives): preserve ALL material facts from the narrative. Short fields: return only the specific value requested. +9. Distinguish completed, current, and planned information according to the destination field. +10. Follow the schema keys, nesting, types, and enums exactly. Add no extra keys. +11. Return raw JSON only. No markdown, no explanation, no commentary. + + +The narrative and target schema follow. diff --git a/app/services/form_filler/template.py b/app/services/form_filler/template.py new file mode 100644 index 00000000..507048ac --- /dev/null +++ b/app/services/form_filler/template.py @@ -0,0 +1,198 @@ +"""PDF -> JSON schema for the model. + +Tables come from the printed grid (geometry.py), not from widget names, and +every field is described by the tooltip (/TU) the form author gave it. +expand_output() maps the model's table rows back to widget names. +""" + +import os +import re + +from pypdf import PdfReader + +from . import geometry + +SIGNATURE_TYPE = 6 + +TYPE_TO_SCHEMA = { + 0: {"type": "string"}, + 1: {"type": "string"}, + 2: {"enum": ["Yes", "Off"]}, + 3: {"type": "string"}, + 4: {"type": "string"}, + 5: {"type": "string"}, + 6: {"type": "string"}, + 7: {"type": "string"}, +} + + +def _sanitize(text): + """Turn printed column header text into a valid JSON property name.""" + s = re.sub(r"[^a-zA-Z0-9]+", "_", text.strip()).strip("_") + return s[:60] if s else "col" + + +def create_template(pdf_path): + """(schema, tables, groups) for a fillable PDF. + + schema is what the model fills: geometry tables as arrays, every other + widget as a scalar, each described by its /TU tooltip. tables and groups + map array rows back to widget names (expand_output). + """ + layout = geometry.extract(os.path.abspath(pdf_path)) + all_tables, all_groups, schema = build_schema(layout) + add_tooltip_descriptions(schema, all_groups, read_tooltips(pdf_path)) + return schema, all_tables, all_groups + + +def build_schema(layout): + """(tables, groups, schema) for a geometry.extract() result.""" + # Widget types from layout (signatures already excluded by geometry.extract) + widget_types = {w["field_name"]: w["field_type"] for w in layout["widgets"]} + + # Track widgets claimed by tables + table_widget_set = set() + + all_tables = [] + all_groups = {} + used_col_names = set() + + for ti, geo_table in enumerate(layout["tables"]): + columns = [] + for ci, col in enumerate(geo_table["columns"]): + col_name = _sanitize(col["text"]) + base = col_name + suffix = 1 + while col_name in used_col_names: + col_name = f"{base}_{suffix}" + suffix += 1 + used_col_names.add(col_name) + + members = [] + for ri, m in enumerate(col["members"]): + if m is not None: + members.append(m["field_name"]) + table_widget_set.add(m["field_name"]) + else: + members.append(f"__empty_{ti}_{ci}_{ri}__") + + all_groups[col_name] = members + columns.append(col_name) + + first_real = next((m for m in col["members"] if m is not None), None) + widget_types[col_name] = first_real["field_type"] if first_real else 7 + + all_tables.append({ + "columns": columns, + "row_count": geo_table["rows"], + "name": f"resource_summary_{ti}_table", + "caption": geo_table["caption"], + "headers": [col["text"] for col in geo_table["columns"]], + }) + + # Scalars: non-table widgets in document order + singles = [ + w["field_name"] + for w in layout["widgets"] + if w["field_name"] not in table_widget_set + ] + + # Build JSON schema + properties = {} + required = [] + + for name in singles: + ft = widget_types.get(name, 7) + properties[name] = {**TYPE_TO_SCHEMA.get(ft, {"type": "string"}), "description": ""} + if name not in required: + required.append(name) + + for table_num, table in enumerate(all_tables): + row_props = {} + for col in table["columns"]: + ft = widget_types.get(col, 7) + row_props[col] = { + **TYPE_TO_SCHEMA.get(ft, {"type": "string"}), + "description": "", + } + properties[table["name"]] = { + "type": "array", + "description": "", + "items": { + "type": "object", + "properties": row_props, + "required": list(row_props.keys()), + }, + "maxItems": table["row_count"], + } + required.append(table["name"]) + + schema = {"type": "object", "properties": properties, "required": required} + return all_tables, all_groups, schema + + +def expand_output(llm_output, tables, groups): + """Map array rows back to original widget field names.""" + flat = {} + + for table_num, table in enumerate(tables): + rows = llm_output.get(table["name"], []) + for i, row in enumerate(rows): + for col in table["columns"]: + if col in groups and i < len(groups[col]) and col in row: + field_name = groups[col][i] + if not field_name.startswith("__empty_"): + flat[field_name] = row[col] + + table_names = {table["name"] for table in tables} + for key, value in llm_output.items(): + if key not in table_names: + flat[key] = value + + return flat + + +# ------------------------------------------------------------------ tooltips + +ROW_SUFFIX = re.compile(r"[_\s]*Row[_\s]*\d*\s*$", re.IGNORECASE) +ITEM_NUMBER = re.compile(r"^\d+\.\s*") + + +def clean_tooltip(tooltip, field_name): + """Tooltip text worth showing: no row suffix, no item number, and blank + when it only repeats the field name.""" + text = re.sub(r"\s+", " ", (tooltip or "")).strip() + text = ROW_SUFFIX.sub("", text) + text = text.rstrip("_ ").strip() + text = ITEM_NUMBER.sub("", text).strip() + if not text or text.casefold() == field_name.casefold(): + return "" + return text + + +def read_tooltips(pdf_path): + tooltips = {} + for page in PdfReader(pdf_path).pages: + for widget in geometry.page_widgets(page): + tu = widget.annotation.get("/TU") + if tu is not None: + tooltips.setdefault(widget.field_name, str(tu)) + return tooltips + + +def add_tooltip_descriptions(schema, groups, tooltips): + """Scalars take their own tooltip; a table column takes its first row's.""" + def describe(key): + members = groups.get(key) + if members: + widget_name = next((m for m in members if not m.startswith("__empty_")), key) + else: + widget_name = key + return clean_tooltip(tooltips.get(widget_name, ""), widget_name) + + for key, prop in schema["properties"].items(): + if "items" in prop: + for column, column_prop in prop["items"]["properties"].items(): + column_prop["description"] = describe(column) + else: + prop["description"] = describe(key) diff --git a/app/services/input.py b/app/services/input.py new file mode 100644 index 00000000..ca6a5af2 --- /dev/null +++ b/app/services/input.py @@ -0,0 +1,115 @@ +from datetime import date, datetime, timezone +from uuid import UUID + +from sqlmodel import Session + +from app.api.schemas.enums import InputStatus, InputType +from app.core.config import AUDIO_DIR +from app.core.errors.base import AppError +from app.db.repositories import create_input, create_job, get_input, update_job +from app.models import Input, Job +from app.tasks.transcribe import transcribe_audio_task + + +class InputService: + def build_voice_input( + self, + original_filename: str, + station_id: str | None = None, + responder_badge: str | None = None, + incident_date_hint: date | None = None, + ) -> Input: + now = datetime.now(timezone.utc) + return Input( + input_type=InputType.voice, + status=InputStatus.queued, + original_filename=original_filename, + station_id=station_id, + responder_badge=responder_badge, + incident_date_hint=incident_date_hint, + created_at=now, + updated_at=now, + ) + + def process_voice_upload( + self, + session: Session, + audio_content: bytes, + ext: str, + filename: str, + station_id: str | None, + responder_badge: str | None, + incident_date_hint: date | None, + ) -> tuple[Input, Job]: + record = self.build_voice_input( + original_filename=filename, + station_id=station_id, + responder_badge=responder_badge, + incident_date_hint=incident_date_hint, + ) + record = create_input(session, record) + + AUDIO_DIR.mkdir(parents=True, exist_ok=True) + audio_path = AUDIO_DIR / f"{record.input_id}.{ext}" + audio_path.write_bytes(audio_content) + + # Create Job, dispatch, and backfill celery_task_id. + # On any failure after the file write, remove the orphaned audio file. + try: + job = Job(celery_task_id="", job_type="transcription", status="queued") + job = create_job(session, job) + result = transcribe_audio_task.delay(str(record.input_id), str(audio_path), job.job_id) + job.celery_task_id = result.id + job = update_job(session, job) + except Exception: + if audio_path.exists(): + audio_path.unlink() + raise + + return record, job + + def resolve_transcript(self, session: Session, input_id: UUID) -> str: + """Look up a stored Input and return its transcript, for form-fill callers. + + Shared by the sync and async fill paths so the input_id -> transcript + resolution (and its 404/409 rules) live in exactly one place. + """ + record = get_input(session, input_id) + if record is None: + raise AppError( + f"Input with ID {input_id} not found", + status_code=404, + error_code="INPUT_NOT_FOUND", + ) + if record.status != InputStatus.ready: + raise AppError( + f"Input {input_id} is not ready (status: {record.status})", + status_code=409, + error_code="INPUT_NOT_READY", + detail={"status": record.status}, + ) + return record.transcript + + def build_text_input( + self, + narrative: str, + station_id: str | None = None, + responder_badge: str | None = None, + incident_date_hint: date | None = None, + ) -> Input: + words = narrative.split() + if len(words) < 10: + raise ValueError("Must contain at least 10 words") + now = datetime.now(timezone.utc) + return Input( + input_type=InputType.text, + status=InputStatus.ready, + transcript=narrative, + character_count=len(narrative), + word_count=len(words), + station_id=station_id, + responder_badge=responder_badge, + incident_date_hint=incident_date_hint, + created_at=now, + updated_at=now, + ) diff --git a/app/services/template.py b/app/services/template.py new file mode 100644 index 00000000..5cf5861f --- /dev/null +++ b/app/services/template.py @@ -0,0 +1,197 @@ +from datetime import datetime, timezone +from pathlib import Path + +from sqlmodel import Session + +from app.api.schemas.templates import ( + ExtractedField, + MakeFillableResponse, + TemplateCreate, + TemplateResponse, + TemplateUploadResponse, +) +from app.core import paths +from app.core.pdf_utils import _count_pdf_widgets, _extract_pdf_fields +from app.db.repositories import ( + create_template, + delete_template, + get_jobs_by_template, + get_submissions_by_template, + list_templates as repo_list_templates, +) +from app.models import Template +from app.services.form_filler.template import create_template as extract_pdf_template + + +class TemplateService: + def __init__(self): + pass + + def _calculate_field_count(self, fields: dict | None, pdf_path: str = "") -> int | None: + + if not fields: + if pdf_path: + return _count_pdf_widgets(pdf_path) + return 0 + if isinstance(fields, dict) and "schema" in fields and isinstance(fields["schema"], dict): + props = fields["schema"].get("properties", {}) + count = 0 + for prop in props.values(): + if isinstance(prop, dict) and prop.get("type") == "array": + count += len(prop.get("items", {}).get("properties", {})) + else: + count += 1 + return count + if isinstance(fields, dict): + return len(fields) + return _count_pdf_widgets(pdf_path) if pdf_path else 0 + + def list_templates(self, session: Session) -> list[TemplateResponse]: + templates = repo_list_templates(session) + return [ + TemplateResponse( + id=t.id, + name=t.name, + pdf_path=t.pdf_path, + fields=t.fields, + field_count=self._calculate_field_count(t.fields, t.pdf_path), + ) + for t in templates + ] + + def resolve_pdf_path(self, path: str) -> Path: + return paths._resolve_project_file(path) + + def save_uploaded_pdf(self, directory: str, filename: str, content: bytes) -> TemplateUploadResponse: + target_dir = paths._resolve_target_directory(directory) + target_dir.mkdir(parents=True, exist_ok=True) + + target_path = target_dir / filename + if target_path.exists(): + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + target_path = target_dir / f"{target_path.stem}_{timestamp}{target_path.suffix}" + + with target_path.open("wb") as output_file: + output_file.write(content) + + relative_path = target_path.relative_to(paths.PROJECT_ROOT).as_posix() + + try: + schema, tables, groups = extract_pdf_template(str(target_path)) + fields_list: list[ExtractedField] = [] + for name, prop in schema.get("properties", {}).items(): + if prop.get("type") == "array": + sub_props = prop.get("items", {}).get("properties", {}) + for col_name, col_prop in sub_props.items(): + fields_list.append( + ExtractedField( + name=f"{name}.{col_name}", + description=col_prop.get("description", "") or col_name, + type=col_prop.get("type", "string"), + ) + ) + else: + fields_list.append( + ExtractedField( + name=name, + description=prop.get("description", "") or name, + type=prop.get("type", "string"), + ) + ) + field_count = len(fields_list) + schema_data = schema + tables_data = tables + except Exception: + extracted = _extract_pdf_fields(relative_path) + fields_list = [ + ExtractedField( + name=f["name"], + description=f.get("description", ""), + type=f.get("type", "string"), + ) + for f in (extracted or []) + ] + field_count = len(fields_list) if extracted is not None else None + schema_data = None + tables_data = None + + return TemplateUploadResponse( + filename=target_path.name, + pdf_path=relative_path, + field_count=field_count, + fields=fields_list, + schema_data=schema_data, + tables=tables_data, + ) + + def create_template(self, session: Session, template: TemplateCreate) -> TemplateResponse: + fields = template.fields + resolved_pdf = self.resolve_pdf_path(template.pdf_path) + + if not fields and resolved_pdf.exists(): + try: + schema, tables, groups = extract_pdf_template(str(resolved_pdf)) + fields = {"schema": schema, "tables": tables, "groups": groups} + except Exception: + fields = {} + + tpl = Template(name=template.name, pdf_path=template.pdf_path, fields=fields) + created = create_template(session, tpl) + return TemplateResponse( + id=created.id, + name=created.name, + pdf_path=created.pdf_path, + fields=created.fields, + field_count=self._calculate_field_count(created.fields, created.pdf_path), + ) + + + def make_fillable(self, resolved_pdf_path: str) -> MakeFillableResponse: + template_path = resolved_pdf_path[:-4] + "_template.pdf" + try: + import os + os.environ["CUDA_VISIBLE_DEVICES"] = "" + from commonforms import prepare_form + prepare_form(resolved_pdf_path, template_path) + except Exception: + template_path = resolved_pdf_path + + new_path = Path(template_path) + if not new_path.is_absolute(): + new_path = (paths.PROJECT_ROOT / new_path).resolve() + relative_path = new_path.relative_to(paths.PROJECT_ROOT).as_posix() + + return MakeFillableResponse( + pdf_path=relative_path, + field_count=_count_pdf_widgets(relative_path), + ) + + + def delete_template(self, session: Session, template: Template) -> None: + # Batched like the original route: only session.delete() per row here, + # single commit at the end (via the delete_template repo call) so the + # cascade stays atomic instead of partially committing on failure. + submissions = get_submissions_by_template(session, template.id) + for sub in submissions: + if sub.output_pdf_path: + try: + resolved_out = paths._resolve_project_file(sub.output_pdf_path) + if resolved_out.exists() and resolved_out.is_file(): + resolved_out.unlink() + except Exception: + pass + session.delete(sub) + + jobs = get_jobs_by_template(session, template.id) + for job in jobs: + session.delete(job) + + if template.pdf_path: + try: + resolved_pdf = paths._resolve_project_file(template.pdf_path) + if resolved_pdf.exists() and resolved_pdf.is_file(): + resolved_pdf.unlink() + except Exception: + pass + + delete_template(session, template) diff --git a/app/services/whisper.py b/app/services/whisper.py new file mode 100644 index 00000000..118af8ec --- /dev/null +++ b/app/services/whisper.py @@ -0,0 +1,38 @@ +import requests + +from app.core.config import WHISPER_HOST + + +def call_whisper_asr(audio_bytes: bytes, filename: str, content_type: str) -> str: + """Post audio to the local Whisper ASR sidecar and return the transcript. + + Raises ConnectionError if the service is unreachable, RuntimeError for any + other HTTP failure. Callers map these to their own error codes. + """ + whisper_url = f"{WHISPER_HOST}/asr" + files = {"audio_file": (filename, audio_bytes, content_type)} + params = {"task": "transcribe", "output": "json", "encode": "true"} + + try: + response = requests.post(whisper_url, params=params, files=files, timeout=120) + response.raise_for_status() + except requests.exceptions.ConnectionError as exc: + raise ConnectionError( + f"Could not connect to the speech-to-text service at {whisper_url}. " + "Please ensure the whisper service is running." + ) from exc + except requests.exceptions.RequestException as exc: + raise RuntimeError(f"Transcription failed: {exc}") from exc + + try: + return (response.json().get("text") or "").strip() + except ValueError: + return response.text.strip() + + +def check_whisper_available() -> bool: + """Return True if the Whisper sidecar responds with a successful status.""" + try: + return requests.get(WHISPER_HOST, timeout=3).ok + except requests.exceptions.RequestException: + return False diff --git a/app/tasks/__init__.py b/app/tasks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/tasks/fill.py b/app/tasks/fill.py new file mode 100644 index 00000000..b73bd136 --- /dev/null +++ b/app/tasks/fill.py @@ -0,0 +1,52 @@ +import logging +from datetime import datetime, timezone +from uuid import UUID + +from app.core.celery import celery_app +from app.db.database import get_session +from app.db.repositories import get_job_by_celery_id, get_template, update_job +from app.services.form import FormService + +logger = logging.getLogger(__name__) + + +@celery_app.task(bind=True, name="fill_form") +def fill_form_task(self, template_id: int, input_text: str, input_id_str: str, model: str | None = None): + session = next(get_session()) + try: + job = get_job_by_celery_id(session, self.request.id) + if not job: + raise RuntimeError(f"No job row for celery task {self.request.id}") + + job.status = "processing" + job.progress_percent = 10 + job.updated_at = datetime.now(timezone.utc) + update_job(session, job) + + template = get_template(session, template_id) + if not template: + raise ValueError(f"Template {template_id} not found") + + submission = FormService().fill_and_persist( + session, template, input_text, UUID(input_id_str), model + ) + + job.status = "completed" + job.progress_percent = 100 + job.result_url = f"/api/v1/forms/{submission.id}/download" + job.updated_at = datetime.now(timezone.utc) + update_job(session, job) + + return {"job_id": job.job_id, "result_url": job.result_url} + + except Exception as e: + logger.exception("fill_form_task failed") + job = get_job_by_celery_id(session, self.request.id) + if job: + job.status = "failed" + job.error = {"error_code": "TASK_FAILED", "message": str(e)} + job.updated_at = datetime.now(timezone.utc) + update_job(session, job) + raise + finally: + session.close() diff --git a/app/tasks/purge.py b/app/tasks/purge.py new file mode 100644 index 00000000..36817947 --- /dev/null +++ b/app/tasks/purge.py @@ -0,0 +1,90 @@ +"""Celery beat task: purge form submissions older than RETENTION_PERIOD_DAYS. + +Runs automatically if Celery Beat is configured. Can also be triggered manually +via the API endpoint POST /api/v1/forms/purge. +""" + +import logging +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from sqlmodel import select + +from app.core.celery import celery_app +from app.core.config import RETENTION_PERIOD_DAYS +from app.core.paths import PROJECT_ROOT +from app.db.database import get_session +from app.db.repositories import delete_form_submission +from app.models import FormSubmission + +logger = logging.getLogger(__name__) + + +def _safe_delete_file(file_path: str) -> bool: + """Delete a project-relative or absolute file safely. Returns True if deleted.""" + try: + candidate = Path(file_path) + if not candidate.is_absolute(): + candidate = (PROJECT_ROOT / candidate).resolve() + else: + candidate = candidate.resolve() + + # Safety: must stay inside project root + if candidate != PROJECT_ROOT and PROJECT_ROOT not in candidate.parents: + logger.warning("Skipping file outside project root: %s", candidate) + return False + + if candidate.exists() and candidate.is_file(): + candidate.unlink() + return True + except Exception as exc: + logger.error("Failed to delete file %s: %s", file_path, exc) + return False + + +@celery_app.task(name="purge_old_submissions") +def purge_old_submissions(retention_days: int | None = None) -> dict: + """Delete form submissions (and associated output PDFs) older than retention_days. + + Args: + retention_days: Number of days to retain data. Defaults to the + RETENTION_PERIOD_DAYS environment variable (default: 30). + + Returns: + dict with purged_count and retention_days_used. + """ + days = retention_days if retention_days is not None else RETENTION_PERIOD_DAYS + cutoff = datetime.now(timezone.utc) - timedelta(days=days) + logger.info("Purging submissions older than %s days (cutoff: %s)", days, cutoff) + + session = next(get_session()) + purged_count = 0 + files_deleted = 0 + + try: + statement = select(FormSubmission).where(FormSubmission.created_at < cutoff) + submissions = list(session.exec(statement)) + + for sub in submissions: + if sub.output_pdf_path: + if _safe_delete_file(sub.output_pdf_path): + files_deleted += 1 + delete_form_submission(session, sub) + purged_count += 1 + + logger.info( + "Purge complete: removed %d submissions and %d output files", + purged_count, + files_deleted, + ) + return { + "purged_count": purged_count, + "files_deleted": files_deleted, + "retention_days_used": days, + } + + except Exception as exc: + logger.exception("purge_old_submissions task failed: %s", exc) + raise + finally: + session.close() diff --git a/app/tasks/transcribe.py b/app/tasks/transcribe.py new file mode 100644 index 00000000..91999c65 --- /dev/null +++ b/app/tasks/transcribe.py @@ -0,0 +1,102 @@ +import logging +import wave +from datetime import datetime, timezone +from pathlib import Path +from uuid import UUID + +from app.api.schemas.enums import InputStatus +from app.core.celery import celery_app +from app.core.config import AUDIO_CONTENT_TYPES +from app.db.database import get_session +from app.db.repositories import get_input, get_job_by_uuid, update_input, update_job +from app.services.whisper import call_whisper_asr + +logger = logging.getLogger(__name__) + + +def _wav_duration(path: Path) -> float | None: + """Return duration in seconds for a WAV file; None for all other formats or on error.""" + if path.suffix.lower() != ".wav": + return None + try: + with wave.open(str(path)) as wf: + return wf.getnframes() / wf.getframerate() + except Exception: + return None + + +def _fail_records( + session, + input_id: UUID, + job_id_str: str, + error_code: str, + message: str, +) -> None: + now = datetime.now(timezone.utc) + record = get_input(session, input_id) + if record: + record.status = InputStatus.failed + record.error_detail = message + record.updated_at = now + update_input(session, record) + job = get_job_by_uuid(session, job_id_str) + if job: + job.status = "failed" + job.error = {"error_code": error_code, "message": message} + job.updated_at = now + update_job(session, job) + + +@celery_app.task(name="transcribe_audio") +def transcribe_audio_task(input_id_str: str, audio_path: str, job_id_str: str) -> dict: + session = next(get_session()) + input_id = UUID(input_id_str) + try: + input_record = get_input(session, input_id) + job = get_job_by_uuid(session, job_id_str) + + # start: mark both records in-flight + now = datetime.now(timezone.utc) + input_record.status = InputStatus.transcribing + input_record.updated_at = now + update_input(session, input_record) + + job.status = "processing" + job.updated_at = now + update_job(session, job) + + # transcribe + p = Path(audio_path) + ext = p.suffix.lstrip(".") + text = call_whisper_asr(p.read_bytes(), p.name, AUDIO_CONTENT_TYPES.get(ext, "audio/wav")) + + # success + words = text.split() + now = datetime.now(timezone.utc) + input_record.status = InputStatus.ready + input_record.transcript = text + input_record.character_count = len(text) + input_record.word_count = len(words) + input_record.audio_duration_seconds = _wav_duration(p) + input_record.updated_at = now + update_input(session, input_record) + + job.status = "completed" + job.result_url = f"/api/v1/input/{input_id_str}" + job.updated_at = now + update_job(session, job) + + return {"input_id": input_id_str, "job_id": job_id_str} + + except ConnectionError as exc: + logger.exception("transcribe_audio_task: Whisper unavailable for input %s", input_id_str) + _fail_records(session, input_id, job_id_str, "STT_UNAVAILABLE", str(exc)) + raise + + except RuntimeError as exc: + logger.exception("transcribe_audio_task: transcription failed for input %s", input_id_str) + _fail_records(session, input_id, job_id_str, "TRANSCRIPTION_FAILED", str(exc)) + raise + + finally: + session.close() diff --git a/benchmark/.env.example b/benchmark/.env.example new file mode 100644 index 00000000..a55aa35c --- /dev/null +++ b/benchmark/.env.example @@ -0,0 +1,13 @@ +# Copy this file to benchmark/.env (gitignored) and fill in values. +# Shell environment variables override these. + +# --- Model under test (read by app.core.config) --------------------------- +OLLAMA_HOST=http://localhost:11434 +OLLAMA_MODEL=qwen2.5:1.5b +OLLAMA_TIMEOUT=300 + +# --- LLM judge ------------------------------------------------------------ +# JUDGE_HOST defaults to OLLAMA_HOST. +# JUDGE_HOST=http://localhost:11434 +JUDGE_MODEL=qwen3.5:4b +JUDGE_TIMEOUT_SECONDS=300 diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 00000000..7d545932 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,131 @@ +# Form-filling benchmark + +Runs `app/services/form_filler` on the ICS narratives and scores the filled +fields against hand-written ground truth, with an LLM judge for answers that +are right but worded differently. + +## Setup + +```bash +pip install -r requirements.txt +cp benchmark/.env.example benchmark/.env # then set OLLAMA_HOST etc. +``` + +`benchmark/.env` is gitignored, so a private server address stays local. +Settings (see `config.py` and `.env.example`): + +| Variable | Default | Used for | +|---|---|---| +| `OLLAMA_HOST` | `http://localhost:11434` | model under test (read by `app.core.config`) | +| `OLLAMA_MODEL` | `qwen2.5:1.5b` | model under test | +| `OLLAMA_TIMEOUT` | `300` | seconds per fill request | +| `JUDGE_HOST` | `OLLAMA_HOST` | LLM judge | +| `JUDGE_MODEL` | `qwen3.5:4b` | LLM judge | +| `JUDGE_TIMEOUT_SECONDS` | `300` | seconds per judge request | + +Shell environment variables override `.env`. Both models must be pulled on +their Ollama server (`ollama pull qwen2.5:1.5b`, `ollama pull qwen3.5:4b`). + +## Running + +```bash +cd benchmark +python3 run.py # all 56 narratives (~10 min on a GPU server) +python3 run.py -d ics_203_1 ics_206_1 # a subset +python3 run.py -m qwen3.5:4b # another model under test +python3 run.py --baseline # full run, then publish it (see Baselines) +``` + +A document whose model call fails or times out, or whose output is cut off +mid-JSON (runaway generation hitting `num_predict`), is skipped and listed; +it is left out of the averages. + +## Output + +Each run gets its own folder, `results/_