diff --git a/.env.example b/.env.example index 0fcd253..65161b5 100644 --- a/.env.example +++ b/.env.example @@ -4,15 +4,31 @@ PORT=8080 # Address the port is published on. # 127.0.0.1 (default) - only reachable from this machine # 0.0.0.0 - reachable from your whole network -# The app has NO login. Only use 0.0.0.0 on a network you fully trust, or put -# a reverse proxy with authentication / a VPN in front of it. +# The app asks you to set a password on first visit (see AUTH_MODE below). +# Even so, prefer a VPN or an https reverse proxy over exposing the port directly. BIND_ADDRESS=127.0.0.1 -# REQUIRED: Flask secret key. The container will not start without it. -# Generate one with: +# Optional: Flask secret key, used to sign login sessions. Leave empty and one +# is generated on first start and saved in ./data (the instance folder), then +# reused. Do not use placeholder values like "change-me": the app refuses them. +# To choose your own: # python -c "import secrets; print(secrets.token_hex(32))" SECRET_KEY= +# How people log in: +# password (default) - set a single password on your first visit +# proxy - a reverse proxy (Authelia, Authentik, oauth2-proxy) +# logs users in and sends the AUTH_PROXY_HEADER header; +# requests without it get 401. The app's port must then +# be reachable ONLY by that proxy, or the header can be forged. +# none - no login. Anyone who can reach the page controls your backups. +AUTH_MODE=password +#AUTH_PROXY_HEADER=Remote-User + +# Set to true only when the app is served over https, so the login cookie is +# never sent over plain http. +SECURE_COOKIES=false + # IANA timezone used to display times and to interpret cron schedules # (e.g. America/New_York, Europe/London). Defaults to UTC. TIMEZONE=UTC @@ -27,3 +43,7 @@ RCLONE_CONFIG=~/.config/rclone # Host path containing the data to back up. It is mounted READ-WRITE at /mnt/ # inside the container. DATA_PATH=/mnt/storage + +# Optional: base URL used for links in notifications, e.g. http://nas.lan:8080. +# Can also be set on the Settings page. +#PUBLIC_URL= diff --git a/README.md b/README.md index 482001f..c2482e1 100644 --- a/README.md +++ b/README.md @@ -24,21 +24,22 @@ It runs in Docker on a home server, or as a Windows app for a desktop PC. Both use the same web UI. > [!CAUTION] -> **This app has no authentication.** Anyone who can reach its port can browse -> the mounted filesystem, create jobs, and run `sync`/`move` operations against -> your data (which is mounted **read-write**). Cross-site requests from other -> websites are rejected, but that doesn't protect against direct access. +> Anyone who can log in can browse the mounted filesystem, create jobs, and run +> `sync`/`move` operations against your data (which is mounted **read-write**). +> Login is on by default: the first visit asks you to choose a password. > > - By default the port is published on `127.0.0.1` only. -> - Only set `BIND_ADDRESS=0.0.0.0` on a network you fully trust. -> - **Never expose it to the internet.** If you need remote access, put it -> behind a VPN (WireGuard, Tailscale) or a reverse proxy that enforces -> authentication. +> - Only set `BIND_ADDRESS=0.0.0.0` on a network you trust. +> - **Never expose it to the internet directly.** For remote access, use a VPN +> (WireGuard, Tailscale) or a reverse proxy that enforces authentication. +> See [Security notes](#security-notes). +> - `AUTH_MODE=none` turns login off. It is an explicit opt-out, not a default. A Flask web GUI for managing [rclone](https://rclone.org) backup jobs, on a home server with Docker or on a Windows PC as a [tray app](#windows-app). Create sync/copy/move jobs, schedule them with cron, watch live transfer progress, and browse run history — from a browser, or from the Windows -app's own window. +app's own window. It also checks that each backup runs on time and succeeds, +and alerts you when one doesn't. This is a standalone app (not the upstream rclone web GUI). It never shells out to rclone; every operation goes through the **rclone RC HTTP API**. @@ -81,8 +82,39 @@ to rclone; every operation goes through the **rclone RC HTTP API**. newest failures; the run's history page lists them all, grouped by cause, with a filter (by file or message), paging, and a CSV export. Up to 50,000 failures are kept per run. +- **Backup health** — the dashboard opens with a summary of every job's state: + `healthy`, `failing`, `overdue`, `warning` (its last run deleted an unusual + number of files), or `pending`. Each job also shows its last success, next + run, failures in a row, and the 7-day success rate. Deadlines come from the + cron schedule, or from a per-job "Must succeed at least every N hours". + `GET /api/health` returns the same data as JSON. + - `failing`: a run has failed or been held since the last success. + - `overdue`: no success by the deadline. For a scheduled job, that is the + first cron time after the last success plus 60 minutes of grace. A running + job is not overdue against its schedule. + - `warning`: the last successful run deleted more than 100 files and more + than ten times the usual number (the median of the ten runs before it). + - `healthy` shows the time of the last success. `pending` means the job has + no successful run yet and no failure to report. +- **Safety checks** — before a sync starts, its source is read. If the source + can't be read or is empty, the run is `held` and rclone is not started. Each + sync also has a delete limit (default 1000 files). A held or delete-limited run + has a **Run anyway** button. See [Keeping backups safe](#keeping-backups-safe). +- **Skipped runs** — if a scheduled run comes due while the previous run of the + same job is still going, it is recorded as `skipped` instead of disappearing. +- **Notifications** — alerts through [Apprise](https://github.com/caronc/apprise) + to ntfy, Discord, Telegram, email, Gotify and 100+ more. You choose which events + notify on the Settings page. Failure messages list the top causes, and a + "Send test notification" button checks the setup. +- **Heartbeat** — pings a dead-man's-switch URL (Healthchecks.io, an Uptime Kuma + push monitor) every N minutes while the app and rclone are working. If either + one dies, the pings stop and your monitor alerts you. +- **Login** — a single password by default. Behind a reverse proxy (Authelia, + Authentik, oauth2-proxy), the app can trust the proxy's login instead. See + [Security notes](#security-notes). - **History** — every run is recorded with final stats and status - (`completed`/`failed`/`stopped`/`interrupted`). + (`completed`/`failed`/`stopped`/`interrupted`/`held`/`skipped`). Each run also + records its deletes, checks, and the exact settings it used. - **Auto-resume** — per-job toggle. Orphaned `running` records are re-dispatched on container restart; failed runs retry after a delay up to `max_retries` (default 3). Each retry is a new history record linked via @@ -104,6 +136,32 @@ to rclone; every operation goes through the **rclone RC HTTP API**. the navigation bar. The choice is stored in your browser, so each browser picks its own. +## Keeping backups safe + +`rclone sync` makes the destination match the source, so it deletes files there +that the source no longer has. If the source disappears (a dropped NFS or SMB +mount, an unplugged drive), it can look empty, and a sync would delete the whole +backup. + +Two checks stop that: + +- **Source check.** Before a sync starts, the app lists the top level of the + source. If that fails or comes back empty, the run is **held**. rclone is not + started, and the run page says why. +- **Delete limit.** A sync refuses to delete more than a set number of files in + one run (default 1000, set per job; `-1` turns it off). It is passed to rclone + as `--max-delete`. A run stopped this way is marked failed, with the reason. + +Neither kind of run is retried automatically, even when auto-resume is on. Check +the run before you run the job again. + +Copy and move jobs are not checked, because they can't delete files at the +destination. + +**Run anyway** (on a held or delete-limited run) starts the job once with both +checks off. It asks for confirmation first. Use it only when you know the +deletions are expected. + ## Why size + mixed ordering? If a backup contains both huge files (videos, disk images) and thousands of @@ -148,8 +206,14 @@ syncs keep running; choose **Quit** from the tray icon's menu to exit. If the PC was asleep or off when a schedule came due, the run starts once when the app is next running, however many were missed. The PC stays awake during a transfer, but the app can't wake a sleeping PC, and schedules only run while you -are logged in. Don't use it on a PC shared with people you don't trust: the web -UI has no login, and other accounts on the PC can reach it. +are logged in. + +The Windows app has no login by default. It listens on loopback only and is +meant for one user. Other accounts on the same PC can still reach that port, so +on a PC shared with people you don't trust, turn login on. Open +`%LOCALAPPDATA%\rClone GUI\settings.json`, add `"auth_mode": "password"`, quit the +app from the tray, and start it again. The window will then ask you to choose a +password. Data is kept in `%LOCALAPPDATA%\rClone GUI` and is left in place on uninstall. Password-encrypted rclone configs aren't supported yet. @@ -164,22 +228,27 @@ Requires Docker with the Compose plugin, and an existing rclone config ```bash cp .env.example .env -python3 -c "import secrets; print(secrets.token_hex(32))" # paste into SECRET_KEY -# Edit .env: SECRET_KEY, TIMEZONE, RCLONE_CONFIG, DATA_PATH, (PORT, BIND_ADDRESS) +# Edit .env: TIMEZONE, RCLONE_CONFIG, DATA_PATH, (PORT, BIND_ADDRESS) docker compose up -d --build ``` -Then open `http://127.0.0.1:${PORT}` (default `8080`). The container -auto-restarts (`unless-stopped`). +Then open `http://127.0.0.1:${PORT}` (default `8080`). The first visit asks you +to choose a password, so do that before anyone else can reach the port. The +container auto-restarts (`unless-stopped`). -The container **refuses to start without a real `SECRET_KEY`**; placeholder -values are rejected. +`SECRET_KEY` is optional. If it is empty, the app generates one on first start +and keeps it in `./data`, so logins survive restarts. Placeholder values such as +`change-me` are rejected. ### Configuration (`.env`) | Key | Default | Description | |-----|---------|-------------| -| `SECRET_KEY` | — (**required**) | Flask secret key. Generate a random value. | +| `SECRET_KEY` | generated | Signs login sessions. Leave empty to have one generated and saved in `./data`, or set a long random value (`python3 -c "import secrets; print(secrets.token_hex(32))"`). Placeholder values are refused. | +| `AUTH_MODE` | `password` | `password`: one password, chosen on first visit. `proxy`: trust the login from a reverse proxy (see [Security notes](#security-notes)). `none`: no login at all. | +| `AUTH_PROXY_HEADER` | `Remote-User` | Header the reverse proxy sets to the logged-in user. Only used when `AUTH_MODE=proxy`. | +| `SECURE_COOKIES` | `false` | Set to `true` when the app is served over https, so the login cookie is never sent over plain http. | +| `PUBLIC_URL` | — | Base address for links in notifications, e.g. `http://nas.lan:8080`. The value saved on the Settings page takes precedence. Passed through by `docker-compose.yml`. | | `PORT` | `8080` | Host port for the web GUI | | `BIND_ADDRESS` | `127.0.0.1` | Host address the port is published on. `0.0.0.0` exposes it to your whole network — see the warning above. | | `TIMEZONE` | `UTC` | IANA timezone (e.g. `America/New_York`) used to display times **and** to interpret cron schedules | @@ -208,6 +277,15 @@ DATA_PATH=/mnt ### Upgrading +This release adds login and safety checks, so two things change on upgrade: + +- After upgrading, the first visit asks you to choose a password. To keep the + old no-login behaviour, set `AUTH_MODE=none` in `.env` and restart. +- Existing sync jobs get a delete limit of 1000 files. If a job legitimately + deletes more than that in one run, raise the limit on the job form, or set it + to `-1` for no limit. A run that hits the limit is marked failed. **Run + anyway** starts it once without the limit. + The database schema is created and upgraded by the app itself on startup (`db.create_all()` plus `_ensure_columns()` in `app/__init__.py`), so there is no separate migration step. Just `docker compose up -d --build`. @@ -250,8 +328,9 @@ Key design rules: ### How a run works 1. Start -> `POST /ops/start/` -> `JobRunner.start_job()` -2. A `RunHistory` row is created (status `running`) and rclone is called - with `_async=true` and `_group=job_{run_id}` +2. A `RunHistory` row is created (status `running`). For a sync, the source + is checked first and the run is `held` if the check fails. Then rclone is + called with `_async=true` and `_group=job_{run_id}` 3. The SSE endpoint polls `core/stats?group=job_{run_id}` (every 2s while active, 5s when idle) 4. A 10-second monitor copies new per-file failures into the DB while a run is @@ -263,22 +342,31 @@ Key design rules: ``` app/ ├── __init__.py # App factory, startup checks, _ensure_columns() schema upgrades -├── config.py # Env-based config (SECRET_KEY, TIMEZONE, RCLONE_RC_URL, ...) +├── auth.py # Login (password / proxy / none), sessions, lockout, secret key +├── config.py # Env-based config (SECRET_KEY, AUTH_MODE, TIMEZONE, RCLONE_RC_URL, ...) ├── extensions.py # SQLAlchemy instance ├── models.py # Job, Schedule, RunHistory (SQLite) ├── rclone_client.py # HTTP wrapper for rclone RC API ├── routes/ +│ ├── health.py # GET /api/health: backup health as JSON │ ├── jobs.py # CRUD for job definitions + directory browse API │ ├── operations.py # Start/stop/resume/status of running syncs │ ├── schedules.py # Cron schedule CRUD + enable/disable │ ├── history.py # Past run list + detail view │ ├── run_errors.py # Per-run failure list (JSON, paged), summary, CSV export │ ├── sse.py # SSE endpoint — polls rclone directly (no bg thread) -│ └── settings.py # Remotes list, global bandwidth rules +│ └── settings.py # Remotes, bandwidth rules, notification and heartbeat settings ├── services/ -│ ├── job_runner.py # Core orchestration: start/stop jobs, finalize runs +│ ├── app_settings.py # Key/value settings edited in the UI (stored in SQLite) +│ ├── bandwidth.py # Weekly bandwidth rules and the single rclone speed limit +│ ├── events.py # In-process event bus: run finished/held/skipped, health changed +│ ├── health.py # Backup health rules: per-job state, deadlines, 7-day success rate +│ ├── job_runner.py # Core orchestration: start/stop jobs, safety checks, finalize runs +│ ├── notifier.py # Notifications (Apprise) and the heartbeat ping +│ ├── remotes.py # Creating rclone remotes from the GUI, including OAuth sign-in │ ├── run_errors.py # Turns rclone transfer errors into stored, grouped failures -│ └── scheduler_service.py # APScheduler cron triggers +│ ├── scheduler_service.py # APScheduler cron triggers and skipped-run records +│ └── suspects.py # Suggests folders to exclude while a run is in progress ├── security.py # Cross-site POST and ALLOWED_HOSTS checks ├── templates/ # Jinja2, Bootstrap 5 (themed via data-theme) └── static/ # CSS, vendored Bootstrap, vanilla JS (SSE, errors, themes) @@ -287,15 +375,30 @@ packaging/ # PyInstaller spec, Inno Setup script, build.ps1 tests/ # pytest suite (no rclone needed) ``` -Tech stack: Flask 3.1, SQLAlchemy (SQLite), APScheduler 3.10, Gunicorn + -gevent, Bootstrap 5 (vendored, no CDN) and vanilla JS (no build tools). The +Tech stack: Flask 3.1, SQLAlchemy (SQLite), APScheduler 3.10, Apprise +(notifications), Gunicorn + gevent, Bootstrap 5 (vendored, no CDN) and vanilla JS (no build tools). The Windows app adds pywebview, pystray, Pillow and PyInstaller. ## Security notes -- **No authentication** — intended for a trusted network only. Blueprints are - structured so Flask-Login can be added later (`@login_required`). -- **No CSRF protection** (would need Flask-WTF). +- **Login** (`AUTH_MODE`): + - `password` (default): one shared password, chosen on the first visit. Login + sessions last 30 days. Five failed attempts from one address lock that + address out for 15 minutes. Changing the password on the Settings page signs + out every other browser. + - `proxy`: the app trusts the user named in the `AUTH_PROXY_HEADER` header + (default `Remote-User`), which a reverse proxy such as Authelia, Authentik or + oauth2-proxy sets after it has logged the user in. Requests without the + header get `401`. The app's port must then be reachable only by that proxy, + or anyone can send the header. + - `none`: no login. The web UI shows a warning banner. +- **Lockouts are per client address.** Behind a reverse proxy in `password` + mode, every client appears to come from the proxy's address, so one person's + failed attempts lock everyone out for 15 minutes. +- **Cross-site requests are rejected** (`app/security.py`). A browser POST that + another website sent is refused, based on the `Sec-Fetch-Site` or `Origin` + header. Requests without those headers (scripts, `curl`) are not covered by + that check, so they still need a login. - The directory browser (`/jobs/api/browse`) and job `extra_flags` let any user of the app list arbitrary paths and set arbitrary rclone options. - The data path is mounted read-write, and mounted rclone config contains your @@ -324,8 +427,10 @@ Gotchas: - `exclude_patterns`/`extra_flags` are JSON strings in SQLite; use `job.get_excludes()` / `job.get_extra_flags()`. -- `RunHistory` does not snapshot job config; the detail page shows the - *current* job config. +- Runs record the settings they were dispatched with (`config_snapshot`), + shown under "Settings used for this run" on the run page. Runs from before + this release have no snapshot, so for those the job's *current* settings are + the only reference. - Move operations may not resume cleanly after an interruption (partial file relocation); a warning is shown in the job form. - New columns on existing tables need an `ALTER TABLE` entry in diff --git a/app/__init__.py b/app/__init__.py index e0e0474..dd29180 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,13 +1,15 @@ -from datetime import timezone +from datetime import datetime, timezone from zoneinfo import ZoneInfo, ZoneInfoNotFoundError from flask import Flask +from app.auth import init_auth, load_or_create_secret_key from app.config import Config from app.extensions import db from app.rclone_client import RcloneClient from app.security import init_request_guards from app.services.bandwidth import BandwidthController, migrate_legacy_limits +from app.services.events import EventBus from app.services.job_runner import JobRunner from app.services.scheduler_service import SchedulerService @@ -23,11 +25,16 @@ def _require_secret_key(app): - if app.config.get("SECRET_KEY") in _INSECURE_SECRET_KEYS: + key = app.config.get("SECRET_KEY") or "" + if key == "": + # Not set: make one once and keep it, so sessions survive restarts. + app.config["SECRET_KEY"] = load_or_create_secret_key(app.instance_path) + elif key in _INSECURE_SECRET_KEYS: raise RuntimeError( - "SECRET_KEY is not set (or is a placeholder). Generate one with " - "`python -c \"import secrets; print(secrets.token_hex(32))\"` " - "and set it in your .env file." + "SECRET_KEY is set to a placeholder value. Remove it from your .env " + "file to have one generated automatically, or set it to a long " + "random string: " + "`python -c \"import secrets; print(secrets.token_hex(32))\"`." ) @@ -62,6 +69,14 @@ def _ensure_columns(): "ALTER TABLE run_history ADD COLUMN retry_of_run_id INTEGER", "ALTER TABLE run_history ADD COLUMN retry_count INTEGER DEFAULT 0", "ALTER TABLE schedule ADD COLUMN catch_up_after DATETIME", + "ALTER TABLE job ADD COLUMN max_delete INTEGER DEFAULT 1000", + "ALTER TABLE run_history ADD COLUMN deletes INTEGER DEFAULT 0", + "ALTER TABLE run_history ADD COLUMN checks INTEGER DEFAULT 0", + "ALTER TABLE run_history ADD COLUMN config_snapshot TEXT", + "ALTER TABLE run_history ADD COLUMN guard VARCHAR(30)", + "ALTER TABLE job ADD COLUMN max_age_hours INTEGER", + "ALTER TABLE job ADD COLUMN health_state VARCHAR(20)", + "ALTER TABLE job ADD COLUMN health_changed_at DATETIME", ] for sql in migrations: try: @@ -72,12 +87,14 @@ def _ensure_columns(): def create_app(config_class=Config): - app = Flask(__name__) + # INSTANCE_PATH lets tests (and unusual installs) relocate the instance folder. + app = Flask(__name__, instance_path=getattr(config_class, "INSTANCE_PATH", None)) app.config.from_object(config_class) _require_secret_key(app) tz = _load_timezone(app.config["TIMEZONE"]) init_request_guards(app) + init_auth(app) # Initialize extensions db.init_app(app) @@ -94,8 +111,12 @@ def create_app(config_class=Config): bandwidth = BandwidthController(rclone, tz) app.config["BANDWIDTH"] = bandwidth + # Runner and health checks announce what happened; notifiers listen + events = EventBus() + app.config["EVENTS"] = events + # Initialize job runner - job_runner = JobRunner(rclone, bandwidth=bandwidth) + job_runner = JobRunner(rclone, bandwidth=bandwidth, events=events) app.config["JOB_RUNNER"] = job_runner # Initialize scheduler @@ -120,16 +141,36 @@ def _collect_errors(): with app.app_context(): job_runner.collect_active_errors() + def _check_health(): + from app.services.health import evaluate_and_record + + with app.app_context(): + evaluate_and_record(app, app.config["EVENTS"]) + monitor.add_job(_check_runs, "interval", seconds=10) + monitor.add_job( + _check_health, "interval", seconds=60, max_instances=1, coalesce=True + ) # rclone only remembers its last ~100 completed transfers, so failures # must be picked up quickly. Overlap is harmless: entries are de-duplicated. monitor.add_job( _collect_errors, "interval", seconds=3, max_instances=1, coalesce=True ) + from app.services.notifier import Notifier + + notifier = Notifier(app) + events.subscribe(notifier) + app.config["NOTIFIER"] = notifier + + # Dead-man's switch: only pings while rclone answers + monitor.add_job( + notifier.heartbeat_tick, "interval", seconds=60, max_instances=1, coalesce=True + ) if app.config["BACKGROUND_TASKS_ENABLED"]: monitor.start() # Register blueprints + from app.routes.health import health_bp from app.routes.history import history_bp from app.routes.jobs import jobs_bp from app.routes.operations import ops_bp @@ -144,6 +185,7 @@ def _collect_errors(): app.register_blueprint(history_bp, url_prefix="/history") app.register_blueprint(run_errors_bp, url_prefix="/runs") app.register_blueprint(sse_bp) + app.register_blueprint(health_bp) app.register_blueprint(settings_bp, url_prefix="/settings") @app.context_processor @@ -184,6 +226,13 @@ def fmt_duration_filter(td): return f"{minutes}m {seconds}s" return f"{seconds}s" + @app.template_filter("time_ago") + def time_ago_filter(dt): + """'5 minutes ago' / 'in 3 hours' for a naive-UTC datetime.""" + from app.services.health import time_ago + + return time_ago(dt, datetime.utcnow()) if dt else "never" + @app.template_filter("fmt_bytes") def fmt_bytes_filter(b): """Format bytes as a human-readable string (e.g. '1.2 GB').""" @@ -219,8 +268,14 @@ def dashboard(): rclone_connected = True except Exception: pass + from app.services.health import current_report + + health_report, _ = current_report( + app.config["TIMEZONE"], app.config["HEALTH_GRACE_MINUTES"] + ) return render_template( "dashboard.html", + health=health_report, jobs=jobs, active_runs=active_runs, recent_runs=recent_runs, diff --git a/app/auth.py b/app/auth.py new file mode 100644 index 0000000..6054bd9 --- /dev/null +++ b/app/auth.py @@ -0,0 +1,336 @@ +"""Login for the web UI. + +AUTH_MODE decides who may use the app: + +* ``password``: a single shared password, chosen on first visit (/setup). +* ``proxy``: a reverse proxy has already authenticated the user and says so + in a header (AUTH_PROXY_HEADER). The app's port must only be + reachable by that proxy, or anyone can forge the header. +* ``none``: no login at all. An explicit opt-out. +""" + +import hashlib +import os +import secrets +import time +from urllib.parse import urlsplit + +from flask import ( + Blueprint, + Response, + current_app, + flash, + jsonify, + redirect, + render_template, + request, + session, + url_for, +) +from werkzeug.security import check_password_hash, generate_password_hash + +from app.services.app_settings import get_setting, set_setting + +AUTH_MODES = ("password", "proxy", "none") +PASSWORD_SETTING = "auth_password_hash" +MIN_PASSWORD_LENGTH = 8 + +# Brute-force slowdown, per remote address. +MAX_FAILURES = 5 +FAILURE_WINDOW_SECONDS = 15 * 60 +LOCKOUT_SECONDS = 15 * 60 + +# Endpoints reachable without logging in. +_PUBLIC_ENDPOINTS = {"auth.login", "auth.setup", "auth.logout", "static", "favicon"} +# Blueprints that do their own authentication (the Windows tray's control token). +_SELF_AUTHENTICATED_BLUEPRINTS = {"desktop_control"} + +auth_bp = Blueprint("auth", __name__) + + +# --- Secret key --- + + +def load_or_create_secret_key(instance_path): + """Return the key stored in ``/secret_key``, creating it once.""" + path = os.path.join(instance_path, "secret_key") + try: + with open(path, encoding="utf-8") as handle: + key = handle.read().strip() + if key: + return key + except FileNotFoundError: + pass + except OSError as exc: + raise RuntimeError(f"Could not read {path}: {exc}") from exc + + key = secrets.token_hex(32) + try: + os.makedirs(instance_path, exist_ok=True) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(key) + except OSError as exc: + raise RuntimeError( + f"SECRET_KEY is not set and {path} could not be written ({exc}). " + "Set SECRET_KEY in your .env file, or make that folder writable." + ) from exc + return key + + +# --- Helpers --- + + +def _mode(): + return current_app.config["AUTH_MODE"] + + +def _password_hash(): + return get_setting(PASSWORD_SETTING) + + +def _hash_fingerprint(password_hash): + """Ties a session to one password, so changing it signs everyone else out.""" + return hashlib.sha256(password_hash.encode()).hexdigest()[:16] + + +def _is_authenticated(): + password_hash = _password_hash() + return bool( + password_hash + and session.get("authenticated") + and session.get("pwv") == _hash_fingerprint(password_hash) + ) + + +def _start_session(password_hash): + session.clear() # new session contents, so nothing from before login survives + session["authenticated"] = True + session["pwv"] = _hash_fingerprint(password_hash) + session.permanent = True + + +def safe_next(target): + """Return ``target`` only if it is a path on this site, else None.""" + if not target or not target.startswith("/") or target.startswith("//"): + return None + if "\\" in target or any(ord(c) < 32 for c in target): + return None + parts = urlsplit(target) + if parts.scheme or parts.netloc: + return None + return target + + +def _wants_json(): + if request.path.startswith("/api/") or request.is_json: + return True + return "text/html" not in request.headers.get("Accept", "") + + +def _state(): + return current_app.extensions["auth_state"] + + +def _locked_until(addr): + now = time.time() + until = _state()["locked"].get(addr, 0) + if until <= now: + _state()["locked"].pop(addr, None) + return 0 + return until + + +def _record_failure(addr): + now = time.time() + state = _state() + # Drop stale entries so the dicts can't grow without bound. + for key in [ + k + for k, times in state["failures"].items() + if not times or times[-1] < now - FAILURE_WINDOW_SECONDS + ]: + del state["failures"][key] + for key in [k for k, until in state["locked"].items() if until <= now]: + del state["locked"][key] + + recent = [t for t in state["failures"].get(addr, []) if t > now - FAILURE_WINDOW_SECONDS] + recent.append(now) + state["failures"][addr] = recent + if len(recent) >= MAX_FAILURES: + state["locked"][addr] = now + LOCKOUT_SECONDS + state["failures"].pop(addr, None) + + +def _clear_failures(addr): + _state()["failures"].pop(addr, None) + _state()["locked"].pop(addr, None) + + +def _lockout_message(until): + minutes = max(1, int((until - time.time() + 59) // 60)) + return ( + "Too many failed attempts. Try again in " + f"{minutes} minute{'s' if minutes != 1 else ''}." + ) + + +def _check_password(password, addr): + """Verify against the stored hash. Returns (ok, error_message).""" + until = _locked_until(addr) + if until: + return False, _lockout_message(until) + password_hash = _password_hash() + if password_hash and check_password_hash(password_hash, password or ""): + _clear_failures(addr) + return True, None + _record_failure(addr) + until = _locked_until(addr) + if until: + return False, _lockout_message(until) + return False, "Incorrect password." + + +# --- Enforcement --- + + +def init_auth(app): + mode = app.config["AUTH_MODE"] + if mode not in AUTH_MODES: + raise RuntimeError( + f"Invalid AUTH_MODE {mode!r}; use one of: {', '.join(AUTH_MODES)}." + ) + header = app.config["AUTH_PROXY_HEADER"] + if mode == "proxy" and not header: + raise RuntimeError("AUTH_MODE=proxy needs AUTH_PROXY_HEADER to be set.") + + app.extensions["auth_state"] = {"failures": {}, "locked": {}} + app.register_blueprint(auth_bp) + + @app.before_request + def _require_login(): + if mode == "none": + return None + if ( + request.endpoint is None + or request.endpoint in _PUBLIC_ENDPOINTS + or request.blueprint in _SELF_AUTHENTICATED_BLUEPRINTS + ): + return None + + if mode == "proxy": + if request.headers.get(header): + return None + return Response( + f"Unauthorized: this app expects your reverse proxy to log you in " + f"and send the {header} header, and it was missing. Check the " + "proxy's configuration.\n", + status=401, + mimetype="text/plain", + ) + + if _is_authenticated(): + return None + if _wants_json(): + return jsonify(error="login required"), 401 + if not _password_hash(): + return redirect(url_for("auth.setup")) + if request.method == "GET": + target = request.full_path.rstrip("?") + return redirect(url_for("auth.login", next=target)) + return redirect(url_for("auth.login")) + + @app.context_processor + def _inject_auth(): + return { + "auth_mode": mode, + "show_logout": mode == "password" and bool(session.get("authenticated")), + "show_auth_warning": mode == "none" and not app.config["DESKTOP_MODE"], + } + + +# --- Routes --- + + +def _password_mode_only(): + """Login pages mean nothing outside password mode.""" + if _mode() != "password": + return redirect(url_for("dashboard")) + return None + + +@auth_bp.route("/setup", methods=["GET", "POST"]) +def setup(): + if (other := _password_mode_only()) is not None: + return other + if _password_hash(): + return redirect(url_for("auth.login")) + + error = None + if request.method == "POST": + password = request.form.get("password", "") + if len(password) < MIN_PASSWORD_LENGTH: + error = f"Use at least {MIN_PASSWORD_LENGTH} characters." + elif password != request.form.get("confirm", ""): + error = "The passwords do not match." + else: + password_hash = generate_password_hash(password) + set_setting(PASSWORD_SETTING, password_hash) + _start_session(password_hash) + return redirect(url_for("dashboard")) + return render_template( + "auth/setup.html", error=error, min_length=MIN_PASSWORD_LENGTH, public_page=True + ), (400 if error else 200) + + +@auth_bp.route("/login", methods=["GET", "POST"]) +def login(): + if (other := _password_mode_only()) is not None: + return other + if not _password_hash(): + return redirect(url_for("auth.setup")) + + target = safe_next(request.values.get("next")) + error = None + status = 200 + if request.method == "POST": + ok, error = _check_password(request.form.get("password", ""), request.remote_addr) + if ok: + _start_session(_password_hash()) + return redirect(target or url_for("dashboard")) + status = 429 if _locked_until(request.remote_addr) else 401 + return ( + render_template("auth/login.html", error=error, next=target, public_page=True), + status, + ) + + +@auth_bp.route("/logout", methods=["POST"]) +def logout(): + session.clear() + if _mode() == "password": + return redirect(url_for("auth.login")) + return redirect(url_for("dashboard")) + + +@auth_bp.route("/account/password", methods=["POST"]) +def change_password(): + if _mode() != "password": + return redirect(url_for("settings.index")) + + ok, error = _check_password( + request.form.get("current_password", ""), request.remote_addr + ) + new = request.form.get("new_password", "") + if not ok: + flash(error.replace("Incorrect password.", "Current password is incorrect."), "danger") + elif len(new) < MIN_PASSWORD_LENGTH: + flash(f"The new password needs at least {MIN_PASSWORD_LENGTH} characters.", "danger") + elif new != request.form.get("confirm_password", ""): + flash("The new passwords do not match.", "danger") + else: + new_hash = generate_password_hash(new) + set_setting(PASSWORD_SETTING, new_hash) + _start_session(new_hash) # other devices are signed out + flash("Password changed. Other signed-in browsers have been logged out.", "success") + return redirect(url_for("settings.index")) diff --git a/app/config.py b/app/config.py index b098493..9d2a2a4 100644 --- a/app/config.py +++ b/app/config.py @@ -1,8 +1,10 @@ import os +from datetime import timedelta class Config: - # No default on purpose: create_app() refuses to start without a real key. + # Empty = create_app() generates one and keeps it in the instance folder. + # Placeholder values are refused. SECRET_KEY = os.environ.get("SECRET_KEY", "") SQLALCHEMY_DATABASE_URI = os.environ.get( "DATABASE_URL", "sqlite:///rclone_gui.db" @@ -11,6 +13,20 @@ class Config: # Localhost cookies are shared across ports, so the name must be our own. SESSION_COOKIE_NAME = "rclone_gui_session" SESSION_COOKIE_SAMESITE = "Lax" + # Browser sessions last 30 days after logging in. + PERMANENT_SESSION_LIFETIME = timedelta(days=30) + SESSION_COOKIE_HTTPONLY = True + # Leave off for plain-http LAN use; turn on when served over https. + SESSION_COOKIE_SECURE = os.environ.get("SECURE_COOKIES", "").strip().lower() in ( + "1", + "true", + "yes", + "on", + ) + # "password" (single-user login), "proxy" (a reverse proxy logs users in and + # sends AUTH_PROXY_HEADER) or "none" (no login at all). + AUTH_MODE = os.environ.get("AUTH_MODE", "password").strip().lower() or "password" + AUTH_PROXY_HEADER = os.environ.get("AUTH_PROXY_HEADER", "Remote-User").strip() # Host headers to accept, e.g. "nas.lan:8080,127.0.0.1:8080". Empty = any. ALLOWED_HOSTS = [ host.strip() @@ -25,11 +41,16 @@ class Config: # IANA name (e.g. "America/New_York"). Used both for displaying times and # for interpreting cron schedules. TIMEZONE = os.environ.get("TIMEZONE", "UTC") + # Base address for links in notifications, e.g. http://nas.lan:8080 + PUBLIC_URL = os.environ.get("PUBLIC_URL", "") # How late a cron run may start after coming due (e.g. the machine was # asleep) before it is skipped. None = no limit. SCHEDULE_MISFIRE_GRACE_SECONDS = 3600 # Run schedules once at startup if they came due while the app was closed. SCHEDULE_CATCH_UP = False + # How long after a job's expected run time it may still be completing + # before it counts as overdue. + HEALTH_GRACE_MINUTES = 60 # True when running as the tray app rather than in Docker. DESKTOP_MODE = False # Tests turn this off so no scheduler threads are started. diff --git a/app/models.py b/app/models.py index ee9455c..e09c92a 100644 --- a/app/models.py +++ b/app/models.py @@ -28,6 +28,12 @@ class Job(db.Model): auto_resume = db.Column(db.Boolean, default=False) max_retries = db.Column(db.Integer, default=3) retry_delay_seconds = db.Column(db.Integer, default=60) + # Stop a sync after this many deletions at the destination (-1 = no limit). + max_delete = db.Column(db.Integer, default=1000) + # Expect a success at least this often (None = derive it from the schedule). + max_age_hours = db.Column(db.Integer, nullable=True) + health_state = db.Column(db.String(20), nullable=True) + health_changed_at = db.Column(db.DateTime, nullable=True) created_at = db.Column(db.DateTime, default=datetime.utcnow) updated_at = db.Column( db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow @@ -79,6 +85,14 @@ def to_rclone_config(self): config["CheckSum"] = True elif self.compare_method == "ignore_existing": config["IgnoreExisting"] = True + # A missing source makes a sync delete the whole destination; this caps + # the damage. Before extra flags so an explicit flag still wins. + if ( + self.sync_type == "sync" + and self.max_delete is not None + and self.max_delete >= 0 + ): + config["MaxDelete"] = self.max_delete config.update(self.get_extra_flags()) return config @@ -155,6 +169,22 @@ class RunHistory(db.Model): db.Integer, db.ForeignKey("run_history.id"), nullable=True ) retry_count = db.Column(db.Integer, default=0) + deletes = db.Column(db.Integer, default=0) + checks = db.Column(db.Integer, default=0) + # JSON of what was dispatched to rclone, so the run can be explained later + # even after the job is edited. + config_snapshot = db.Column(db.Text, nullable=True) + # Safety check that stopped the run: source_empty | source_unreachable | + # delete_limit. + guard = db.Column(db.String(30), nullable=True) + + def get_config_snapshot(self): + if not self.config_snapshot: + return None + try: + return json.loads(self.config_snapshot) + except ValueError: + return None retry_of = db.relationship( "RunHistory", remote_side="RunHistory.id", uselist=False @@ -204,3 +234,16 @@ def to_dict(self): "message": self.message, "category": self.category, } + + +class AppSetting(db.Model): + """App-wide settings that are edited in the UI, stored as JSON values. + + Read and write through ``app.services.app_settings`` rather than directly. + """ + + key = db.Column(db.String(100), primary_key=True) + value = db.Column(db.Text, nullable=False, default="null") + updated_at = db.Column( + db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow + ) diff --git a/app/rclone_client.py b/app/rclone_client.py index 98a201a..f04f3d3 100644 --- a/app/rclone_client.py +++ b/app/rclone_client.py @@ -94,6 +94,10 @@ def set_bwlimit(self, rate): # --- Config / remotes --- + def noop(self): + """Cheap liveness check.""" + return self._call("rc/noop", timeout=5) + def list_remotes(self): return self._call("config/listremotes") @@ -147,6 +151,18 @@ def list_files(self, fs, remote="", recurse=False): {"fs": fs, "remote": remote, "opt": {"recurse": recurse}}, ) + def list_top(self, fs, timeout=120): + """Entries directly inside ``fs``; raises if it can't be read. + + Generous timeout: a bucket with no folders lists every object here. + """ + reply = self._call( + "operations/list", + {"fs": fs, "remote": "", "opt": {"recurse": False}}, + timeout=timeout, + ) + return reply.get("list") or [] + def list_dirs(self, fs, remote="", timeout=None): return self._call( "operations/list", diff --git a/app/routes/health.py b/app/routes/health.py new file mode 100644 index 0000000..e3eb075 --- /dev/null +++ b/app/routes/health.py @@ -0,0 +1,17 @@ +from datetime import datetime + +from flask import Blueprint, current_app, jsonify + +from app.services import health + +health_bp = Blueprint("health", __name__) + + +@health_bp.route("/api/health") +def api_health(): + report, _jobs = health.current_report( + current_app.config["TIMEZONE"], + current_app.config["HEALTH_GRACE_MINUTES"], + datetime.utcnow(), + ) + return jsonify(health.report_to_dict(report)) diff --git a/app/routes/jobs.py b/app/routes/jobs.py index c0f6ddf..6eba275 100644 --- a/app/routes/jobs.py +++ b/app/routes/jobs.py @@ -78,6 +78,24 @@ def _parse_compare_method(form): return method +def _parse_max_delete(form): + """Delete limit: an int >= -1 (-1 = no limit); 1000 if missing or invalid.""" + try: + value = int(form.get("max_delete", 1000)) + except (TypeError, ValueError): + return 1000 + return value if value >= -1 else 1000 + + +def _parse_max_age_hours(form): + """Hours a job may go without a success; None if blank, invalid or <= 0.""" + try: + value = int(form.get("max_age_hours", "").strip()) + except (TypeError, ValueError): + return None + return value if value > 0 else None + + def _parse_bandwidth_rules(form): """The job form's bandwidth rules: a list of rule values, None if the form didn't include any (leave them alone), or ValueError.""" @@ -123,6 +141,8 @@ def create_job(): order_by_direction=order_by_direction, order_by_mixed_window=order_by_mixed_window, compare_method=_parse_compare_method(request.form), + max_delete=_parse_max_delete(request.form), + max_age_hours=_parse_max_age_hours(request.form), auto_resume="auto_resume" in request.form, max_retries=int(request.form.get("max_retries", 3)), retry_delay_seconds=int( @@ -183,6 +203,8 @@ def edit_job(job_id): job.order_by_direction = order_by_direction job.order_by_mixed_window = order_by_mixed_window job.compare_method = _parse_compare_method(request.form) + job.max_delete = _parse_max_delete(request.form) + job.max_age_hours = _parse_max_age_hours(request.form) job.auto_resume = "auto_resume" in request.form job.max_retries = int(request.form.get("max_retries", 3)) job.retry_delay_seconds = int( diff --git a/app/routes/operations.py b/app/routes/operations.py index 5c999c4..f522b65 100644 --- a/app/routes/operations.py +++ b/app/routes/operations.py @@ -32,12 +32,20 @@ def running(): def start_job(job_id): job = Job.query.get_or_404(job_id) runner = current_app.config["JOB_RUNNER"] + override = request.form.get("override") == "1" try: - run = runner.start_job(job, triggered_by="manual") - flash( - f"Job '{job.name}' started (rclone job #{run.rclone_jobid}).", - "success", + run = runner.start_job( + job, + triggered_by="manual_override" if override else "manual", + allow_deletes=override, ) + if run.status == "held": + flash(f"Job '{job.name}' was not started. {run.error_message}", "warning") + else: + flash( + f"Job '{job.name}' started (rclone job #{run.rclone_jobid}).", + "success", + ) except Exception as e: flash(f"Failed to start job: {e}", "danger") return redirect(url_for("ops.running")) @@ -59,6 +67,10 @@ def resume_run(run_id): flash("Only failed or interrupted runs can be resumed.", "warning") return redirect(url_for("history.detail", run_id=run_id)) + if run.guard: + flash("A safety check stopped this run; use Run anyway instead.", "warning") + return redirect(url_for("history.detail", run_id=run_id)) + job = run.job if not job: flash("Cannot resume: job has been deleted.", "danger") @@ -66,6 +78,9 @@ def resume_run(run_id): runner = current_app.config["JOB_RUNNER"] new_run = runner._resume_run(run, reason="manual_resume") + if new_run and new_run.status == "held": + flash(f"Job '{job.name}' was not resumed. {new_run.error_message}", "warning") + return redirect(url_for("history.detail", run_id=new_run.id)) if new_run: flash(f"Job '{job.name}' resumed as Run #{new_run.id}.", "success") return redirect(url_for("ops.running")) diff --git a/app/routes/settings.py b/app/routes/settings.py index e13838f..d849ac2 100644 --- a/app/routes/settings.py +++ b/app/routes/settings.py @@ -24,9 +24,17 @@ def index(): global_rules = ( BandwidthRule.query.filter_by(job_id=None).order_by(BandwidthRule.id).all() ) + from app.services import notifier + limit = current_app.config["BANDWIDTH"].current() + hb_url, hb_minutes = notifier.get_heartbeat() return render_template( "settings.html", + notify_urls=notifier.get_urls(), + notify_on=notifier.get_notify_on(), + public_url=notifier.get_public_url(current_app), + heartbeat_url=hb_url, + heartbeat_minutes=hb_minutes, remotes=remotes, rclone_connected=rclone_connected, bandwidth_rules=[bandwidth.rule_to_dict(r) for r in global_rules], @@ -53,6 +61,100 @@ def save_bandwidth(): return redirect(url_for("settings.index")) +# --- Notifications --- + + +def _notification_form(): + """Parse the form; returns (urls, rejected line numbers, values or None).""" + import apprise + + urls, rejected = [], [] + probe = apprise.Apprise() + for number, line in enumerate( + (request.form.get("notify_urls") or "").splitlines(), start=1 + ): + line = line.strip() + if not line: + continue + if probe.add(line): + urls.append(line) + else: + rejected.append(number) + return urls, rejected + + +@settings_bp.route("/notifications", methods=["POST"]) +def save_notifications(): + from app.services.app_settings import set_setting + from app.services.notifier import DEFAULT_NOTIFY_ON + + urls, rejected = _notification_form() + if rejected: + flash( + "Not saved. Could not understand notification URL on line " + + ", ".join(str(n) for n in rejected) + + ".", + "danger", + ) + return redirect(url_for("settings.index")) + + heartbeat_url = (request.form.get("heartbeat_url") or "").strip() + if heartbeat_url and not heartbeat_url.lower().startswith(("http://", "https://")): + flash("Not saved. The heartbeat URL must start with http:// or https://.", "danger") + return redirect(url_for("settings.index")) + try: + minutes = max(1, int(request.form.get("heartbeat_minutes") or 5)) + except ValueError: + flash("Not saved. Heartbeat minutes must be a whole number.", "danger") + return redirect(url_for("settings.index")) + + public_url = (request.form.get("public_url") or "").strip().rstrip("/") + if public_url and not public_url.lower().startswith(("http://", "https://")): + flash("Not saved. The link base URL must start with http:// or https://.", "danger") + return redirect(url_for("settings.index")) + + set_setting("notify_urls", urls) + set_setting( + "notify_on", + {key: key in request.form.getlist("notify_on") for key in DEFAULT_NOTIFY_ON}, + ) + set_setting("public_url", public_url) + set_setting("heartbeat_url", heartbeat_url) + set_setting("heartbeat_minutes", minutes) + flash("Notification settings saved.", "success") + return redirect(url_for("settings.index")) + + +@settings_bp.route("/notifications/test", methods=["POST"]) +def test_notifications(): + urls, rejected = _notification_form() + if rejected: + flash( + "Could not understand notification URL on line " + + ", ".join(str(n) for n in rejected) + + ".", + "danger", + ) + return redirect(url_for("settings.index")) + if not urls: + from app.services.notifier import get_urls + + urls = get_urls() + if not urls: + flash("Add at least one notification URL first.", "warning") + return redirect(url_for("settings.index")) + try: + ok = current_app.config["NOTIFIER"].send_test(urls) + except Exception: + current_app.logger.exception("Test notification failed") + ok = False + if ok: + flash("Test notification sent.", "success") + else: + flash("The test notification could not be delivered. Check the URLs and the log.", "danger") + return redirect(url_for("settings.index")) + + # --- Adding remotes --- diff --git a/app/services/app_settings.py b/app/services/app_settings.py new file mode 100644 index 0000000..f714a8d --- /dev/null +++ b/app/services/app_settings.py @@ -0,0 +1,26 @@ +"""Key/value settings edited in the UI (notification targets, auth, ...).""" + +import json + +from app.extensions import db +from app.models import AppSetting + + +def get_setting(key, default=None): + row = db.session.get(AppSetting, key) + if row is None: + return default + try: + return json.loads(row.value) + except ValueError: + return default + + +def set_setting(key, value): + """Store a JSON-serialisable value and commit.""" + row = db.session.get(AppSetting, key) + if row is None: + row = AppSetting(key=key) + db.session.add(row) + row.value = json.dumps(value) + db.session.commit() diff --git a/app/services/events.py b/app/services/events.py new file mode 100644 index 0000000..8b223e6 --- /dev/null +++ b/app/services/events.py @@ -0,0 +1,39 @@ +"""In-process events, so the runner and health checks don't call notifiers directly. + +Listeners run synchronously in the emitter's greenlet. A listener that raises +is logged and skipped; it never breaks the code that emitted the event. + +Events and their data (all keyword arguments): + +``run_finished`` + run_id, job_id, job_name, status (completed | failed | stopped | + interrupted), errors (int), error_message (str or None), guard (str or + None), will_retry (bool: an automatic retry has been scheduled) +``run_held`` + run_id, job_id, job_name, guard (source_empty | source_unreachable), + reason (str, readable) +``run_skipped`` + run_id, job_id, job_name, reason (str) +``health_changed`` + job_id, job_name, old_state, new_state, detail (str, readable) +""" + +import logging + +logger = logging.getLogger(__name__) + + +class EventBus: + def __init__(self): + self._listeners = [] + + def subscribe(self, listener): + """``listener(event, data)`` is called for every emitted event.""" + self._listeners.append(listener) + + def emit(self, event, **data): + for listener in list(self._listeners): + try: + listener(event, data) + except Exception: + logger.exception("Listener %r failed on %s", listener, event) diff --git a/app/services/health.py b/app/services/health.py new file mode 100644 index 0000000..a3cbddf --- /dev/null +++ b/app/services/health.py @@ -0,0 +1,361 @@ +"""Backup health: is each job succeeding on time? + +``evaluate_job`` and ``build_report`` are pure given explicit ``now`` (naive +UTC, like every DB datetime), the display timezone and the grace period. +``load_runs`` fetches only the runs the rules need, and +``evaluate_and_record`` stores state changes and emits ``health_changed``. +""" + +import logging +import statistics +from dataclasses import asdict, dataclass +from datetime import datetime, timedelta +from datetime import timezone as dt_timezone + +from apscheduler.triggers.cron import CronTrigger +from sqlalchemy import and_, or_ + +logger = logging.getLogger(__name__) + +FAILURE_STATUSES = ("failed", "held") +ATTENTION_STATES = ("failing", "overdue", "warning") +_STATE_ORDER = { + "failing": 0, + "overdue": 1, + "warning": 2, + "pending": 3, + "healthy": 4, +} +_TIME_FMT = "%Y-%m-%d %H:%M" + + +@dataclass +class JobHealth: + job_id: int + name: str + source: str + destination: str + state: str + detail: str + running: bool + consecutive_failures: int + last_success: datetime = None + last_run: datetime = None + last_run_status: str = None + next_run: datetime = None + success_rate_7d: float = None + + def to_dict(self): + data = asdict(self) + for key in ("last_success", "last_run", "next_run"): + data[key] = iso_z(data[key]) + return data + + +def iso_z(dt): + return dt.isoformat() + "Z" if dt else None + + +def time_ago(dt, now): + """'just now', '5 minutes ago', 'in 3 hours' for a naive-UTC datetime.""" + if dt is None: + return "never" + seconds = int((now - dt).total_seconds()) + future = seconds < 0 + seconds = abs(seconds) + if seconds < 60: + return "in a moment" if future and seconds >= 10 else "just now" + for size, unit in ((86400, "day"), (3600, "hour"), (60, "minute")): + if seconds >= size: + n = seconds // size + text = f"{n} {unit}{'' if n == 1 else 's'}" + return f"in {text}" if future else f"{text} ago" + + +def _local_text(dt, tz): + return dt.replace(tzinfo=dt_timezone.utc).astimezone(tz).strftime(_TIME_FMT) + + +def next_fire(cron_expression, after, tz): + """First cron fire time strictly after ``after`` (naive UTC), as naive UTC.""" + try: + trigger = CronTrigger.from_crontab(cron_expression, timezone=tz) + fire = trigger.get_next_fire_time( + None, after.replace(tzinfo=dt_timezone.utc) + timedelta(microseconds=1) + ) + except Exception: + logger.warning("Cannot evaluate cron %r", cron_expression) + return None + if fire is None: + return None + return fire.astimezone(dt_timezone.utc).replace(tzinfo=None) + + +def _run_key(run): + return (run.started_at or datetime.min, run.id or 0) + + +def _finished(run): + return run.finished_at or run.started_at + + +def evaluate_job(job, runs, now, tz, grace): + """Health of one job from its relevant ``runs`` (any order).""" + runs = sorted(runs, key=_run_key, reverse=True) + schedule = job.schedule + enabled_schedule = schedule if schedule is not None and schedule.enabled else None + + completed = [r for r in runs if r.status == "completed"] + last_success = completed[0] if completed else None + if last_success is None: + newer = runs + else: + newer = [r for r in runs if _run_key(r) > _run_key(last_success)] + failures = [r for r in newer if r.status in FAILURE_STATUSES] + running = any(r.status == "running" for r in runs) + last_run = runs[0] if runs else None + + week_ago = now - timedelta(days=7) + recent = [r for r in runs if r.started_at and r.started_at >= week_ago] + ok = sum(1 for r in recent if r.status == "completed") + bad = sum(1 for r in recent if r.status in FAILURE_STATUSES) + rate = ok / (ok + bad) if ok + bad else None + + next_run = None + if enabled_schedule: + next_run = next_fire(enabled_schedule.cron_expression, now, tz) + + state, detail = None, "" + if failures: + newest = failures[0] + msg = (newest.error_message or "").strip() + if newest.status == "held": + detail = msg or "Held by a safety check." + elif msg: + detail = "Last run failed: " + msg[:120] + elif newest.errors: + detail = f"Last run failed: {newest.errors} errors" + else: + detail = "Last run failed" + if running: + detail += " (retrying now)" + state = "failing" + + if state is None: + overdue = _overdue_detail( + job, enabled_schedule, last_success, running, now, tz, grace + ) + if overdue: + state, detail = "overdue", overdue + + if state is None and last_success is not None: + previous = [r.deletes or 0 for r in completed[1:11]] + if len(previous) >= 3: + usual = statistics.median(previous) + if (last_success.deletes or 0) > max(100, 10 * usual): + state = "warning" + detail = ( + f"Last run deleted {last_success.deletes} files " + f"(usually about {int(usual)})." + ) + + if state is None and last_success is not None: + state = "healthy" + detail = "Last success " + time_ago(_finished(last_success), now) + if state is None: + state = "pending" + detail = ( + "Waiting for its first run" + if enabled_schedule or runs + else "Not scheduled" + ) + + return JobHealth( + job_id=job.id, + name=job.name, + source=job.source, + destination=job.destination, + state=state, + detail=detail, + running=running, + consecutive_failures=len(failures), + last_success=_finished(last_success) if last_success else None, + last_run=last_run.started_at if last_run else None, + last_run_status=last_run.status if last_run else None, + next_run=next_run, + success_rate_7d=rate, + ) + + +def _overdue_detail(job, schedule, last_success, running, now, tz, grace): + since = ( + f"since {_local_text(_finished(last_success), tz)}" + if last_success + else "since it was created" + ) + if job.max_age_hours: + reference = _finished(last_success) if last_success else job.created_at + if reference and now - reference > timedelta(hours=job.max_age_hours): + return ( + f"No successful run {since}; one was due at least every " + f"{job.max_age_hours} hours." + ) + return None + if schedule is None or running: + return None + if last_success: + baseline = last_success.started_at + else: + stamps = [d for d in (job.created_at, schedule.created_at) if d] + baseline = max(stamps) if stamps else None + if baseline is None: + return None + first_due = next_fire(schedule.cron_expression, baseline, tz) + if first_due is not None and now > first_due + grace: + return ( + f"No successful run {since}; one was due by " + f"{_local_text(first_due, tz)}." + ) + return None + + +def sort_entries(entries): + return sorted(entries, key=lambda e: (_STATE_ORDER[e.state], e.name.lower())) + + +def build_report(jobs, runs_by_job, now, tz, grace): + """Per-job entries (attention first) plus summary counts.""" + entries = sort_entries( + [ + evaluate_job(job, runs_by_job.get(job.id, []), now, tz, grace) + for job in jobs + ] + ) + ok = bad = 0 + week_ago = now - timedelta(days=7) + for runs in runs_by_job.values(): + for r in runs: + if r.started_at and r.started_at >= week_ago: + if r.status == "completed": + ok += 1 + elif r.status in FAILURE_STATUSES: + bad += 1 + return { + "generated_at": now, + "total": len(entries), + "healthy": sum(1 for e in entries if e.state == "healthy"), + "attention": sum(1 for e in entries if e.state in ATTENTION_STATES), + "pending": sum(1 for e in entries if e.state == "pending"), + "success_rate_7d": ok / (ok + bad) if ok + bad else None, + "jobs": entries, + } + + +def report_to_dict(report): + return { + "generated_at": iso_z(report["generated_at"]), + "total": report["total"], + "healthy": report["healthy"], + "attention": report["attention"], + "pending": report["pending"], + "success_rate_7d": report["success_rate_7d"], + "jobs": [e.to_dict() for e in report["jobs"]], + } + + +def load_runs(jobs, now): + """Only the runs the rules need, per job id: failures since the last + success, the last 11 completed runs, running runs, the last 7 days and the + newest run.""" + from app.models import RunHistory + + result = {} + week_ago = now - timedelta(days=7) + newest_first = (RunHistory.started_at.desc(), RunHistory.id.desc()) + for job in jobs: + base = RunHistory.query.filter_by(job_id=job.id) + completed = base.filter_by(status="completed").order_by(*newest_first) + last_ok = completed.first() + since = base.filter(RunHistory.status.in_(FAILURE_STATUSES)) + if last_ok is not None: + since = since.filter( + or_( + RunHistory.started_at > last_ok.started_at, + and_( + RunHistory.started_at == last_ok.started_at, + RunHistory.id > last_ok.id, + ), + ) + ) + queries = [ + completed.limit(11), + base.filter( + or_( + RunHistory.status == "running", + RunHistory.started_at >= week_ago, + ) + ), + base.order_by(*newest_first).limit(1), + since.limit(1000), + ] + found = {} + for query in queries: + for run in query.all(): + found[run.id] = run + result[job.id] = list(found.values()) + return result + + +def current_report(tz_name, grace_minutes, now=None): + """(report, jobs) for the current database contents.""" + from zoneinfo import ZoneInfo + + from app.models import Job + + now = now or datetime.utcnow() + jobs = Job.query.order_by(Job.name).all() + report = build_report( + jobs, + load_runs(jobs, now), + now, + ZoneInfo(tz_name), + timedelta(minutes=grace_minutes), + ) + return report, jobs + + +def evaluate_and_record(app, events=None, now=None): + """Re-evaluate every job; store and announce state changes. + + The first evaluation of a job (stored state None) is recorded silently. + """ + from app.extensions import db + + with app.app_context(): + events = events or app.config.get("EVENTS") + now = now or datetime.utcnow() + report, jobs = current_report( + app.config["TIMEZONE"], app.config["HEALTH_GRACE_MINUTES"], now + ) + by_id = {j.id: j for j in jobs} + changes = [] + for entry in report["jobs"]: + job = by_id[entry.job_id] + if job.health_state != entry.state: + old = job.health_state + job.health_state = entry.state + job.health_changed_at = now + if old is not None: + changes.append((entry, old)) + db.session.commit() + if events is not None: + for entry, old in changes: + events.emit( + "health_changed", + job_id=entry.job_id, + job_name=entry.name, + old_state=old, + new_state=entry.state, + detail=entry.detail, + ) + return report diff --git a/app/services/job_runner.py b/app/services/job_runner.py index 58f460f..6cf6b5b 100644 --- a/app/services/job_runner.py +++ b/app/services/job_runner.py @@ -1,3 +1,4 @@ +import json import logging from datetime import datetime @@ -10,29 +11,39 @@ class JobAlreadyRunning(Exception): - pass + def __init__(self, message, existing_run_id=None): + super().__init__(message) + self.existing_run_id = existing_run_id + + +def _mentions_max_delete(text): + lowered = (text or "").lower() + return "max-delete" in lowered or "max delete" in lowered class JobRunner: - def __init__(self, rclone_client, bandwidth=None): + def __init__(self, rclone_client, bandwidth=None, events=None): self.rclone = rclone_client self.bandwidth = bandwidth # BandwidthController; None = no limits + self.events = events # EventBus; None = nobody listening self._active_runs = {} # {run_history_id: rclone_jobid} self._pending_retries = set() # run_ids awaiting delayed retry self.suspects = SuspectTracker() - def start_job(self, job, triggered_by="manual"): + def start_job(self, job, triggered_by="manual", allow_deletes=False): + """Start a run; returns it, already ``held`` if a safety check stopped it. + + ``allow_deletes`` is the one-off override: no preflight, no delete limit. + """ existing = RunHistory.query.filter_by( job_id=job.id, status="running" ).first() if existing: raise JobAlreadyRunning( - f"Job '{job.name}' is already running as Run #{existing.id}" + f"Job '{job.name}' is already running as Run #{existing.id}", + existing_run_id=existing.id, ) - config = job.to_rclone_config() - filters = job.to_rclone_filter() - # Create run record first to get an ID for the stats group run = RunHistory( job_id=job.id, @@ -42,30 +53,121 @@ def start_job(self, job, triggered_by="manual"): db.session.add(run) db.session.flush() # get run.id - stats_group = f"job_{run.id}" - run.stats_group = stats_group + run.stats_group = f"job_{run.id}" + if self._dispatch(run, job, allow_deletes=allow_deletes): + self._active_runs[run.id] = run.rclone_jobid + return run + + def _dispatch(self, run, job, allow_deletes=False): + """Preflight, record the settings and start rclone for ``run``. + + Returns False (run marked held, rclone untouched) if a safety check + failed, True once rclone has accepted the job and the run is committed. + """ + config = job.to_rclone_config() + if allow_deletes: + config.pop("MaxDelete", None) + filters = job.to_rclone_filter() + + if job.sync_type == "sync" and not allow_deletes: + held = self._preflight(job) + if held: + self._hold(run, job, *held) + return False + + run.config_snapshot = json.dumps( + { + "source": job.source, + "destination": job.destination, + "sync_type": job.sync_type, + "config": config, + "filter": filters, + } + ) dispatch = { "sync": self.rclone.start_sync, "copy": self.rclone.start_copy, "move": self.rclone.start_move, } - fn = dispatch[job.sync_type] self._update_bandwidth(starting=job) result = fn( src_fs=job.source, dst_fs=job.destination, - group=stats_group, + group=run.stats_group, _config=config, _filter=filters, ) - run.rclone_jobid = result["jobid"] db.session.commit() + return True - self._active_runs[run.id] = run.rclone_jobid - return run + def _preflight(self, job): + """(guard, reason) if a sync must not start, else None. + + An empty or unreadable source (say a dropped NFS mount) would make the + sync delete everything at the destination. + """ + try: + entries = self.rclone.list_top(job.source) + except Exception as exc: + return ( + "source_unreachable", + f"Source could not be read, so the sync was not started: {exc}", + ) + if not entries: + return ( + "source_empty", + "Source folder is empty. A sync would delete everything at " + "the destination, so it was not started.", + ) + return None + + def _hold(self, run, job, guard, reason): + now = datetime.utcnow() + run.status = "held" + run.guard = guard + run.error_message = reason + run.started_at = run.finished_at = now + db.session.commit() + logger.warning("Held Run #%d of '%s': %s", run.id, job.name, reason) + if self.events: + self.events.emit( + "run_held", + run_id=run.id, + job_id=job.id, + job_name=job.name, + guard=guard, + reason=reason, + ) + + def _emit_finished(self, run, will_retry=False): + if not self.events: + return + job = db.session.get(Job, run.job_id) + self.events.emit( + "run_finished", + run_id=run.id, + job_id=run.job_id, + job_name=job.name if job else None, + status=run.status, + errors=run.errors or 0, + error_message=run.error_message, + guard=run.guard, + will_retry=will_retry, + ) + + def _flag_delete_limit(self, run): + """Mark a run that rclone stopped at --max-delete.""" + snapshot = run.get_config_snapshot() or {} + limit = (snapshot.get("config") or {}).get("MaxDelete") + what = f" of {limit} files" if limit is not None else "" + run.guard = "delete_limit" + run.error_message = ( + f"Stopped after reaching the delete limit{what}. " + + (run.error_message or "") + ).strip() def _update_bandwidth(self, starting=None): if self.bandwidth: @@ -87,6 +189,7 @@ def stop_job(self, run): db.session.commit() self._active_runs.pop(run.id, None) self._update_bandwidth() + self._emit_finished(run) def _parse_rclone_time(self, time_str): """Parse an rclone ISO8601 timestamp (e.g. '2025-01-01T12:00:00.123Z') @@ -153,6 +256,8 @@ def _apply_stats(run, stats): "bytes_transferred": stats.get("bytes", 0), "files_transferred": stats.get("transfers", 0), "errors": stats.get("errors", 0), + "deletes": stats.get("deletes", 0), + "checks": stats.get("checks", 0), } changed = False for attr, value in values.items(): @@ -217,11 +322,22 @@ def check_and_update_runs(self): ): run.error_message = str(stats["lastError"]) + if not success and ( + _mentions_max_delete(run.error_message) + or _mentions_max_delete((stats or {}).get("lastError")) + ): + self._flag_delete_limit(run) + db.session.commit() del self._active_runs[run_id] - # Auto-retry on failure - if not success and run_id not in self._pending_retries: + # Auto-retry on failure (never after a safety stop) + will_retry = False + if ( + not success + and not run.guard + and run_id not in self._pending_retries + ): job = db.session.get(Job, run.job_id) if ( job @@ -229,6 +345,8 @@ def check_and_update_runs(self): and run.retry_count < job.max_retries ): self._schedule_retry(run, job.retry_delay_seconds) + will_retry = True + self._emit_finished(run, will_retry=will_retry) # Limits depend on the time of day and on which runs are active, so # this also opens and closes bandwidth windows. @@ -291,9 +409,15 @@ def restore_active_runs(self): "(duplicate or rclone error)", run.id, ) + # A held or skipped resume needs attention, so don't hush it. + self._emit_finished( + run, + will_retry=bool(new_run and new_run.status == "running"), + ) else: run.status = "failed" db.session.commit() + self._emit_finished(run) logger.info( "Run #%d marked failed (auto_resume=%s, retries=%d/%s)", run.id, @@ -362,27 +486,11 @@ def _resume_run(self, old_run, reason="auto_retry"): db.session.add(new_run) db.session.flush() - stats_group = f"job_{new_run.id}" - new_run.stats_group = stats_group - - dispatch = { - "sync": self.rclone.start_sync, - "copy": self.rclone.start_copy, - "move": self.rclone.start_move, - } + new_run.stats_group = f"job_{new_run.id}" try: - fn = dispatch[job.sync_type] - self._update_bandwidth(starting=job) - result = fn( - src_fs=job.source, - dst_fs=job.destination, - group=stats_group, - _config=job.to_rclone_config(), - _filter=job.to_rclone_filter(), - ) - new_run.rclone_jobid = result["jobid"] - db.session.commit() + if not self._dispatch(new_run, job): + return new_run # held by a safety check self._active_runs[new_run.id] = new_run.rclone_jobid logger.info( "Resumed job '%s' as Run #%d (retry #%d, reason=%s)", @@ -419,7 +527,7 @@ def _do_retry(): with app.app_context(): self._pending_retries.discard(run_id) run = db.session.get(RunHistory, run_id) - if not run or run.status != "failed": + if not run or run.status != "failed" or run.guard: return # status changed (e.g., user manually restarted) self._resume_run(run, reason="auto_retry") diff --git a/app/services/notifier.py b/app/services/notifier.py new file mode 100644 index 0000000..69f7674 --- /dev/null +++ b/app/services/notifier.py @@ -0,0 +1,267 @@ +"""Phone notifications (through Apprise) and the heartbeat ping. + +The Notifier listens to the EventBus, decides whether an event is worth a +message, builds it inside the app context and sends it off the request path. +Settings live in AppSetting: notify_urls, notify_on, public_url, +heartbeat_url, heartbeat_minutes. +""" + +import logging +import time +from urllib.parse import urlsplit + +import requests +from sqlalchemy import func + +from app.extensions import db +from app.models import Job, RunError +from app.services.app_settings import get_setting + +logger = logging.getLogger(__name__) + +DEFAULT_NOTIFY_ON = { + "failed": True, + "held": True, + "skipped": True, + "overdue": True, + "recovered": True, + "warning": True, + "succeeded": False, +} + +NOTIFY_TYPES = { + "failed": "failure", + "held": "failure", + "overdue": "failure", + "skipped": "warning", + "warning": "warning", + "recovered": "success", + "succeeded": "success", +} + +DEFAULT_HEARTBEAT_MINUTES = 5 + + +def _service(url): + """Scheme only: URLs carry tokens and must never be logged.""" + return str(url).split("://", 1)[0] if "://" in str(url) else "unknown" + + +def apprise_send(urls, title, body, notify_type): + """Send through Apprise. Returns True if every service accepted it.""" + import apprise + + sender = apprise.Apprise() + added = 0 + for url in urls: + if sender.add(url): + added += 1 + else: + logger.warning("Ignoring invalid notification URL (%s)", _service(url)) + if not added: + return False + return bool(sender.notify(title=title, body=body, notify_type=notify_type)) + + +def _spawn(func_, *args): + import gevent + + gevent.spawn(func_, *args) + + +def get_urls(): + urls = get_setting("notify_urls", []) or [] + return [u.strip() for u in urls if isinstance(u, str) and u.strip()] + + +def get_notify_on(): + chosen = get_setting("notify_on", {}) or {} + return {k: bool(chosen.get(k, default)) for k, default in DEFAULT_NOTIFY_ON.items()} + + +def get_public_url(app): + value = (get_setting("public_url", "") or "").strip() + if not value: + value = (app.config.get("PUBLIC_URL") or "").strip() + return value.rstrip("/") + + +def get_heartbeat(): + url = (get_setting("heartbeat_url", "") or "").strip() + try: + minutes = int(get_setting("heartbeat_minutes", DEFAULT_HEARTBEAT_MINUTES)) + except (TypeError, ValueError): + minutes = DEFAULT_HEARTBEAT_MINUTES + return url, max(1, minutes) + + +def _truncate(text, limit=300): + text = (text or "").strip() + return text if len(text) <= limit else text[: limit - 1] + "…" + + +class Notifier: + def __init__(self, app, sender=apprise_send, spawner=_spawn): + self.app = app + self.sender = sender + self.spawner = spawner + self._last_ping = None # monotonic time of the last successful ping + + # --- events --- + + def __call__(self, event, data): + try: + self.handle(event, data) + except Exception: + logger.exception("Could not process notification for %s", event) + + def handle(self, event, data): + with self.app.app_context(): + message = self.build(event, data) + if message is None: + return + category, title, body = message + urls = get_urls() + if not urls or not get_notify_on().get(category): + return + notify_type = NOTIFY_TYPES[category] + self.spawner(self._send, urls, title, body, notify_type) + + def _send(self, urls, title, body, notify_type): + try: + ok = self.sender(urls, title, body, notify_type) + if not ok: + logger.warning( + "Notification not delivered (%s)", + ", ".join(sorted({_service(u) for u in urls})), + ) + except Exception: + logger.exception("Sending notification failed") + + # --- message building (needs app context) --- + + def build(self, event, data): + """Return (category, title, body) or None when nothing should be sent.""" + name = data.get("job_name") or "Unknown job" + if event == "run_finished": + status = data.get("status") + if data.get("will_retry"): + return None + if status in ("failed", "interrupted"): + if data.get("guard") == "delete_limit": + reason = data.get("error_message") or ( + "The delete limit was reached, so the sync was stopped " + "before deleting more." + ) + return ( + "held", + f"Backup stopped at delete limit: {name}", + self._body(data, reason, run=True), + ) + reason = data.get("error_message") or f"Run {status}." + return ( + "failed", + f"Backup failed: {name}", + self._body(data, reason, run=True, causes=True), + ) + if status == "completed": + return ( + "succeeded", + f"Backup succeeded: {name}", + self._body(data, "Finished without errors.", run=True), + ) + return None + if event == "run_held": + return ( + "held", + f"Backup held: {name}", + self._body(data, data.get("reason") or "Held by a safety check.", run=True), + ) + if event == "run_skipped": + return ( + "skipped", + f"Scheduled run skipped: {name}", + self._body(data, data.get("reason") or "Skipped.", run=True), + ) + if event == "health_changed": + new, old = data.get("new_state"), data.get("old_state") + detail = data.get("detail") or "" + if new == "overdue": + return "overdue", f"Backup overdue: {name}", self._body(data, detail) + if new == "warning": + return "warning", f"Backup warning: {name}", self._body(data, detail) + if new == "healthy" and old in ("failing", "overdue", "warning"): + return ( + "recovered", + f"Backup recovered: {name}", + self._body(data, detail), + ) + return None + + def _body(self, data, reason, run=False, causes=False): + lines = [] + job = db.session.get(Job, data["job_id"]) if data.get("job_id") else None + if job is not None: + lines.append(f"{job.source} → {job.destination}") + if reason: + lines.append(_truncate(reason)) + run_id = data.get("run_id") + if causes and run_id: + lines.extend(self._top_causes(run_id)) + base = get_public_url(self.app) + if base: + lines.append(f"{base}/history/{run_id}" if run and run_id else f"{base}/") + return "\n".join(lines) + + @staticmethod + def _top_causes(run_id): + rows = ( + db.session.query(RunError.category, func.count(RunError.id)) + .filter(RunError.run_id == run_id) + .group_by(RunError.category) + .order_by(func.count(RunError.id).desc()) + .limit(3) + .all() + ) + return [f"{count:,} × {_truncate(category, 100)}" for category, count in rows] + + # --- test message --- + + def send_test(self, urls): + return bool( + self.sender( + urls, + "rClone GUI test", + "Notifications are working.", + "success", + ) + ) + + # --- heartbeat --- + + def heartbeat_tick(self, now=None): + """Ping the external monitor if due and rclone is alive.""" + with self.app.app_context(): + url, minutes = get_heartbeat() + if not url: + return False + now = time.monotonic() if now is None else now + if self._last_ping is not None and now - self._last_ping < minutes * 60: + return False + try: + self.app.config["RCLONE_CLIENT"].noop() + except Exception as exc: + logger.warning("Heartbeat skipped: rclone is not responding (%s)", exc) + return False + try: + resp = requests.get(url, timeout=10) + resp.raise_for_status() + except Exception as exc: + logger.warning( + "Heartbeat ping to %s failed: %s", + urlsplit(url).netloc or "?", + type(exc).__name__, + ) + return False + self._last_ping = now + return True diff --git a/app/services/scheduler_service.py b/app/services/scheduler_service.py index ab49bab..e4849d7 100644 --- a/app/services/scheduler_service.py +++ b/app/services/scheduler_service.py @@ -100,6 +100,38 @@ def _run_scheduled_job(self, job_id): self.job_runner.start_job(job, triggered_by="schedule") except JobAlreadyRunning as exc: logger.info("Skipping scheduled run: %s", exc) + self._record_skipped(job, exc.existing_run_id) + + def _record_skipped(self, job, existing_run_id): + """Leave a visible trace when a run came due while the last one ran.""" + from app.extensions import db + from app.models import RunHistory + + now = datetime.utcnow() + reason = ( + f"Skipped: Run #{existing_run_id} was still running." + if existing_run_id + else "Skipped: the job was still running." + ) + run = RunHistory( + job_id=job.id, + triggered_by="schedule", + status="skipped", + started_at=now, + finished_at=now, + error_message=reason, + ) + db.session.add(run) + db.session.commit() + events = self.job_runner.events + if events: + events.emit( + "run_skipped", + run_id=run.id, + job_id=job.id, + job_name=job.name, + reason=reason, + ) def get_next_run(self, schedule_id): job_id = f"schedule_{schedule_id}" diff --git a/app/static/js/app.js b/app/static/js/app.js index 4589f23..438caf0 100644 --- a/app/static/js/app.js +++ b/app/static/js/app.js @@ -2,6 +2,15 @@ let evtSource = null; let lastData = null; +// An expired login answers fetches with 401: reload so the user lands on the login page. +const originalFetch = window.fetch.bind(window); +window.fetch = function(...args) { + return originalFetch(...args).then(response => { + if (response.status === 401) location.reload(); + return response; + }); +}; + // Forms with data-confirm ask first, in our own dialog: the browser's native one // shows the page's address as its title, even inside the Windows app's window. document.addEventListener('submit', event => { @@ -12,6 +21,7 @@ document.addEventListener('submit', event => { const modal = bootstrap.Modal.getOrCreateInstance(document.getElementById('confirmModal')); document.getElementById('confirm-message').textContent = message; + document.getElementById('confirm-ok').textContent = form.dataset.confirmLabel || 'Delete'; document.getElementById('confirm-ok').onclick = () => { modal.hide(); form.submit(); // doesn't fire 'submit' again @@ -64,6 +74,7 @@ function updateRunningBadge(count) { } function connectSSE() { + if (document.body.dataset.public) return; // login/setup pages have no stream if (evtSource) { evtSource.close(); } diff --git a/app/templates/auth/login.html b/app/templates/auth/login.html new file mode 100644 index 0000000..dd18dea --- /dev/null +++ b/app/templates/auth/login.html @@ -0,0 +1,24 @@ +{% extends "base.html" %} +{% block title %}Log in - rClone GUI{% endblock %} + +{% block content %} +
+
+
+
Log in
+
+ {% if error %}
{{ error }}
{% endif %} +
+ {% if next %}{% endif %} +
+ + +
+ +
+
+
+
+
+{% endblock %} diff --git a/app/templates/auth/setup.html b/app/templates/auth/setup.html new file mode 100644 index 0000000..354a994 --- /dev/null +++ b/app/templates/auth/setup.html @@ -0,0 +1,32 @@ +{% extends "base.html" %} +{% block title %}Set a password - rClone GUI{% endblock %} + +{% block content %} +
+
+
+
Welcome - set a password
+
+

+ Choose the password that protects this app. You will need it to log in from now on. +

+ {% if error %}
{{ error }}
{% endif %} +
+
+ + +
At least {{ min_length }} characters.
+
+
+ + +
+ +
+
+
+
+
+{% endblock %} diff --git a/app/templates/base.html b/app/templates/base.html index d40fe0c..f0c3f52 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -12,7 +12,14 @@ - + + {% if public_page %} + + {% else %} + {% endif %} + + {% if show_auth_warning %} + + + {% endif %}
{% with messages = get_flashed_messages(with_categories=true) %} diff --git a/app/templates/dashboard.html b/app/templates/dashboard.html index 2b25e9f..83a87e9 100644 --- a/app/templates/dashboard.html +++ b/app/templates/dashboard.html @@ -2,6 +2,85 @@ {% block title %}Dashboard - rClone GUI{% endblock %} {% block content %} +{% set state_style = { + 'healthy': ('success', 'bi-check-circle', 'Healthy'), + 'failing': ('danger', 'bi-x-circle', 'Failing'), + 'overdue': ('danger', 'bi-clock-history', 'Overdue'), + 'warning': ('warning', 'bi-exclamation-triangle', 'Warning'), + 'pending': ('secondary', 'bi-hourglass', 'Pending'), +} %} + +
+
+ Backup health + {% if health.success_rate_7d is not none %} + Successful runs, last 7 days: {{ '%.1f' % (health.success_rate_7d * 100) }}% + {% endif %} +
+
+
+
+
+
{{ health.total }}
+
Jobs
+
+
+
+
+
{{ health.healthy }}
+
Healthy
+
+
+
+
+
{{ health.attention }}
+
Need attention
+
+
+
+ {% if health.jobs %} +
+ + + + + + + + + + + {% for h in health.jobs %} + {% set style = state_style[h.state] %} + + + + + + + {% endfor %} + +
StateJobLast successNext run
+ {{ style[2] }} + + {{ h.name }} + {% if h.running %} running{% endif %} +
{{ h.source }} → {{ h.destination }}
+
{{ h.detail }}{% if h.consecutive_failures > 0 %} · {{ h.consecutive_failures }} failure{{ '' if h.consecutive_failures == 1 else 's' }} in a row{% endif %}
+ 		+ {% if h.last_success %} + {{ h.last_success|time_ago }} + {% else %}never{% endif %} + + {% if h.next_run %}{{ h.next_run|local_time }}{% else %}--{% endif %} +
+
+ {% else %} +

No jobs yet. Once you create one, its health shows up here.

+ {% endif %} +
+
+
@@ -157,6 +236,10 @@ Stopped {% elif run.status == 'interrupted' %} Interrupted + {% elif run.status == 'held' %} + Held + {% elif run.status == 'skipped' %} + Skipped {% endif %} {{ run.started_at|local_time }} diff --git a/app/templates/history/detail.html b/app/templates/history/detail.html index 81fe4f9..052febf 100644 --- a/app/templates/history/detail.html +++ b/app/templates/history/detail.html @@ -8,13 +8,22 @@

Run #{{ run.id }}

- {% if run.status in ['failed', 'interrupted'] and run.job %} + {% if run.status in ['failed', 'interrupted'] and run.job and not run.guard %}
{% endif %} + {% if run.guard and run.status in ['held', 'failed'] and run.job %} +
+ + +
+ {% endif %} Back @@ -43,6 +52,10 @@

Run #{{ run.id }}

Stopped {% elif run.status == 'interrupted' %} Interrupted + {% elif run.status == 'held' %} + Held + {% elif run.status == 'skipped' %} + Skipped {% endif %} {% if run.retry_count > 0 %} Retry #{{ run.retry_count }} @@ -97,6 +110,21 @@

Run #{{ run.id }}

+ {% set snapshot = run.get_config_snapshot() %} + {% if snapshot %} +
+ Settings used for this run +
+ + + + +
Source{{ snapshot.source }}
Destination{{ snapshot.destination }}
Type{{ snapshot.sync_type }}
+ 
{{ {'config': snapshot.config, 'filter': snapshot.filter}|tojson(indent=2) }}
+
+
+ {% endif %} +
Transfer Stats
diff --git a/app/templates/history/list.html b/app/templates/history/list.html index 7a88685..d8ccc03 100644 --- a/app/templates/history/list.html +++ b/app/templates/history/list.html @@ -36,6 +36,10 @@

Run History

Stopped {% elif run.status == 'interrupted' %} Interrupted + {% elif run.status == 'held' %} + Held + {% elif run.status == 'skipped' %} + Skipped {% endif %} {% if run.retry_count > 0 %} (retry #{{ run.retry_count }}) @@ -72,13 +76,22 @@

Run History

- {% if run.status in ['failed', 'interrupted'] and run.job %} + {% if run.status in ['failed', 'interrupted'] and run.job and not run.guard %}
{% endif %} + {% if run.guard and run.status in ['held', 'failed'] and run.job %} +
+ + +
+ {% endif %} {% endfor %} diff --git a/app/templates/jobs/form.html b/app/templates/jobs/form.html index 78a456c..c5853fa 100644 --- a/app/templates/jobs/form.html +++ b/app/templates/jobs/form.html @@ -150,6 +150,18 @@

How rclone decides whether a file needs to be transferred.

+
+ + +
Stop a sync after deleting this many files at the destination. Protects against a missing or empty source. -1 = no limit.
+
+
+ + +
Leave blank to expect a successful run after every scheduled time. Set it for jobs without a schedule, or to allow more slack.
+
@@ -432,4 +444,14 @@
document.getElementById('order_by_direction').addEventListener('change', toggleOrderByMixedWindow); toggleOrderByMixedWindow(); + {% endblock %} diff --git a/app/templates/settings.html b/app/templates/settings.html index 74d1936..ca411b6 100644 --- a/app/templates/settings.html +++ b/app/templates/settings.html @@ -25,6 +25,37 @@

Settings

+ {% if auth_mode == 'password' %} +
+
+
Password
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + Other browsers will be logged out. +
+
+
+
+
+ {% endif %} +
Bandwidth Limits
@@ -55,6 +86,74 @@

Settings

+ +
+
+
Notifications
+
+
+
+ + +
+ Works with ntfy, Discord, Telegram, email, Gotify, Pushover and 100+ more. + URL formats. + These contain secrets, so keep this page private. +
+
+
+
Tell me when
+ {% for key, label in [ + ('failed', 'A backup fails'), + ('held', 'A safety check holds a sync'), + ('skipped', 'A scheduled run is skipped because the last one is still running'), + ('overdue', 'A backup is overdue'), + ('warning', 'A backup has a warning (for example, many files fail)'), + ('recovered', 'A backup recovers'), + ('succeeded', 'Every successful backup'), + ] %} +
+ + +
+ {% endfor %} +
“A safety check holds a sync” also covers a sync that deletes unusually many files.
+
+
+
+ + +
Used for links in messages. Leave blank for no links.
+
+
+
+
+ + +
+ Pinged while this app and rclone are running (Healthchecks.io, Uptime Kuma push). + If pings stop, that service alerts you. +
+
+
+ + +
+
+ + +
+
+
+
diff --git a/desktop/autostart.py b/desktop/autostart.py index 8324fdf..467733e 100644 --- a/desktop/autostart.py +++ b/desktop/autostart.py @@ -17,7 +17,7 @@ def is_enabled(value_name=paths.APP_NAME): def enable(value_name=paths.APP_NAME): command = subprocess.list2cmdline(paths.app_command("--background")) - with winreg.OpenKey(winreg.HKEY_CURRENT_USER, _RUN_KEY, 0, winreg.KEY_SET_VALUE) as key: + with winreg.CreateKeyEx(winreg.HKEY_CURRENT_USER, _RUN_KEY, 0, winreg.KEY_SET_VALUE) as key: winreg.SetValueEx(key, value_name, 0, winreg.REG_SZ, command) diff --git a/desktop/server.py b/desktop/server.py index c9ce35c..e591fc5 100644 --- a/desktop/server.py +++ b/desktop/server.py @@ -42,6 +42,9 @@ class DesktopConfig(Config): SCHEDULE_MISFIRE_GRACE_SECONDS = None SCHEDULE_CATCH_UP = True DESKTOP_MODE = True + # Loopback-only and single-user, so no login by default. Add + # "auth_mode": "password" to settings.json to require one. + AUTH_MODE = user_settings.get("auth_mode") or "none" return DesktopConfig diff --git a/docker-compose.yml b/docker-compose.yml index a4ed1d5..9e1fde9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,8 +2,9 @@ services: rclone-gui: build: . ports: - # Published on loopback by default because the app has NO authentication. - # Set BIND_ADDRESS=0.0.0.0 in .env only on a network you trust. + # Published on loopback by default. Set BIND_ADDRESS=0.0.0.0 in .env to + # reach it from other machines (you will be asked to set a password on + # first visit; see AUTH_MODE). - "${BIND_ADDRESS:-127.0.0.1}:${PORT:-8080}:8080" volumes: # Persist SQLite database @@ -13,8 +14,18 @@ services: # Mount the data you want to back up or restore (read-write!) - ${DATA_PATH:-/mnt/storage}:/mnt/:rw environment: - - SECRET_KEY=${SECRET_KEY:?Set SECRET_KEY in .env (see .env.example)} + # Optional: generated once and kept in ./data when empty. + - SECRET_KEY=${SECRET_KEY:-} + # password (default) | proxy | none. See .env.example. + - AUTH_MODE=${AUTH_MODE:-password} + - AUTH_PROXY_HEADER=${AUTH_PROXY_HEADER:-Remote-User} + - SECURE_COOKIES=${SECURE_COOKIES:-false} - TIMEZONE=${TIMEZONE:-UTC} - ALLOWED_HOSTS=${ALLOWED_HOSTS:-} + # Base URL for links in notifications (also settable in Settings). + - PUBLIC_URL=${PUBLIC_URL:-} - RCLONE_RC_URL=http://127.0.0.1:5572 + dns: + - 1.1.1.1 + - 192.168.10.1 restart: unless-stopped diff --git a/docs/standalone-windows.md b/docs/standalone-windows.md index 44dcd59..f845cd3 100644 --- a/docs/standalone-windows.md +++ b/docs/standalone-windows.md @@ -93,9 +93,10 @@ rClone GUI.exe (app window: pywebview + WebView2; tray icon: pystray; no gevent) others are marked failed. - The window has no right-click menu, because pywebview ties it to debug mode. Ctrl+C and Ctrl+V still work in text fields. -- The web UI has no login, and the local server behind the window is still - reachable from a browser. Other accounts on the same PC can reach its loopback - port, so don't use it on a PC shared with people you don't trust. +- The web UI has no login unless `"auth_mode": "password"` is set in + `settings.json`. The local server behind the window is still reachable from a + browser, and other accounts on the same PC can reach its loopback port, so + turn login on for a PC shared with people you don't trust. - rclone configs encrypted with a password aren't supported yet. `rcd` runs without a console, so it can't ask for the password. - The installer is unsigned, so SmartScreen warns on first run until a diff --git a/packaging/rclone-gui.spec b/packaging/rclone-gui.spec index 81ba588..809c46f 100644 --- a/packaging/rclone-gui.spec +++ b/packaging/rclone-gui.spec @@ -11,6 +11,8 @@ datas = [ # APScheduler finds its triggers and executors through package metadata. datas += copy_metadata("APScheduler") datas += collect_data_files("tzdata") +# Apprise loads its notification plugins dynamically. +datas += collect_data_files("apprise") analysis = Analysis( ["entry.py"], @@ -20,6 +22,7 @@ analysis = Analysis( hiddenimports=[ "pystray._win32", *collect_submodules("apscheduler"), + *collect_submodules("apprise"), ], # pywebview's own hook collects its WebView2 and .NET files. excludes=["gunicorn", "pytest", "tkinter"], diff --git a/requirements.txt b/requirements.txt index 8af2da1..00773d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ gevent==24.* requests==2.32.* APScheduler==3.10.* tzdata +apprise==1.* diff --git a/tests/conftest.py b/tests/conftest.py index 52c110f..7389356 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,6 +11,7 @@ class TestConfig(Config): SQLALCHEMY_DATABASE_URI = "sqlite://" TIMEZONE = "America/New_York" BACKGROUND_TASKS_ENABLED = False + AUTH_MODE = "none" @pytest.fixture() @@ -32,6 +33,7 @@ class FakeRclone: def __init__(self): self.calls = [] + self.listed = [] # fs values list_top was asked about self.bytes_per_second = -1 # what rclone's limiter is set to def get_bwlimit(self): @@ -55,6 +57,34 @@ def start_copy(self, **kwargs): def start_move(self, **kwargs): return self._start("move", **kwargs) + # --- run lifecycle (configurable by tests) --- + top_entries = [{"Name": "a", "IsDir": False}] + top_error = None # exception raised by list_top + status = {"finished": False} # what job_status returns + stats = {} # what get_stats returns + + def list_top(self, fs, timeout=30): + self.listed.append(fs) + if self.top_error: + raise self.top_error + return self.top_entries + + def job_status(self, jobid): + return self.status + + def get_stats(self, group=None): + return self.stats + + def get_transferred(self, group=None): + return {"transferred": []} + + def job_stop(self, jobid): + self.calls.append(("job_stop", jobid)) + return {} + + def noop(self): + return {} + def list_remotes(self): return {"remotes": ["gdrive", "pcloud"]} diff --git a/tests/test_auth.py b/tests/test_auth.py new file mode 100644 index 0000000..daa684f --- /dev/null +++ b/tests/test_auth.py @@ -0,0 +1,336 @@ +import os +import stat + +import pytest + +from app import create_app +from app.auth import MAX_FAILURES +from app.extensions import db +from tests.conftest import TestConfig + +HTML = {"Accept": "text/html"} +PASSWORD = "correct horse" + + +def _make_app(**overrides): + config = type("AuthConfig", (TestConfig,), overrides) + return create_app(config) + + +@pytest.fixture() +def pw_app(): + app = _make_app(AUTH_MODE="password") + yield app + with app.app_context(): + db.session.remove() + db.drop_all() + + +@pytest.fixture() +def pw_client(pw_app): + return pw_app.test_client() + + +def _setup(client, password=PASSWORD): + return client.post("/setup", data={"password": password, "confirm": password}) + + +@pytest.fixture() +def logged_in(pw_client): + _setup(pw_client) + return pw_client + + +@pytest.fixture() +def configured(pw_app): + """A password is set, but this client is not logged in.""" + _setup(pw_app.test_client()) + return pw_app.test_client() + + +# --- First run --- + + +def test_redirects_to_setup_when_no_password(pw_client): + response = pw_client.get("/jobs/", headers=HTML) + assert response.status_code == 302 + assert response.headers["Location"].endswith("/setup") + + +def test_setup_sets_hash_and_logs_in(pw_client): + response = _setup(pw_client) + assert response.status_code == 302 + assert pw_client.get("/jobs/", headers=HTML).status_code == 200 + from app.services.app_settings import get_setting + + with pw_client.application.app_context(): + stored = get_setting("auth_password_hash") + assert stored and PASSWORD not in stored + + +def test_setup_validates_password(pw_client): + assert _setup(pw_client, "short").status_code == 400 + mismatch = pw_client.post("/setup", data={"password": PASSWORD, "confirm": "other-one!"}) + assert mismatch.status_code == 400 + assert pw_client.get("/jobs/", headers=HTML).status_code == 302 + + +def test_setup_refused_once_password_exists(configured): + response = configured.post( + "/setup", data={"password": "hijack-attempt", "confirm": "hijack-attempt"} + ) + assert response.status_code == 302 + assert response.headers["Location"].endswith("/login") + assert configured.get("/setup").headers["Location"].endswith("/login") + # The original password still works + assert configured.post("/login", data={"password": PASSWORD}).status_code == 302 + + +# --- Login --- + + +def test_unauthenticated_page_redirects_to_login_with_next(configured): + response = configured.get("/jobs/", headers=HTML) + assert response.status_code == 302 + assert "/login?next=/jobs/" in response.headers["Location"] + + +def test_login_success_and_failure(configured): + bad = configured.post("/login", data={"password": "wrong"}) + assert bad.status_code == 401 + assert b"Incorrect password" in bad.data + assert configured.get("/jobs/", headers=HTML).status_code == 302 + + good = configured.post("/login", data={"password": PASSWORD}) + assert good.status_code == 302 + assert configured.get("/jobs/", headers=HTML).status_code == 200 + + +def test_login_redirects_to_safe_next(configured): + response = configured.post("/login?next=/jobs/", data={"password": PASSWORD}) + assert response.headers["Location"].endswith("/jobs/") + + +@pytest.mark.parametrize( + "target", + ["//evil.com", "https://evil.com", "/\\evil.com", "evil.com", "javascript:alert(1)"], +) +def test_login_rejects_open_redirect(configured, target): + response = configured.post("/login", data={"password": PASSWORD, "next": target}) + assert response.status_code == 302 + location = response.headers["Location"] + assert "evil" not in location and "javascript" not in location + + +def test_lockout_after_repeated_failures(configured): + for _ in range(MAX_FAILURES): + configured.post("/login", data={"password": "wrong"}) + # Even the right password is refused while locked out + locked = configured.post("/login", data={"password": PASSWORD}) + assert locked.status_code == 429 + assert b"Too many failed attempts" in locked.data + assert configured.get("/jobs/", headers=HTML).status_code == 302 + + +def test_lockout_expires(configured, monkeypatch): + import app.auth as auth + + for _ in range(MAX_FAILURES): + configured.post("/login", data={"password": "wrong"}) + real_time = auth.time.time + monkeypatch.setattr(auth.time, "time", lambda: real_time() + auth.LOCKOUT_SECONDS + 1) + assert configured.post("/login", data={"password": PASSWORD}).status_code == 302 + + +def test_logout_requires_post(logged_in): + assert logged_in.get("/logout").status_code == 405 + assert logged_in.get("/jobs/", headers=HTML).status_code == 200 + response = logged_in.post("/logout") + assert response.status_code == 302 + assert logged_in.get("/jobs/", headers=HTML).status_code == 302 + + +def test_session_cookie_is_httponly_and_permanent(configured): + response = configured.post("/login", data={"password": PASSWORD}) + cookie = response.headers["Set-Cookie"] + assert "HttpOnly" in cookie + assert "Expires=" in cookie # permanent session + assert "Secure" not in cookie + + +# --- Enforcement --- + + +def test_json_and_sse_get_401_not_redirect(configured): + for path in ("/api/events", "/jobs/"): + response = configured.get(path) # no text/html in Accept + assert response.status_code == 401 + assert response.get_json() == {"error": "login required"} + sse = configured.get("/api/events", headers=HTML) + assert sse.status_code == 401 + + +def test_static_files_and_favicon_reachable(pw_client): + assert pw_client.get("/static/css/style.css").status_code == 200 + assert pw_client.get("/favicon.ico").status_code == 200 + + +def test_login_page_has_no_nav_or_stream(configured): + html = configured.get("/login").data.decode() + assert 'id="navbarNav"' not in html + assert 'data-public="1"' in html + + +def test_navbar_logout_button_when_logged_in(logged_in): + html = logged_in.get("/").data.decode() + assert 'action="/logout"' in html + + +# --- Change password --- + + +def test_change_password_flow(logged_in, pw_app): + other = pw_app.test_client() + other.post("/login", data={"password": PASSWORD}) + assert other.get("/jobs/", headers=HTML).status_code == 200 + + wrong = logged_in.post( + "/account/password", + data={"current_password": "nope", "new_password": "brand new pw", "confirm_password": "brand new pw"}, + follow_redirects=True, + ) + assert b"Current password is incorrect" in wrong.data + + short = logged_in.post( + "/account/password", + data={"current_password": PASSWORD, "new_password": "short", "confirm_password": "short"}, + follow_redirects=True, + ) + assert b"at least 8" in short.data + + ok = logged_in.post( + "/account/password", + data={"current_password": PASSWORD, "new_password": "brand new pw", "confirm_password": "brand new pw"}, + follow_redirects=True, + ) + assert b"Password changed" in ok.data + # This browser stays signed in, others are signed out + assert logged_in.get("/jobs/", headers=HTML).status_code == 200 + assert other.get("/jobs/", headers=HTML).status_code == 302 + + fresh = pw_app.test_client() + assert fresh.post("/login", data={"password": PASSWORD}).status_code == 401 + assert fresh.post("/login", data={"password": "brand new pw"}).status_code == 302 + + +def test_settings_shows_password_section_only_in_password_mode(logged_in, client): + assert b"Change password" in logged_in.get("/settings/").data + assert b"Change password" not in client.get("/settings/").data + + +def test_change_password_requires_login(configured): + response = configured.post("/account/password", data={}) + assert response.status_code == 401 + + +# --- Proxy mode --- + + +@pytest.fixture() +def proxy_app(): + app = _make_app(AUTH_MODE="proxy") + yield app + with app.app_context(): + db.session.remove() + db.drop_all() + + +def test_proxy_mode_requires_header(proxy_app): + client = proxy_app.test_client() + missing = client.get("/jobs/", headers=HTML) + assert missing.status_code == 401 + assert b"Remote-User" in missing.data + assert client.get("/jobs/", headers={**HTML, "Remote-User": "joe"}).status_code == 200 + assert client.get("/jobs/", headers={**HTML, "Remote-User": ""}).status_code == 401 + + +def test_proxy_mode_custom_header(): + app = _make_app(AUTH_MODE="proxy", AUTH_PROXY_HEADER="X-Forwarded-User") + client = app.test_client() + assert client.get("/jobs/").status_code == 401 + assert client.get("/jobs/", headers={"X-Forwarded-User": "joe"}).status_code == 200 + # No login pages in proxy mode + assert client.get("/login").status_code == 302 + + +def test_proxy_mode_statics_reachable(proxy_app): + assert proxy_app.test_client().get("/static/css/style.css").status_code == 200 + + +# --- Config and banner --- + + +@pytest.mark.parametrize("mode", ["", "Password1", "basic", "off"]) +def test_invalid_auth_mode_fails_at_startup(mode): + with pytest.raises(RuntimeError, match="AUTH_MODE"): + _make_app(AUTH_MODE=mode) + + +def test_none_mode_shows_warning_banner(client): + html = client.get("/").data.decode() + assert "Login is turned off" in html + assert b"Log out" not in client.get("/").data + + +def test_desktop_mode_hides_warning_banner(): + app = _make_app(AUTH_MODE="none", DESKTOP_MODE=True) + html = app.test_client().get("/").data.decode() + assert "Login is turned off" not in html + + +def test_password_mode_has_no_warning_banner(logged_in): + assert b"Login is turned off" not in logged_in.get("/").data + + +def test_secure_cookies_setting_is_applied(): + app = _make_app(AUTH_MODE="password", SESSION_COOKIE_SECURE=True) + client = app.test_client() + response = client.post( + "/setup", data={"password": PASSWORD, "confirm": PASSWORD}, base_url="https://localhost" + ) + assert "Secure" in response.headers["Set-Cookie"] + + +# --- SECRET_KEY autogeneration --- + + +def test_secret_key_generated_once_and_reused(tmp_path): + first = _make_app(SECRET_KEY="", INSTANCE_PATH=str(tmp_path)) + key_file = tmp_path / "secret_key" + assert key_file.exists() + key = key_file.read_text().strip() + assert len(key) >= 32 + assert first.config["SECRET_KEY"] == key + + second = _make_app(SECRET_KEY="", INSTANCE_PATH=str(tmp_path)) + assert second.config["SECRET_KEY"] == key + assert key_file.read_text().strip() == key + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX file modes") +def test_secret_key_file_is_private(tmp_path): + _make_app(SECRET_KEY="", INSTANCE_PATH=str(tmp_path)) + mode = stat.S_IMODE((tmp_path / "secret_key").stat().st_mode) + assert mode == 0o600 + + +def test_explicit_secret_key_is_not_written(tmp_path): + app = _make_app(SECRET_KEY="my-own-key", INSTANCE_PATH=str(tmp_path)) + assert app.config["SECRET_KEY"] == "my-own-key" + assert not (tmp_path / "secret_key").exists() + + +def test_placeholder_secret_key_still_rejected(tmp_path): + with pytest.raises(RuntimeError, match="placeholder"): + _make_app(SECRET_KEY="change-me", INSTANCE_PATH=str(tmp_path)) + assert not (tmp_path / "secret_key").exists() diff --git a/tests/test_guards.py b/tests/test_guards.py new file mode 100644 index 0000000..7212bfb --- /dev/null +++ b/tests/test_guards.py @@ -0,0 +1,251 @@ +import json +from datetime import datetime + +import pytest + +from app.extensions import db +from app.models import Job, RunHistory +from app.services.job_runner import JobAlreadyRunning + + +@pytest.fixture() +def events(app): + seen = [] + app.config["EVENTS"].subscribe(lambda name, data: seen.append((name, data))) + return seen + + +def make_job(sync_type="sync", **kwargs): + job = Job( + name="Photos", + source="/mnt/photos", + destination="b2:bucket", + sync_type=sync_type, + **kwargs, + ) + db.session.add(job) + db.session.commit() + return job + + +def starts(fake): + return [c for c in fake.calls if c[0] in ("sync", "copy", "move")] + + +def runner_of(app): + return app.config["JOB_RUNNER"] + + +def test_empty_source_holds_sync(app, fake_rclone, events): + fake_rclone.top_entries = [] + with app.app_context(): + job = make_job() + run = runner_of(app).start_job(job) + assert run.status == "held" + assert run.guard == "source_empty" + assert run.finished_at is not None + assert "empty" in run.error_message + assert starts(fake_rclone) == [] + assert run.id not in runner_of(app)._active_runs + held = [d for n, d in events if n == "run_held"] + assert len(held) == 1 + assert held[0]["guard"] == "source_empty" + assert held[0]["job_name"] == "Photos" + + +def test_unreachable_source_holds_sync(app, fake_rclone, events): + fake_rclone.top_error = RuntimeError("directory not found") + with app.app_context(): + run = runner_of(app).start_job(make_job()) + assert run.status == "held" + assert run.guard == "source_unreachable" + assert "directory not found" in run.error_message + assert starts(fake_rclone) == [] + + +def test_copy_skips_preflight(app, fake_rclone): + fake_rclone.top_entries = [] + with app.app_context(): + run = runner_of(app).start_job(make_job("copy")) + assert run.status == "running" + assert len(starts(fake_rclone)) == 1 + assert fake_rclone.listed == [] + + +def test_max_delete_in_config(app): + with app.app_context(): + assert make_job().to_rclone_config()["MaxDelete"] == 1000 + assert "MaxDelete" not in make_job("copy").to_rclone_config() + assert "MaxDelete" not in make_job(max_delete=-1).to_rclone_config() + assert make_job(max_delete=5).to_rclone_config()["MaxDelete"] == 5 + job = make_job(extra_flags=json.dumps({"MaxDelete": 7})) + assert job.to_rclone_config()["MaxDelete"] == 7 + + +def test_override_route_skips_checks_and_limit(app, client, fake_rclone): + fake_rclone.top_entries = [] + with app.app_context(): + job_id = make_job().id + resp = client.post(f"/ops/start/{job_id}", data={"override": "1"}) + assert resp.status_code == 302 + (kind, kwargs), = starts(fake_rclone) + assert "MaxDelete" not in kwargs["_config"] + with app.app_context(): + run = RunHistory.query.one() + assert run.status == "running" + assert run.triggered_by == "manual_override" + assert "MaxDelete" not in run.get_config_snapshot()["config"] + + +def test_plain_route_flashes_when_held(app, client, fake_rclone): + fake_rclone.top_entries = [] + with app.app_context(): + job_id = make_job().id + resp = client.post(f"/ops/start/{job_id}", follow_redirects=True) + assert b"was not started" in resp.data + assert starts(fake_rclone) == [] + + +def finish_with_error(app, fake_rclone, error, **job_kwargs): + with app.app_context(): + job = make_job(**job_kwargs) + runner = runner_of(app) + run = runner.start_job(job) + fake_rclone.status = {"finished": True, "success": False, "error": error} + runner.check_and_update_runs() + return run.id + + +def test_delete_limit_detected_and_not_retried(app, fake_rclone, events, monkeypatch): + retries = [] + monkeypatch.setattr( + runner_of(app), "_schedule_retry", lambda *a: retries.append(a) + ) + run_id = finish_with_error( + app, + fake_rclone, + "sync failed: --max-delete threshold reached", + auto_resume=True, + ) + with app.app_context(): + run = db.session.get(RunHistory, run_id) + assert run.status == "failed" + assert run.guard == "delete_limit" + assert run.error_message.startswith( + "Stopped after reaching the delete limit of 1000 files." + ) + assert retries == [] + finished = [d for n, d in events if n == "run_finished"] + assert finished[0]["guard"] == "delete_limit" + assert finished[0]["will_retry"] is False + + +def test_run_finished_will_retry(app, fake_rclone, events, monkeypatch): + retries = [] + monkeypatch.setattr( + runner_of(app), "_schedule_retry", lambda *a: retries.append(a) + ) + run_id = finish_with_error(app, fake_rclone, "boom", auto_resume=True) + assert len(retries) == 1 + finished = [d for n, d in events if n == "run_finished"] + assert finished[0]["run_id"] == run_id + assert finished[0]["status"] == "failed" + assert finished[0]["will_retry"] is True + assert finished[0]["guard"] is None + + +def test_run_finished_without_retry_and_completed(app, fake_rclone, events): + with app.app_context(): + runner = runner_of(app) + runner.start_job(make_job()) + fake_rclone.status = {"finished": True, "success": True} + runner.check_and_update_runs() + finished = [d for n, d in events if n == "run_finished"] + assert finished[0]["status"] == "completed" + assert finished[0]["will_retry"] is False + + +def test_stop_job_emits_finished(app, fake_rclone, events): + with app.app_context(): + runner = runner_of(app) + run = runner.start_job(make_job()) + runner.stop_job(run) + finished = [d for n, d in events if n == "run_finished"] + assert finished[0]["status"] == "stopped" + assert finished[0]["will_retry"] is False + + +def test_stats_and_snapshot_persisted(app, fake_rclone): + fake_rclone.stats = {"bytes": 10, "transfers": 2, "deletes": 3, "checks": 9} + with app.app_context(): + runner = runner_of(app) + run = runner.start_job(make_job()) + snapshot = run.get_config_snapshot() + assert snapshot["source"] == "/mnt/photos" + assert snapshot["destination"] == "b2:bucket" + assert snapshot["sync_type"] == "sync" + assert snapshot["config"]["MaxDelete"] == 1000 + assert snapshot["filter"] is None + fake_rclone.status = {"finished": True, "success": True} + runner.check_and_update_runs() + run = db.session.get(RunHistory, run.id) + assert (run.deletes, run.checks) == (3, 9) + + +def test_scheduled_run_skipped_while_running(app, fake_rclone, events): + with app.app_context(): + job = make_job() + first = runner_of(app).start_job(job) + first_id = first.id + job_id = job.id + app.config["SCHEDULER_SERVICE"]._run_scheduled_job(job_id) + skipped = RunHistory.query.filter_by(status="skipped").one() + assert skipped.triggered_by == "schedule" + assert skipped.error_message == f"Skipped: Run #{first_id} was still running." + assert skipped.started_at == skipped.finished_at + assert [d["run_id"] for n, d in events if n == "run_skipped"] == [skipped.id] + + +def test_already_running_carries_run_id(app, fake_rclone): + with app.app_context(): + job = make_job() + first = runner_of(app).start_job(job) + with pytest.raises(JobAlreadyRunning) as exc: + runner_of(app).start_job(job) + assert exc.value.existing_run_id == first.id + + +def test_history_pages_render_held_and_skipped(app, client, fake_rclone): + with app.app_context(): + job = make_job() + now = datetime.utcnow() + held = RunHistory( + job_id=job.id, status="held", guard="source_empty", + started_at=now, finished_at=now, error_message="empty", + ) + skipped = RunHistory( + job_id=job.id, status="skipped", triggered_by="schedule", + started_at=now, finished_at=now, error_message="Skipped", + ) + db.session.add_all([held, skipped]) + db.session.commit() + held_id, skipped_id = held.id, skipped.id + page = client.get("/history/") + assert page.status_code == 200 + assert b"Held" in page.data and b"Skipped" in page.data + assert b"Run anyway" in page.data + assert b"/ops/resume/" not in page.data + for run_id in (held_id, skipped_id): + assert client.get(f"/history/{run_id}").status_code == 200 + assert client.get("/").status_code == 200 + assert b"Run anyway" in client.get(f"/history/{held_id}").data + + +def test_job_form_parses_max_delete(app, client, fake_rclone): + base = {"name": "J", "source": "/a", "destination": "/b", "sync_type": "sync"} + client.post("/jobs/create", data={**base, "max_delete": "-1"}) + client.post("/jobs/create", data={**base, "name": "K", "max_delete": "junk"}) + client.post("/jobs/create", data={**base, "name": "L", "max_delete": "-5"}) + with app.app_context(): + values = {j.name: j.max_delete for j in Job.query} + assert values == {"J": -1, "K": 1000, "L": 1000} diff --git a/tests/test_health.py b/tests/test_health.py new file mode 100644 index 0000000..7ac84ae --- /dev/null +++ b/tests/test_health.py @@ -0,0 +1,421 @@ +from datetime import datetime, timedelta +from zoneinfo import ZoneInfo + +import pytest + +from app.extensions import db +from app.models import Job, RunHistory, Schedule +from app.services import health + +TZ = ZoneInfo("America/New_York") +GRACE = timedelta(minutes=60) +# 2026-03-10 12:00 UTC = 08:00 New York (EDT) +NOW = datetime(2026, 3, 10, 12, 0) + + +def make_job(name="Photos", cron=None, enabled=True, created=None, **kw): + created = created or NOW - timedelta(days=30) + job = Job(name=name, source="/src", destination="b2:dst", created_at=created, **kw) + if cron: + job.schedule = Schedule( + cron_expression=cron, enabled=enabled, created_at=created + ) + return job + + +_ids = iter(range(1, 100000)) + + +def run(status="completed", ago=timedelta(hours=1), **kw): + started = NOW - ago + kw.setdefault("finished_at", started + timedelta(minutes=5)) + return RunHistory(id=next(_ids), status=status, started_at=started, **kw) + + +def ev(job, runs, now=NOW): + return health.evaluate_job(job, runs, now, TZ, GRACE) + + +def test_healthy(): + h = ev(make_job(cron="0 2 * * *"), [run(ago=timedelta(hours=2))]) + assert h.state == "healthy" + assert h.detail.startswith("Last success 1 hour ago") or "ago" in h.detail + assert h.consecutive_failures == 0 + assert h.next_run == datetime(2026, 3, 11, 6, 0) # 02:00 EDT + + +def test_failing_after_failed_run_and_count(): + runs = [ + run(ago=timedelta(days=3)), + run("failed", ago=timedelta(days=2), error_message="boom"), + run("failed", ago=timedelta(days=1), error_message="x" * 300), + ] + h = ev(make_job(), runs) + assert h.state == "failing" + assert h.consecutive_failures == 2 + assert h.detail == "Last run failed: " + "x" * 120 + + +def test_failed_without_message_uses_error_count(): + h = ev(make_job(), [run("failed", errors=7)]) + assert h.detail == "Last run failed: 7 errors" + + +def test_failing_after_held_run_uses_reason(): + runs = [ + run(ago=timedelta(days=2)), + run("held", ago=timedelta(days=1), error_message="Source is empty"), + ] + h = ev(make_job(), runs) + assert h.state == "failing" + assert h.detail == "Source is empty" + + +def test_running_failure_says_retrying(): + runs = [ + run("failed", ago=timedelta(hours=2), error_message="boom"), + run("running", ago=timedelta(minutes=1), finished_at=None), + ] + h = ev(make_job(), runs) + assert h.state == "failing" and h.running + assert h.detail.endswith("(retrying now)") + + +@pytest.mark.parametrize("status", ["stopped", "skipped", "interrupted", "running"]) +def test_ignored_statuses_do_not_count(status): + runs = [ + run(ago=timedelta(days=2)), + run(status, ago=timedelta(days=1)), + ] + h = ev(make_job(), runs) + assert h.state == "healthy" + assert h.consecutive_failures == 0 + + +def test_ignored_status_between_failures(): + runs = [ + run("failed", ago=timedelta(days=3)), + run("stopped", ago=timedelta(days=2)), + run("failed", ago=timedelta(days=1)), + ] + h = ev(make_job(), runs) + assert h.consecutive_failures == 2 + + +def test_overdue_from_cron(): + # Last success two days ago; the 02:00 run since then never succeeded. + runs = [run(ago=timedelta(days=2))] + h = ev(make_job(cron="0 2 * * *"), runs) + assert h.state == "overdue" + assert "No successful run since" in h.detail + assert "one was due by" in h.detail + + +def test_within_grace_not_overdue(): + # Success started 2026-03-09 06:00Z (02:00 EDT); next due 03-10 06:00Z. + started_ago = NOW - datetime(2026, 3, 9, 6, 0) + runs = [run(ago=started_ago)] + job = make_job(cron="0 2 * * *") + assert ev(job, runs, now=datetime(2026, 3, 10, 6, 59)).state == "healthy" + assert ev(job, runs, now=datetime(2026, 3, 10, 7, 1)).state == "overdue" + + +def test_running_job_not_overdue(): + runs = [ + run(ago=timedelta(days=2)), + run("running", ago=timedelta(minutes=30), finished_at=None), + ] + h = ev(make_job(cron="0 2 * * *"), runs) + assert h.state == "healthy" and h.running + + +def test_disabled_schedule_not_overdue(): + h = ev(make_job(cron="0 2 * * *", enabled=False), [run(ago=timedelta(days=9))]) + assert h.state == "healthy" + assert h.next_run is None + + +def test_max_age_hours(): + job = make_job(max_age_hours=24) + assert ev(job, [run(ago=timedelta(hours=30))]).state == "overdue" + assert "every 24 hours" in ev(job, [run(ago=timedelta(hours=30))]).detail + assert ev(job, [run(ago=timedelta(hours=10))]).state == "healthy" + + +def test_max_age_never_succeeded_uses_created_at(): + job = make_job(max_age_hours=24, created=NOW - timedelta(hours=30)) + h = ev(job, []) + assert h.state == "overdue" + assert "since it was created" in h.detail + job = make_job(max_age_hours=24, created=NOW - timedelta(hours=3)) + assert ev(job, []).state == "pending" + + +def test_max_age_overrides_schedule(): + job = make_job(cron="0 2 * * *", max_age_hours=100) + assert ev(job, [run(ago=timedelta(days=2))]).state == "healthy" + + +def test_never_run_scheduled_job_overdue(): + job = make_job(cron="0 2 * * *", created=NOW - timedelta(days=2)) + h = ev(job, []) + assert h.state == "overdue" + assert "since it was created" in h.detail + + +def test_never_run_before_first_due_is_pending(): + job = make_job(cron="0 2 * * *", created=NOW - timedelta(hours=1)) + h = ev(job, []) + assert h.state == "pending" + assert h.detail == "Waiting for its first run" + + +def test_pending_manual_job_not_scheduled(): + h = ev(make_job(), []) + assert h.state == "pending" + assert h.detail == "Not scheduled" + + +def test_manual_job_never_overdue(): + assert ev(make_job(), [run(ago=timedelta(days=300))]).state == "healthy" + + +def history(deletes_latest, previous): + runs = [run(ago=timedelta(hours=1), deletes=deletes_latest)] + for i, d in enumerate(previous): + runs.append(run(ago=timedelta(days=i + 1), deletes=d)) + return runs + + +def test_delete_anomaly_warning(): + h = ev(make_job(), history(5000, [10, 20, 30])) + assert h.state == "warning" + assert h.detail == "Last run deleted 5000 files (usually about 20)." + + +def test_delete_anomaly_needs_history_and_threshold(): + assert ev(make_job(), history(5000, [10, 20])).state == "healthy" + assert ev(make_job(), history(100, [0, 0, 0])).state == "healthy" + assert ev(make_job(), history(900, [100, 100, 100])).state == "healthy" + assert ev(make_job(), history(1500, [100, 100, 100])).state == "warning" + + +def test_warning_clears_with_normal_success(): + runs = history(5000, [10, 20, 30]) + runs.append(run(ago=timedelta(minutes=10), deletes=5)) + assert ev(make_job(), runs).state == "healthy" + + +def test_failing_beats_overdue_and_warning(): + runs = history(5000, [10, 20, 30]) + [run("failed", ago=timedelta(minutes=5))] + assert ev(make_job(), runs).state == "failing" + + +def test_success_rate_7d(): + runs = [ + run(ago=timedelta(days=1)), + run(ago=timedelta(days=2)), + run(ago=timedelta(days=3)), + run("failed", ago=timedelta(days=4)), + run("failed", ago=timedelta(days=20)), # outside the window + run("stopped", ago=timedelta(days=1)), # not counted + ] + assert ev(make_job(), runs).success_rate_7d == 0.75 + assert ev(make_job(), []).success_rate_7d is None + + +def test_report_ordering_and_summary(): + jobs = [] + runs_by_job = {} + specs = { + "zeta-healthy": [run()], + "alpha-healthy": [run()], + "pending": [], + "bfail": [run("failed")], + "afail": [run("failed")], + "overdue": [run(ago=timedelta(hours=100))], + } + for i, (name, runs) in enumerate(specs.items(), start=1): + job = make_job(name=name, max_age_hours=24 if name == "overdue" else None) + job.id = i + jobs.append(job) + runs_by_job[i] = runs + report = health.build_report(jobs, runs_by_job, NOW, TZ, GRACE) + assert [e.name for e in report["jobs"]] == [ + "afail", + "bfail", + "overdue", + "pending", + "alpha-healthy", + "zeta-healthy", + ] + assert report["total"] == 6 + assert report["healthy"] == 2 + assert report["attention"] == 3 + assert report["success_rate_7d"] == pytest.approx(3 / 5) + + +def test_time_ago(): + assert health.time_ago(NOW - timedelta(seconds=5), NOW) == "just now" + assert health.time_ago(NOW - timedelta(minutes=5), NOW) == "5 minutes ago" + assert health.time_ago(NOW - timedelta(hours=1), NOW) == "1 hour ago" + assert health.time_ago(NOW - timedelta(hours=2), NOW) == "2 hours ago" + assert health.time_ago(NOW - timedelta(days=3), NOW) == "3 days ago" + assert health.time_ago(NOW + timedelta(hours=3), NOW) == "in 3 hours" + + +# --- transitions, with the database --- + + +@pytest.fixture() +def events(app): + seen = [] + app.config["EVENTS"].subscribe(lambda name, data: seen.append((name, data))) + return seen + + +def add_job(**kw): + job = Job(name="Photos", source="/s", destination="d:", **kw) + db.session.add(job) + db.session.commit() + return job + + +def add_run(job, status, started, **kw): + r = RunHistory( + job_id=job.id, + status=status, + started_at=started, + finished_at=started + timedelta(minutes=1), + **kw, + ) + db.session.add(r) + db.session.commit() + return r + + +def test_first_evaluation_is_silent(app, events): + with app.app_context(): + job = add_job() + health.evaluate_and_record(app, now=NOW) + assert db.session.get(Job, job.id).health_state == "pending" + assert events == [] + + +def test_change_emits_event_then_quiet(app, events): + with app.app_context(): + job = add_job() + health.evaluate_and_record(app, now=NOW) + add_run(job, "failed", NOW - timedelta(minutes=30), error_message="boom") + health.evaluate_and_record(app, now=NOW) + health.evaluate_and_record(app, now=NOW) + stored = db.session.get(Job, job.id) + assert stored.health_state == "failing" + assert stored.health_changed_at == NOW + job_id = job.id + assert events == [ + ( + "health_changed", + { + "job_id": job_id, + "job_name": "Photos", + "old_state": "pending", + "new_state": "failing", + "detail": "Last run failed: boom", + }, + ) + ] + + +# --- routes --- + + +def test_dashboard_with_no_jobs(client): + resp = client.get("/") + assert resp.status_code == 200 + assert b"Backup health" in resp.data + assert b"No jobs yet" in resp.data + + +def test_dashboard_and_api_with_each_state(app, client): + now = datetime.utcnow() + with app.app_context(): + healthy = add_job() + healthy.name = "ok" + add_run(healthy, "completed", now - timedelta(hours=2)) + failing = Job(name="bad", source="/s", destination="d:") + db.session.add(failing) + stale = Job(name="stale", source="/s", destination="d:", max_age_hours=1) + stale.created_at = now - timedelta(days=2) + db.session.add(stale) + db.session.add(Job(name="new", source="/s", destination="d:")) + warn = Job(name="warn", source="/s", destination="d:") + db.session.add(warn) + db.session.commit() + add_run(failing, "failed", now - timedelta(hours=1), error_message="nope") + for i in range(4): + add_run( + warn, + "completed", + now - timedelta(days=4 - i, hours=3), + deletes=5000 if i == 3 else 1, + ) + resp = client.get("/") + assert resp.status_code == 200 + for text in (b"Healthy", b"Failing", b"Overdue", b"Warning", b"Pending"): + assert text in resp.data + + data = client.get("/api/health").get_json() + assert data["total"] == 5 + assert data["healthy"] == 1 + assert data["attention"] == 3 + assert data["pending"] == 1 + assert data["generated_at"].endswith("Z") + states = {j["name"]: j["state"] for j in data["jobs"]} + assert states == { + "ok": "healthy", + "bad": "failing", + "stale": "overdue", + "new": "pending", + "warn": "warning", + } + first = data["jobs"][0] + for key in ( + "job_id", + "name", + "state", + "detail", + "running", + "consecutive_failures", + "last_success", + "next_run", + "success_rate_7d", + ): + assert key in first + ok = next(j for j in data["jobs"] if j["name"] == "ok") + assert ok["last_success"].endswith("Z") + + +def form(**extra): + data = { + "name": "J", + "source": "/s", + "destination": "d:", + "sync_type": "copy", + } + data.update(extra) + return data + + +def test_job_form_saves_max_age_hours(app, client): + client.post("/jobs/create", data=form(max_age_hours="36")) + with app.app_context(): + job = Job.query.one() + assert job.max_age_hours == 36 + job_id = job.id + assert b'value="36"' in client.get(f"/jobs/{job_id}/edit").data + for blank in ("", "abc", "0", "-5"): + client.post(f"/jobs/{job_id}/edit", data=form(max_age_hours=blank)) + with app.app_context(): + assert db.session.get(Job, job_id).max_age_hours is None + client.post(f"/jobs/{job_id}/edit", data=form(max_age_hours="12")) diff --git a/tests/test_notifier.py b/tests/test_notifier.py new file mode 100644 index 0000000..094f3b0 --- /dev/null +++ b/tests/test_notifier.py @@ -0,0 +1,294 @@ +import pytest + +from app.extensions import db +from app.models import Job, RunError, RunHistory +from app.services import notifier as notifier_mod +from app.services.app_settings import set_setting + + +@pytest.fixture() +def sent(app): + """Notifier with a recording sender and a synchronous spawner.""" + calls = [] + n = app.config["NOTIFIER"] + n.sender = lambda urls, title, body, kind: calls.append((urls, title, body, kind)) or True + n.spawner = lambda fn, *a: fn(*a) + with app.app_context(): + set_setting("notify_urls", ["ntfy://ntfy.sh/x"]) + return calls + + +@pytest.fixture() +def job(app): + with app.app_context(): + j = Job(name="Photos", source="/data/photos", destination="b2:photos") + db.session.add(j) + db.session.commit() + return j.id + + +def emit(app, event, **data): + app.config["EVENTS"].emit(event, **data) + + +def finished(job_id, status="failed", **kw): + base = dict(run_id=7, job_id=job_id, job_name="Photos", status=status, + errors=0, error_message=None, guard=None, will_retry=False) + base.update(kw) + return base + + +def test_failed(app, sent, job): + emit(app, "run_finished", **finished(job, error_message="boom")) + (urls, title, body, kind), = sent + assert title == "Backup failed: Photos" + assert kind == "failure" + assert "/data/photos → b2:photos" in body + assert "boom" in body + + +def test_interrupted_counts_as_failed(app, sent, job): + emit(app, "run_finished", **finished(job, status="interrupted")) + assert sent[0][1] == "Backup failed: Photos" + + +def test_will_retry_and_stopped_ignored(app, sent, job): + emit(app, "run_finished", **finished(job, will_retry=True)) + emit(app, "run_finished", **finished(job, status="stopped")) + assert sent == [] + + +def test_delete_limit_is_held(app, sent, job): + emit(app, "run_finished", **finished(job, guard="delete_limit")) + assert sent[0][1] == "Backup stopped at delete limit: Photos" + assert sent[0][3] == "failure" + + +def test_succeeded_off_by_default_then_on(app, sent, job): + emit(app, "run_finished", **finished(job, status="completed")) + assert sent == [] + with app.app_context(): + set_setting("notify_on", {"succeeded": True}) + emit(app, "run_finished", **finished(job, status="completed")) + assert sent[0][1] == "Backup succeeded: Photos" + assert sent[0][3] == "success" + + +def test_held_and_skipped(app, sent, job): + emit(app, "run_held", run_id=1, job_id=job, job_name="Photos", + guard="source_empty", reason="Source is empty") + emit(app, "run_skipped", run_id=2, job_id=job, job_name="Photos", reason="Still running") + assert sent[0][1] == "Backup held: Photos" + assert sent[1][1] == "Scheduled run skipped: Photos" + assert sent[1][3] == "warning" + assert "Still running" in sent[1][2] + + +@pytest.mark.parametrize( + "old,new,title,kind", + [ + ("healthy", "overdue", "Backup overdue: Photos", "failure"), + ("healthy", "warning", "Backup warning: Photos", "warning"), + ("failing", "healthy", "Backup recovered: Photos", "success"), + ("overdue", "healthy", "Backup recovered: Photos", "success"), + ("warning", "healthy", "Backup recovered: Photos", "success"), + ], +) +def test_health_transitions(app, sent, job, old, new, title, kind): + emit(app, "health_changed", job_id=job, job_name="Photos", + old_state=old, new_state=new, detail="because") + (_, t, body, k), = sent + assert (t, k) == (title, kind) + assert "because" in body + + +@pytest.mark.parametrize( + "old,new", + [("healthy", "failing"), ("overdue", "failing"), ("pending", "healthy"), + ("healthy", "pending"), ("healthy", "healthy")], +) +def test_health_ignored(app, sent, job, old, new): + emit(app, "health_changed", job_id=job, job_name="Photos", + old_state=old, new_state=new, detail="") + assert sent == [] + + +def test_disabled_category(app, sent, job): + with app.app_context(): + set_setting("notify_on", {"failed": False}) + emit(app, "run_finished", **finished(job)) + assert sent == [] + + +def test_no_urls(app, sent, job): + with app.app_context(): + set_setting("notify_urls", []) + emit(app, "run_finished", **finished(job)) + assert sent == [] + + +def test_failure_body_causes_and_link(app, sent, job): + with app.app_context(): + run = RunHistory(job_id=job, status="failed") + db.session.add(run) + db.session.commit() + run_id = run.id + for i in range(3): + db.session.add(RunError(run_id=run_id, message="quota exceeded", + category="quota exceeded", fingerprint=f"a{i}")) + db.session.add(RunError(run_id=run_id, message="denied", + category="permission denied", fingerprint="b")) + db.session.commit() + set_setting("public_url", "http://nas.lan:8080/") + emit(app, "run_finished", **finished(job, run_id=run_id, error_message="x" * 500)) + body = sent[0][2] + assert "3 × quota exceeded" in body + assert "1 × permission denied" in body + assert body.index("quota") < body.index("denied") + assert f"http://nas.lan:8080/history/{run_id}" in body + assert "x" * 400 not in body + + +def test_health_link_is_dashboard(app, sent, job): + with app.app_context(): + set_setting("public_url", "http://nas.lan:8080") + emit(app, "health_changed", job_id=job, job_name="Photos", + old_state="healthy", new_state="overdue", detail="late") + assert sent[0][2].splitlines()[-1] == "http://nas.lan:8080/" + + +def test_public_url_falls_back_to_config(app, sent, job): + app.config["PUBLIC_URL"] = "http://cfg:1" + emit(app, "run_finished", **finished(job)) + assert "http://cfg:1/history/7" in sent[0][2] + + +def test_deleted_job(app, sent): + emit(app, "run_finished", **finished(9999, error_message="gone")) + assert sent[0][1] == "Backup failed: Photos" + assert "→" not in sent[0][2] + + +def test_sender_exception_is_contained(app, job): + n = app.config["NOTIFIER"] + n.spawner = lambda fn, *a: fn(*a) + + def boom(*a): + raise RuntimeError("down") + + n.sender = boom + with app.app_context(): + set_setting("notify_urls", ["ntfy://ntfy.sh/x"]) + emit(app, "run_finished", **finished(job)) + + +# --- settings routes --- + + +def test_save_settings(app, client): + r = client.post("/settings/notifications", data={ + "notify_urls": "ntfy://ntfy.sh/a\n\n discord://1/2 ", + "notify_on": ["failed", "succeeded"], + "public_url": "http://nas.lan:8080/", + "heartbeat_url": "https://hc-ping.com/abc", + "heartbeat_minutes": "10", + }) + assert r.status_code == 302 + with app.app_context(): + assert notifier_mod.get_urls() == ["ntfy://ntfy.sh/a", "discord://1/2"] + on = notifier_mod.get_notify_on() + assert on["succeeded"] and on["failed"] and not on["held"] + assert notifier_mod.get_public_url(app) == "http://nas.lan:8080" + assert notifier_mod.get_heartbeat() == ("https://hc-ping.com/abc", 10) + page = client.get("/settings/").get_data(as_text=True) + assert "ntfy://ntfy.sh/a" in page + + +def test_bad_url_rejected_by_line_number(app, client): + r = client.post("/settings/notifications", data={ + "notify_urls": "ntfy://ntfy.sh/a\nnot a url SECRETTOKEN", + }, follow_redirects=True) + text = r.get_data(as_text=True) + assert "line 2" in text + assert "SECRETTOKEN" not in text + with app.app_context(): + assert notifier_mod.get_urls() == [] + + +def test_heartbeat_must_be_http(app, client): + r = client.post("/settings/notifications", data={ + "heartbeat_url": "ftp://x", "heartbeat_minutes": "5"}, follow_redirects=True) + assert "must start with http" in r.get_data(as_text=True) + with app.app_context(): + assert notifier_mod.get_heartbeat()[0] == "" + + +def test_test_notification(app, client): + calls = [] + app.config["NOTIFIER"].sender = lambda *a: calls.append(a) or True + r = client.post("/settings/notifications/test", + data={"notify_urls": "ntfy://ntfy.sh/a"}, follow_redirects=True) + assert "Test notification sent" in r.get_data(as_text=True) + assert calls[0][0] == ["ntfy://ntfy.sh/a"] + + app.config["NOTIFIER"].sender = lambda *a: False + r = client.post("/settings/notifications/test", + data={"notify_urls": "ntfy://ntfy.sh/a"}, follow_redirects=True) + assert "could not be delivered" in r.get_data(as_text=True) + + +def test_test_notification_without_urls(client): + r = client.post("/settings/notifications/test", follow_redirects=True) + assert "Add at least one" in r.get_data(as_text=True) + + +# --- heartbeat --- + + +@pytest.fixture() +def pings(app, monkeypatch, fake_rclone): + calls = [] + + class Resp: + def raise_for_status(self): + pass + + monkeypatch.setattr(notifier_mod.requests, "get", + lambda url, timeout: calls.append(url) or Resp()) + with app.app_context(): + set_setting("heartbeat_url", "https://hc-ping.com/abc") + set_setting("heartbeat_minutes", 5) + return calls + + +def test_heartbeat_due_and_interval(app, pings): + n = app.config["NOTIFIER"] + assert n.heartbeat_tick(now=1000) + assert not n.heartbeat_tick(now=1000 + 299) + assert n.heartbeat_tick(now=1000 + 300) + assert len(pings) == 2 + + +def test_heartbeat_not_sent_when_rclone_down(app, pings, fake_rclone): + def down(): + raise ConnectionError("no rclone") + + fake_rclone.noop = down + assert not app.config["NOTIFIER"].heartbeat_tick(now=1000) + assert pings == [] + + +def test_heartbeat_no_url(app, monkeypatch, fake_rclone): + monkeypatch.setattr(notifier_mod.requests, "get", + lambda *a, **k: pytest.fail("should not ping")) + assert not app.config["NOTIFIER"].heartbeat_tick(now=1000) + + +def test_heartbeat_failure_retries_next_tick(app, pings, monkeypatch): + def fail(url, timeout): + raise OSError("net") + + monkeypatch.setattr(notifier_mod.requests, "get", fail) + n = app.config["NOTIFIER"] + assert not n.heartbeat_tick(now=1000) + assert n._last_ping is None diff --git a/tests/test_schedules.py b/tests/test_schedules.py index 97a3b78..3a99b28 100644 --- a/tests/test_schedules.py +++ b/tests/test_schedules.py @@ -111,7 +111,9 @@ def test_scheduled_run_records_baseline_and_skips_overlap(app, client, fake_rclo assert len(fake_rclone.calls) == 1 with app.app_context(): - assert RunHistory.query.count() == 1 + # The overlap leaves a "skipped" row but never a second rclone job. + assert RunHistory.query.filter_by(status="running").count() == 1 + assert RunHistory.query.filter_by(status="skipped").count() == 1 assert Schedule.query.one().catch_up_after is not None diff --git a/tests/test_startup.py b/tests/test_startup.py index ec489af..46aada1 100644 --- a/tests/test_startup.py +++ b/tests/test_startup.py @@ -16,9 +16,9 @@ def _config(**overrides): @pytest.mark.parametrize( - "key", ["", "change-me", "change-me-in-production", "dev-secret-key-change-me"] + "key", ["change-me", "change-me-in-production", "dev-secret-key-change-me"] ) -def test_refuses_to_start_without_real_secret_key(key): +def test_refuses_placeholder_secret_key(key): with pytest.raises(RuntimeError, match="SECRET_KEY"): create_app(_config(SECRET_KEY=key))