diff --git a/app/__init__.py b/app/__init__.py index 0c38ff4..d18c2f4 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -111,7 +111,16 @@ def _check_runs(): with app.app_context(): job_runner.check_and_update_runs() + def _collect_errors(): + with app.app_context(): + job_runner.collect_active_errors() + monitor.add_job(_check_runs, "interval", seconds=10) + # rclone only remembers its last ~100 completed transfers, so failures + # must be picked up quickly. Overlap is harmless: entries are de-duplicated. + monitor.add_job( + _collect_errors, "interval", seconds=3, max_instances=1, coalesce=True + ) if app.config["BACKGROUND_TASKS_ENABLED"]: monitor.start() @@ -119,6 +128,7 @@ def _check_runs(): from app.routes.history import history_bp from app.routes.jobs import jobs_bp from app.routes.operations import ops_bp + from app.routes.run_errors import run_errors_bp from app.routes.schedules import schedules_bp from app.routes.settings import settings_bp from app.routes.sse import sse_bp @@ -127,6 +137,7 @@ def _check_runs(): app.register_blueprint(ops_bp, url_prefix="/ops") app.register_blueprint(schedules_bp, url_prefix="/schedules") app.register_blueprint(history_bp, url_prefix="/history") + app.register_blueprint(run_errors_bp, url_prefix="/runs") app.register_blueprint(sse_bp) app.register_blueprint(settings_bp, url_prefix="/settings") diff --git a/app/models.py b/app/models.py index e3b5b56..a5722c8 100644 --- a/app/models.py +++ b/app/models.py @@ -120,3 +120,48 @@ class RunHistory(db.Model): retry_of = db.relationship( "RunHistory", remote_side="RunHistory.id", uselist=False ) + # (``errors`` is already the int count of failures rclone reported.) + error_log = db.relationship( + "RunError", backref="run", lazy="dynamic", cascade="all,delete-orphan" + ) + + +class RunError(db.Model): + """One failed transfer or check captured while a run was in progress. + + rclone's RC API only keeps the most recent ~100 completed transfers, so the + runner copies failures here as they happen. Runs can fail thousands of + files, hence the indexes for grouping and cursor paging. + """ + + __table_args__ = ( + db.UniqueConstraint("run_id", "fingerprint", name="uq_run_error"), + db.Index("ix_run_error_run_category", "run_id", "category"), + ) + + id = db.Column(db.Integer, primary_key=True) + run_id = db.Column( + db.Integer, db.ForeignKey("run_history.id"), nullable=False, index=True + ) + occurred_at = db.Column(db.DateTime, default=datetime.utcnow) + path = db.Column(db.String(1000), nullable=True) + kind = db.Column(db.String(10), default="transfer") # transfer | check + size = db.Column(db.BigInteger, nullable=True) + message = db.Column(db.Text, nullable=False) + # ``message`` with ids/urls/quoted names stripped, so thousands of failures + # with the same cause collapse into one group. + category = db.Column(db.String(200), nullable=False) + fingerprint = db.Column(db.String(40), nullable=False) + + def to_dict(self): + return { + "id": self.id, + "occurred_at": ( + self.occurred_at.isoformat() + "Z" if self.occurred_at else None + ), + "path": self.path, + "kind": self.kind, + "size": self.size, + "message": self.message, + "category": self.category, + } diff --git a/app/rclone_client.py b/app/rclone_client.py index c408a5a..7b4f1c8 100644 --- a/app/rclone_client.py +++ b/app/rclone_client.py @@ -72,8 +72,12 @@ def get_stats(self, group=None): params["group"] = group return self._call("core/stats", params) - def get_transferred(self): - return self._call("core/transferred") + def get_transferred(self, group=None): + """Recently completed transfers (rclone keeps only the last ~100).""" + params = {} + if group: + params["group"] = group + return self._call("core/transferred", params) def set_bwlimit(self, rate): return self._call("core/bwlimit", {"rate": rate}) diff --git a/app/routes/history.py b/app/routes/history.py index 9d2cd59..841b9fb 100644 --- a/app/routes/history.py +++ b/app/routes/history.py @@ -1,6 +1,6 @@ from flask import Blueprint, render_template -from app.models import RunHistory +from app.models import RunError, RunHistory history_bp = Blueprint("history", __name__) @@ -18,4 +18,7 @@ def list_history(): @history_bp.route("/") def detail(run_id): run = RunHistory.query.get_or_404(run_id) - return render_template("history/detail.html", run=run) + error_count = RunError.query.filter_by(run_id=run.id).count() + return render_template( + "history/detail.html", run=run, error_count=error_count + ) diff --git a/app/routes/run_errors.py b/app/routes/run_errors.py new file mode 100644 index 0000000..95c789d --- /dev/null +++ b/app/routes/run_errors.py @@ -0,0 +1,148 @@ +import csv +import io + +from flask import Blueprint, Response, jsonify, request, stream_with_context +from sqlalchemy import func, or_ + +from app.extensions import db +from app.models import RunError, RunHistory + +run_errors_bp = Blueprint("run_errors", __name__) + +DEFAULT_LIMIT = 100 +MAX_LIMIT = 500 + + +def _filtered(run_id): + """Errors of a run narrowed by the ?category= and ?q= query parameters.""" + query = RunError.query.filter(RunError.run_id == run_id) + category = request.args.get("category") + if category: + query = query.filter(RunError.category == category) + needle = (request.args.get("q") or "").strip() + if needle: + escaped = ( + needle.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + ) + pattern = f"%{escaped}%" + query = query.filter( + or_( + RunError.path.ilike(pattern, escape="\\"), + RunError.message.ilike(pattern, escape="\\"), + ) + ) + return query + + +def _int_arg(name, default=None): + try: + return int(request.args.get(name, "")) + except ValueError: + return default + + +@run_errors_bp.route("//errors") +def list_errors(run_id): + """Failures of a run as JSON, newest first, paged by id cursors. + + ``?after=`` returns what failed since that id (for the live feed) and + ``?before=`` the next older page. Cursors rather than offsets keep + paging cheap and stable while new failures keep arriving. With + ``?summary=1`` the reply also groups the run's failures by cause. + """ + run = RunHistory.query.get_or_404(run_id) + limit = max(1, min(_int_arg("limit", DEFAULT_LIMIT), MAX_LIMIT)) + after = _int_arg("after") + before = _int_arg("before") + + query = _filtered(run.id) + matched = query.count() if request.args.get("count") else None + if after is not None: + query = query.filter(RunError.id > after) + if before is not None: + query = query.filter(RunError.id < before) + + rows = query.order_by(RunError.id.desc()).limit(limit + 1).all() + has_more = len(rows) > limit + rows = rows[:limit] + + reply = { + "errors": [row.to_dict() for row in rows], + "has_more": has_more, + "total": RunError.query.filter_by(run_id=run.id).count(), + "last_id": db.session.query(func.max(RunError.id)) + .filter(RunError.run_id == run.id) + .scalar() + or 0, + "status": run.status, + } + if matched is not None: + reply["matched"] = matched + if request.args.get("summary"): + reply["categories"] = _categories(run.id) + return jsonify(reply) + + +def _categories(run_id): + groups = ( + db.session.query( + RunError.category, + func.count(RunError.id), + func.min(RunError.occurred_at), + func.max(RunError.occurred_at), + ) + .filter(RunError.run_id == run_id) + .group_by(RunError.category) + .order_by(func.count(RunError.id).desc(), RunError.category) + .limit(100) + .all() + ) + return [ + { + "category": category, + "count": count, + "first": first.isoformat() + "Z" if first else None, + "last": last.isoformat() + "Z" if last else None, + } + for category, count, first, last in groups + ] + + +def _spreadsheet_safe(value): + """Stop file names like '=cmd|...' from being run as formulas when opened.""" + text = "" if value is None else str(value) + return "'" + text if text[:1] in ("=", "+", "-", "@", "\t", "\r") else text + + +@run_errors_bp.route("//errors.csv") +def export_errors(run_id): + """Every stored failure of a run (honouring filters) as a CSV download.""" + run = RunHistory.query.get_or_404(run_id) + query = _filtered(run.id).order_by(RunError.id) + + def generate(): + buffer = io.StringIO() + writer = csv.writer(buffer) + writer.writerow(["time_utc", "kind", "path", "size", "message"]) + yield buffer.getvalue() + for row in query.yield_per(500): + buffer.seek(0) + buffer.truncate() + writer.writerow( + [ + row.occurred_at.isoformat() if row.occurred_at else "", + row.kind, + _spreadsheet_safe(row.path), + row.size if row.size is not None else "", + _spreadsheet_safe(row.message), + ] + ) + yield buffer.getvalue() + + return Response( + stream_with_context(generate()), + mimetype="text/csv", + headers={ + "Content-Disposition": f'attachment; filename="run-{run.id}-errors.csv"' + }, + ) diff --git a/app/services/job_runner.py b/app/services/job_runner.py index b807b5c..d8440f7 100644 --- a/app/services/job_runner.py +++ b/app/services/job_runner.py @@ -3,6 +3,7 @@ from app.extensions import db from app.models import Job, RunHistory +from app.services.run_errors import record_failures logger = logging.getLogger(__name__) @@ -70,6 +71,7 @@ def stop_job(self, run): pass # Capture final stats before marking stopped + self.collect_errors(run) self._save_final_stats(run) run.status = "stopped" @@ -93,12 +95,13 @@ def _parse_rclone_time(self, time_str): def _save_final_stats(self, run): """Fetch per-group stats from rclone and persist them on the run.""" if not run.stats_group: - return + return None try: stats = self.rclone.get_stats(group=run.stats_group) run.bytes_transferred = stats.get("bytes", 0) run.files_transferred = stats.get("transfers", 0) run.errors = stats.get("errors", 0) + return stats except Exception as exc: logger.warning( "Failed to fetch final stats for Run #%d (group=%s): %s", @@ -106,6 +109,31 @@ def _save_final_stats(self, run): run.stats_group, exc, ) + return None + + def collect_errors(self, run): + """Copy failed transfers/checks of a run out of rclone into the DB.""" + if not run.stats_group: + return 0 + try: + reply = self.rclone.get_transferred(group=run.stats_group) + return record_failures(run, (reply or {}).get("transferred")) + except Exception as exc: + db.session.rollback() + logger.warning( + "Failed to collect errors for Run #%d (group=%s): %s", + run.id, + run.stats_group, + exc, + ) + return 0 + + def collect_active_errors(self): + """Poll every tracked run; called often since rclone forgets old entries.""" + for run_id in list(self._active_runs): + run = db.session.get(RunHistory, run_id) + if run: + self.collect_errors(run) def check_and_update_runs(self): for run_id, rclone_jobid in list(self._active_runs.items()): @@ -131,7 +159,18 @@ def check_and_update_runs(self): if status.get("error"): run.error_message = str(status["error"]) - self._save_final_stats(run) + self.collect_errors(run) + stats = self._save_final_stats(run) + # Failures not tied to a file (listing, deleting, ...) only show + # up as rclone's last error; keep it if nothing else explains + # the failure. + if ( + not success + and not run.error_message + and stats + and stats.get("lastError") + ): + run.error_message = str(stats["lastError"]) db.session.commit() del self._active_runs[run_id] diff --git a/app/services/run_errors.py b/app/services/run_errors.py new file mode 100644 index 0000000..968dbc2 --- /dev/null +++ b/app/services/run_errors.py @@ -0,0 +1,131 @@ +"""Capturing per-file failures from rclone so they can be inspected later. + +rclone's RC API reports an error *count* and the single most recent error, and +its list of completed transfers (the only place per-file errors appear) is +capped at roughly the last 100. Runs can fail thousands of files, so failures +are copied into the database while the run is in progress. +""" + +import hashlib +import logging +import posixpath +import re +from datetime import datetime + +from sqlalchemy.exc import IntegrityError + +from app.extensions import db +from app.models import RunError + +logger = logging.getLogger(__name__) + +# Stop storing past this many failures per run; the count rclone reports and +# the grouped summary still tell the story, and the table can't grow forever. +MAX_ERRORS_PER_RUN = 50_000 + +_URL = re.compile(r"https?://\S+") +_IPV4 = re.compile(r"\b\d{1,3}(?:\.\d{1,3}){3}(?::\d+)?\b") +_PATH = re.compile(r"(?= 3: + for needle in {path, posixpath.basename(path.replace("\\", "/"))}: + if len(needle) >= 3: + text = text.replace(needle, "") + text = _URL.sub("", text) + text = _IPV4.sub("", text) + text = _QUOTED.sub("'…'", text) + text = _PATH.sub("", text) + text = _LONG_HEX.sub("", text) + text = _LONG_NUMBER.sub("", text) + text = _SPACES.sub(" ", text).strip() + return (text or "Unknown error")[:200] + + +def _fingerprint(path, stamp, message): + raw = f"{path or ''}\0{stamp or ''}\0{message}" + return hashlib.sha1(raw.encode("utf-8", "replace")).hexdigest() + + +def _parse_time(value): + if not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).replace( + tzinfo=None + ) + except (ValueError, TypeError): + return None + + +def record_failures(run, transferred): + """Store failed entries of an rclone ``core/transferred`` reply. + + Safe to call repeatedly with overlapping data: entries already stored for + the run are recognised by fingerprint and skipped. Returns how many new + rows were added. + """ + candidates = {} + for entry in transferred or []: + message = str(entry.get("error") or "").strip() + if not message: + continue + path = entry.get("name") or None + stamp = entry.get("completed_at") or entry.get("started_at") + candidates[_fingerprint(path, stamp, message)] = (entry, path, message) + if not candidates: + return 0 + + seen = { + fp + for (fp,) in db.session.query(RunError.fingerprint).filter( + RunError.run_id == run.id, + RunError.fingerprint.in_(list(candidates)), + ) + } + fresh = [(fp, c) for fp, c in candidates.items() if fp not in seen] + if not fresh: + return 0 + + room = MAX_ERRORS_PER_RUN - RunError.query.filter_by(run_id=run.id).count() + if room <= 0: + return 0 + + # Oldest first so ids (the live feed's cursor) follow the order they failed. + fresh.sort(key=lambda item: item[1][0].get("completed_at") or "") + for fp, (entry, path, message) in fresh[:room]: + db.session.add( + RunError( + run_id=run.id, + occurred_at=_parse_time( + entry.get("completed_at") or entry.get("started_at") + ) + or datetime.utcnow(), + path=path[:1000] if path else None, + kind="check" if entry.get("checked") else "transfer", + size=entry.get("size"), + message=message, + category=categorize(message, path), + fingerprint=fp, + ) + ) + try: + db.session.commit() + except IntegrityError: + # Another writer stored some of these first; they're recorded either way. + db.session.rollback() + return 0 + return min(len(fresh), room) diff --git a/app/static/css/style.css b/app/static/css/style.css index 57f185b..1111bad 100644 --- a/app/static/css/style.css +++ b/app/static/css/style.css @@ -51,6 +51,40 @@ code { vertical-align: top; } +/* Failure lists (errors.js) */ +.run-errors .re-scroll { + max-height: 320px; + overflow-y: auto; +} + +.run-errors .re-row { + cursor: pointer; +} + +.run-errors .re-msg { + max-width: 0; /* lets the cell truncate instead of widening the table */ + width: 60%; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.run-errors .re-row.expanded .re-msg { + white-space: normal; + overflow-wrap: anywhere; +} + +.run-errors .re-path { + width: 35%; +} + +.run-errors .re-categories { + display: flex; + flex-wrap: wrap; + gap: 0.4rem; + align-items: center; +} + /* The tray app's window focuses the first link whenever it reopens. */ .navbar-brand:focus-visible { outline: none; diff --git a/app/static/js/errors.js b/app/static/js/errors.js new file mode 100644 index 0000000..4bfdbaa --- /dev/null +++ b/app/static/js/errors.js @@ -0,0 +1,220 @@ +// Failure lists for runs (see templates/partials/run_errors.html). +// +// Runs can fail thousands of files, so nothing here loads them all: the server +// groups failures by cause, and rows are fetched a page at a time using id +// cursors. The live feed on the Running page keeps only the newest rows. +// File names and messages come from remote systems and are untrusted: always +// build nodes with textContent only. + +function makeEl(tag, className, text) { + const node = document.createElement(tag); + if (className) node.className = className; + if (text !== undefined) node.textContent = text; + return node; +} + +class RunErrors { + constructor(root) { + this.root = root; + this.runId = root.dataset.runId; + this.live = root.dataset.live === '1'; + this.maxRows = parseInt(root.dataset.maxRows, 10) || 0; // 0 = no limit + this.pageSize = parseInt(root.dataset.pageSize, 10) || 100; + this.category = ''; + this.query = ''; + this.lastId = 0; // newest failure shown, for the live feed + this.oldestId = 0; // oldest failure shown, for "Load older" + this.total = 0; + this.generation = 0; // bumped on filter changes to drop stale replies + this.busy = false; + + this.rows = root.querySelector('.re-rows'); + this.moreBtn = root.querySelector('.re-more'); + this.moreBtn.addEventListener('click', () => this.loadOlder()); + + let timer = null; + root.querySelector('.re-search').addEventListener('input', event => { + clearTimeout(timer); + timer = setTimeout(() => { + this.query = event.target.value.trim(); + this.reset(); + }, 300); + }); + + if (this.live) { + window.addEventListener('rclone-stats', event => this.onStats(event.detail)); + } else { + this.reset(); + } + } + + // --- Live feed --------------------------------------------------------- + + onStats(data) { + const run = (data.runs || {})[this.runId]; + if (!run || !run.stats) return; + const reported = run.stats.errors || 0; + const latest = this.root.querySelector('.re-latest'); + // rclone's own latest error shows up before the stored list catches up. + if (run.stats.lastError) { + latest.textContent = 'Latest: ' + run.stats.lastError; + latest.classList.remove('d-none'); + } + if (reported > 0 || this.total > 0) { + this.root.classList.remove('d-none'); + this.poll(); + } + } + + async poll() { + if (this.busy) return; + if (this.lastId === 0 && this.rows.children.length === 0) { + return this.reset(); + } + const generation = this.generation; + this.busy = true; + try { + const params = { + after: this.lastId, limit: this.maxRows || this.pageSize, summary: 1, + }; + if (this.category || this.query) params.count = 1; + const data = await this.fetchErrors(params); + if (generation !== this.generation) return; + if (data.errors.length > 0) { + // Replies are newest first, so this keeps the feed in order. + this.rows.prepend(...data.errors.map(e => this.renderRow(e))); + this.lastId = data.errors[0].id; + while (this.maxRows && this.rows.children.length > this.maxRows) { + this.rows.lastElementChild.remove(); + } + this.oldestId = parseInt(this.rows.lastElementChild.dataset.id, 10); + } + this.applySummary(data); + } catch (e) { + // Transient; the next stats event retries. + } finally { + this.busy = false; + } + } + + // --- Browsing ---------------------------------------------------------- + + async reset() { + this.generation++; + const generation = this.generation; + this.busy = true; + try { + const data = await this.fetchErrors({ + limit: this.pageSize, summary: 1, count: 1, + }); + if (generation !== this.generation) return; + this.rows.replaceChildren(...data.errors.map(e => this.renderRow(e))); + this.lastId = data.errors.length ? data.errors[0].id : data.last_id; + this.oldestId = data.errors.length ? data.errors[data.errors.length - 1].id : 0; + this.moreBtn.classList.toggle('d-none', !data.has_more); + this.applySummary(data); + } catch (e) { + this.root.querySelector('.re-status').textContent = 'Could not load failures.'; + } finally { + if (generation === this.generation) this.busy = false; + } + } + + async loadOlder() { + if (this.busy || !this.oldestId) return; + const generation = this.generation; + this.busy = true; + this.moreBtn.disabled = true; + try { + const data = await this.fetchErrors({ before: this.oldestId, limit: this.pageSize }); + if (generation !== this.generation) return; + this.rows.append(...data.errors.map(e => this.renderRow(e))); + if (data.errors.length) this.oldestId = data.errors[data.errors.length - 1].id; + this.moreBtn.classList.toggle('d-none', !data.has_more); + this.applySummary(data); + } catch (e) { + this.root.querySelector('.re-status').textContent = 'Could not load more.'; + } finally { + this.moreBtn.disabled = false; + this.busy = false; + } + } + + async fetchErrors(params) { + const search = new URLSearchParams(params); + if (this.category) search.set('category', this.category); + if (this.query) search.set('q', this.query); + const response = await fetch(`/runs/${this.runId}/errors?${search}`); + if (!response.ok) throw new Error(response.status); + return response.json(); + } + + // --- Rendering --------------------------------------------------------- + + applySummary(data) { + this.total = data.total; + if (data.matched !== undefined) this.matched = data.matched; + this.root.querySelector('.re-total').textContent = data.total.toLocaleString(); + if (data.total > 0) this.root.classList.remove('d-none'); + if (data.categories) this.renderCategories(data.categories); + + const filtered = this.category || this.query; + const shown = this.rows.children.length; + const of = filtered && this.matched !== undefined ? this.matched : data.total; + let text = `Showing ${shown.toLocaleString()} of ${of.toLocaleString()}`; + if (filtered) text += ' matching'; + if (this.live && of > shown) text += ' (newest first)'; + this.root.querySelector('.re-status').textContent = shown ? text : ( + filtered ? 'No failures match.' : '' + ); + + const csv = new URLSearchParams(); + if (this.category) csv.set('category', this.category); + if (this.query) csv.set('q', this.query); + const base = `/runs/${this.runId}/errors.csv`; + this.root.querySelector('.re-csv').href = csv.toString() ? `${base}?${csv}` : base; + } + + renderCategories(categories) { + const box = this.root.querySelector('.re-categories'); + const limit = this.live ? 5 : categories.length; + const items = categories.slice(0, limit).map(c => { + const button = makeEl('button', 'btn btn-sm re-cat d-flex align-items-baseline gap-2 text-start'); + button.type = 'button'; + button.classList.add(c.category === this.category ? 'btn-danger' : 'btn-outline-danger'); + button.append( + makeEl('span', 'badge bg-dark text-nowrap', c.count.toLocaleString()), + makeEl('span', 'text-break', c.category), + ); + button.addEventListener('click', () => { + this.category = this.category === c.category ? '' : c.category; + this.reset(); + }); + return button; + }); + if (categories.length > limit) { + items.push(makeEl('small', 'text-muted', `+${categories.length - limit} more causes`)); + } + box.replaceChildren(...items); + } + + renderRow(error) { + const tr = makeEl('tr', 're-row'); + tr.dataset.id = error.id; + const time = error.occurred_at ? new Date(error.occurred_at) : null; + const when = makeEl('td', 'text-muted text-nowrap small', + time ? time.toLocaleTimeString() : ''); + const path = makeEl('td', 'font-monospace small re-path text-break', + error.path || '(no file)'); + path.title = error.path || ''; + const message = makeEl('td', 'small re-msg', error.message); + tr.append(when, path, message); + // Messages can be long; click a row to read all of it. + tr.addEventListener('click', () => tr.classList.toggle('expanded')); + return tr; + } +} + +document.addEventListener('DOMContentLoaded', () => { + document.querySelectorAll('.run-errors').forEach(root => new RunErrors(root)); +}); diff --git a/app/templates/history/detail.html b/app/templates/history/detail.html index 023ea59..18072ec 100644 --- a/app/templates/history/detail.html +++ b/app/templates/history/detail.html @@ -1,4 +1,5 @@ {% extends "base.html" %} +{% from "partials/run_errors.html" import run_errors %} {% block title %}Run #{{ run.id }} - rClone GUI{% endblock %} {% block content %} @@ -125,6 +126,14 @@
{{ run.errors or 0 }}
{% endif %} + {% if run.errors or error_count %} +
+
+ {{ run_errors(run) }} +
+
+ {% endif %} + {% if run.job %}
Job Configuration
@@ -144,3 +153,7 @@
{{ run.errors or 0 }}
{% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/app/templates/operations/running.html b/app/templates/operations/running.html index fa3245e..74b1a6b 100644 --- a/app/templates/operations/running.html +++ b/app/templates/operations/running.html @@ -1,4 +1,5 @@ {% extends "base.html" %} +{% from "partials/run_errors.html" import run_errors %} {% block title %}Running Operations - rClone GUI{% endblock %} {% block content %} @@ -64,6 +65,8 @@
{{ run.job.name if run.job else 'Unknown Job' }}
Currently transferring:
+ + {{ run_errors(run, live=True) }} {% endfor %} @@ -81,4 +84,5 @@
{{ run.job.name if run.job else 'Unknown Job' }}
{% block scripts %} + {% endblock %} diff --git a/app/templates/partials/run_errors.html b/app/templates/partials/run_errors.html new file mode 100644 index 0000000..9518f48 --- /dev/null +++ b/app/templates/partials/run_errors.html @@ -0,0 +1,40 @@ +{# Failure list for a run, driven by static/js/errors.js. + live=True: hidden Running-page feed that shows the newest failures as they happen. + live=False: full browser (paging, filter, export) for a run's detail page. #} +{% macro run_errors(run, live=False) %} +
+
+
+ Failures + 0 +
+
+ + + CSV + + {% if live %} + + View all + + {% endif %} +
+
+
+
+
+ + +
+
+
+ + +
+
+{% endmacro %} diff --git a/tests/test_frontend_safety.py b/tests/test_frontend_safety.py index 0688aea..0a29aae 100644 --- a/tests/test_frontend_safety.py +++ b/tests/test_frontend_safety.py @@ -18,3 +18,9 @@ def test_no_native_confirm_dialogs(): app_dir = STATIC_JS.parent.parent for path in [*app_dir.glob("templates/**/*.html"), *STATIC_JS.glob("*.js")]: assert not re.search(r"\bconfirm\(", path.read_text(encoding="utf-8")), path + + +def test_failure_list_never_injects_remote_text_via_innerhtml(): + """File names and error messages come from remote systems.""" + source = (STATIC_JS / "errors.js").read_text(encoding="utf-8") + assert not re.search(r"innerHTML|insertAdjacentHTML|outerHTML", source) diff --git a/tests/test_run_errors.py b/tests/test_run_errors.py new file mode 100644 index 0000000..3e63b09 --- /dev/null +++ b/tests/test_run_errors.py @@ -0,0 +1,200 @@ +import csv +import io + +import pytest + +from app.extensions import db +from app.models import Job, RunError, RunHistory +from app.services.run_errors import MAX_ERRORS_PER_RUN, categorize + + +def failure(name, error="googleapi: Error 403: Rate Limit Exceeded", at="12:00:00"): + return { + "name": name, + "size": 10, + "error": error, + "checked": False, + "started_at": f"2025-01-01T{at}.000Z", + "completed_at": f"2025-01-01T{at}.500Z", + } + + +@pytest.fixture() +def run(app, fake_rclone): + with app.app_context(): + job = Job(name="Photos", source="/src", destination="remote:dst") + db.session.add(job) + db.session.flush() + run = RunHistory(job_id=job.id, stats_group="job_1", rclone_jobid=42) + db.session.add(run) + db.session.commit() + fake_rclone.transferred = [] + fake_rclone.get_transferred = lambda group=None: { + "transferred": fake_rclone.transferred + } + yield run + + +def collect(app): + with app.app_context(): + runner = app.config["JOB_RUNNER"] + return runner.collect_errors(db.session.get(RunHistory, 1)) + + +def test_categorize_groups_causes_but_keeps_status_codes(): + a = categorize("open /mnt/photos/a/IMG_1.jpg: permission denied", "a/IMG_1.jpg") + b = categorize("open /mnt/photos/b/IMG_2.jpg: permission denied", "b/IMG_2.jpg") + assert a == b == "open : permission denied" + assert categorize("Error 403: rate limit") != categorize("Error 429: rate limit") + assert categorize("dial tcp 10.0.0.5:443: i/o timeout") == "dial tcp : i/o timeout" + assert categorize("failed for 'secret.txt' id 1a2b3c4d5e6f7a8b") == "failed for '…' id " + assert categorize("") == "Unknown error" + + +def test_only_failures_are_stored_and_repeat_polls_dont_duplicate(app, fake_rclone, run): + fake_rclone.transferred = [ + failure("a.jpg"), + {"name": "ok.jpg", "size": 1, "error": "", "completed_at": "2025-01-01T12:00:01.000Z"}, + ] + assert collect(app) == 1 + assert collect(app) == 0 + fake_rclone.transferred.append(failure("b.jpg", at="12:00:02")) + assert collect(app) == 1 + with app.app_context(): + assert sorted(e.path for e in RunError.query) == ["a.jpg", "b.jpg"] + + +def test_same_file_failing_again_is_a_new_entry(app, fake_rclone, run): + fake_rclone.transferred = [failure("a.jpg", at="12:00:00"), failure("a.jpg", at="12:05:00")] + assert collect(app) == 2 + + +def test_failures_survive_rclone_forgetting_them(app, fake_rclone, run): + fake_rclone.transferred = [failure("a.jpg")] + collect(app) + fake_rclone.transferred = [] + collect(app) + with app.app_context(): + assert RunError.query.count() == 1 + + +def test_collection_stops_at_the_cap(app, fake_rclone, run, monkeypatch): + monkeypatch.setattr("app.services.run_errors.MAX_ERRORS_PER_RUN", 2) + fake_rclone.transferred = [failure(f"{i}.jpg", at=f"12:00:0{i}") for i in range(4)] + assert collect(app) == 2 + assert collect(app) == 0 + assert MAX_ERRORS_PER_RUN == 50_000 + + +def test_rclone_errors_during_collection_are_swallowed(app, fake_rclone, run): + def boom(group=None): + raise RuntimeError("rclone is down") + + fake_rclone.get_transferred = boom + assert collect(app) == 0 + + +def seed(app, count, categories=("403", "denied")): + with app.app_context(): + for i in range(count): + kind = categories[i % len(categories)] + db.session.add( + RunError( + run_id=1, + path=f"dir/file{i}.bin", + message=f"{kind} for file{i}.bin", + category=kind, + fingerprint=f"fp{i}", + ) + ) + db.session.commit() + + +def test_listing_is_newest_first_and_paged_by_cursor(app, client, run): + seed(app, 25) + page = client.get("/runs/1/errors?limit=10&summary=1").get_json() + ids = [e["id"] for e in page["errors"]] + assert ids == sorted(ids, reverse=True) and len(ids) == 10 + assert page["total"] == 25 and page["has_more"] and page["last_id"] == ids[0] + assert {c["category"]: c["count"] for c in page["categories"]} == {"403": 13, "denied": 12} + + older = client.get(f"/runs/1/errors?limit=100&before={ids[-1]}").get_json() + assert len(older["errors"]) == 15 and not older["has_more"] + + newer = client.get(f"/runs/1/errors?after={ids[2]}").get_json() + assert [e["id"] for e in newer["errors"]] == ids[:2] + + +def test_listing_filters_by_category_and_text(app, client, run): + seed(app, 20) + by_category = client.get("/runs/1/errors?category=denied&count=1&limit=500").get_json() + assert by_category["matched"] == 10 + assert all(e["category"] == "denied" for e in by_category["errors"]) + + by_text = client.get("/runs/1/errors?q=file7.bin").get_json() + assert [e["path"] for e in by_text["errors"]] == ["dir/file7.bin"] + + +def test_text_filter_treats_wildcards_literally(app, client, run): + seed(app, 5) + assert client.get("/runs/1/errors?q=%25").get_json()["errors"] == [] + assert client.get("/runs/1/errors?q=_").get_json()["errors"] == [] + + +def test_limit_is_clamped_and_bad_numbers_ignored(app, client, run): + seed(app, 3) + assert len(client.get("/runs/1/errors?limit=999999").get_json()["errors"]) == 3 + assert len(client.get("/runs/1/errors?limit=abc&after=x").get_json()["errors"]) == 3 + assert client.get("/runs/99/errors").status_code == 404 + + +def test_csv_export_neutralises_formulas(app, client, run): + with app.app_context(): + db.session.add( + RunError( + run_id=1, + path='=HYPERLINK("http://x")', + message='boom, "quoted"', + category="boom", + fingerprint="x", + ) + ) + db.session.commit() + resp = client.get("/runs/1/errors.csv") + assert resp.mimetype == "text/csv" + rows = list(csv.reader(io.StringIO(resp.data.decode()))) + assert rows[0] == ["time_utc", "kind", "path", "size", "message"] + assert rows[1][2].startswith("'=") + assert rows[1][4] == 'boom, "quoted"' + + +def test_finishing_a_run_captures_late_failures_and_last_error(app, fake_rclone, run): + fake_rclone.transferred = [failure("late.jpg")] + fake_rclone.job_status = lambda jobid: {"finished": True, "success": False} + fake_rclone.get_stats = lambda group=None: { + "errors": 1, + "bytes": 5, + "transfers": 0, + "lastError": "couldn't list directory", + } + with app.app_context(): + runner = app.config["JOB_RUNNER"] + runner._active_runs[1] = 42 + runner.check_and_update_runs() + finished = db.session.get(RunHistory, 1) + assert finished.status == "failed" + assert finished.error_message == "couldn't list directory" + assert RunError.query.one().path == "late.jpg" + + +def test_deleting_a_job_removes_its_failures(app, client, run): + seed(app, 3) + client.post("/jobs/1/delete") + with app.app_context(): + assert RunError.query.count() == 0 + + +def test_run_page_shows_failure_browser_only_when_there_are_failures(app, client, run): + assert b"run-errors" not in client.get("/history/1").data + seed(app, 1) + assert b"run-errors" in client.get("/history/1").data