From 4f18c08be5d39405cb638b54ef4008336a5f9e13 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 4 Aug 2026 16:21:13 +0200 Subject: [PATCH 1/2] Add privacy-conscious course analytics --- docs/analytics.md | 91 +++++++++++ docs/architecture.md | 4 +- migrations/011_course_analytics.sql | 67 ++++++++ src/bin/server.rs | 230 +++++++++++++++++++++++++++- static/js/analytics.js | 83 ++++++++++ static/js/inline-editor.js | 4 + templates/dashboard.html | 18 ++- templates/exercise.html | 29 +++- templates/tour.html | 18 ++- 9 files changed, 537 insertions(+), 7 deletions(-) create mode 100644 docs/analytics.md create mode 100644 migrations/011_course_analytics.sql create mode 100644 static/js/analytics.js diff --git a/docs/analytics.md b/docs/analytics.md new file mode 100644 index 0000000..5fa1316 --- /dev/null +++ b/docs/analytics.md @@ -0,0 +1,91 @@ +# Course analytics + +The course records a small set of first-party events in SQLite's +`course_events` table. The goal is to answer where learners stop or ask for +help without collecting source code, names, URLs, user agents, or arbitrary +client metadata. + +## Events + +| Event | Recorded when | Result fields | +|---|---|---| +| `chapter_view` | An exercise chapter loads | — | +| `editor_focus` | An editor first receives focus in a browser-tab session | — | +| `hint_opened` | A hint disclosure is first opened | — | +| `solution_revealed` | A full solution is first opened | — | +| `next_chapter_clicked` | The next-chapter CTA is clicked | — | +| `exercise_run` | The server receives a Rust Playground response | result, tests passed/total, duration, first structured Rust error code | + +UI events are deduplicated per `(session_id, event_type, exercise_name)`. Runs +are never deduplicated because repeated runs are the primary difficulty signal. +A session ID is a random UUID kept in `sessionStorage`, so it expires with the +browser tab. `participant_id` is nullable for anonymous learners. + +Every row includes `course_version` and `git_hash`, allowing reports to avoid +mixing results from incompatible course revisions. + +## Example queries + +Exercises with the most repeated unsuccessful runs: + +```sql +SELECT + exercise_name, + COUNT(*) AS runs, + COUNT(DISTINCT COALESCE(participant_id, session_id)) AS learners, + ROUND(AVG(result != 'passed') * 100, 1) AS unsuccessful_pct, + ROUND(AVG(duration_ms)) AS average_duration_ms +FROM course_events +WHERE event_type = 'exercise_run' +GROUP BY exercise_name +HAVING learners >= 3 +ORDER BY unsuccessful_pct DESC, runs DESC; +``` + +Hint and solution usage: + +```sql +SELECT + exercise_name, + SUM(event_type = 'hint_opened') AS hint_opens, + SUM(event_type = 'solution_revealed') AS solution_reveals +FROM course_events +WHERE event_type IN ('hint_opened', 'solution_revealed') +GROUP BY exercise_name +ORDER BY solution_reveals DESC, hint_opens DESC; +``` + +Most common structured compiler errors: + +```sql +SELECT diagnostic_code, COUNT(*) AS occurrences, + COUNT(DISTINCT COALESCE(participant_id, session_id)) AS learners +FROM course_events +WHERE event_type = 'exercise_run' AND diagnostic_code IS NOT NULL +GROUP BY diagnostic_code +ORDER BY occurrences DESC; +``` + +Chapter activation from view to editor use: + +```sql +WITH events_by_chapter AS ( + SELECT *, CASE + WHEN instr(exercise_name, '/') > 0 + THEN substr(exercise_name, 1, instr(exercise_name, '/') - 1) + ELSE exercise_name + END AS chapter + FROM course_events +) +SELECT + chapter, + COUNT(DISTINCT CASE WHEN event_type = 'chapter_view' THEN session_id END) AS views, + COUNT(DISTINCT CASE WHEN event_type = 'editor_focus' THEN session_id END) AS editors +FROM events_by_chapter +WHERE event_type IN ('chapter_view', 'editor_focus') +GROUP BY chapter; +``` + +For reports, prefer a read-only SQLite backup rather than querying the live file. +SQLite's backup API or `.backup` command produces a consistent snapshot without +interrupting the server. diff --git a/docs/architecture.md b/docs/architecture.md index 7fda5b9..eff8965 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -14,7 +14,8 @@ Two things in one crate: 2. **A small Axum server + CLI** that hosts the same exercises in a browser and tracks per-participant progress against a SQLite database. Optional for self-study, required for instructor-led - workshops. + workshops. Privacy-conscious learning events are stored separately in + `course_events`; see `docs/analytics.md`. The Cargo package is `cargo-course` (Rust edition 2024). It exposes a library plus two binaries (`server`, `cargo-course` aka the CLI). @@ -39,6 +40,7 @@ course/ │ ├── 4_.rs # another step │ └── 5_hints.md # optional; slug `hints` is special (see below) ├── migrations/ # SQLx migrations, applied in order at startup +├── docs/analytics.md # Event schema, privacy boundaries, report queries ├── src/ │ ├── lib.rs # re-exports `exercises` and `types` │ ├── types.rs # API request/response + newtype wrappers diff --git a/migrations/011_course_analytics.sql b/migrations/011_course_analytics.sql new file mode 100644 index 0000000..74a41f3 --- /dev/null +++ b/migrations/011_course_analytics.sql @@ -0,0 +1,67 @@ +-- Privacy-conscious course analytics. +-- +-- Run outcomes are written by the server after it receives the Rust Playground +-- response. UI events use a small server-side allowlist. No source code, +-- participant names, URLs, user agents, or arbitrary metadata are stored here. +CREATE TABLE course_events ( + id TEXT PRIMARY KEY, + participant_id TEXT, + session_id TEXT NOT NULL CHECK(length(session_id) BETWEEN 1 AND 64), + event_type TEXT NOT NULL CHECK(event_type IN ( + 'chapter_view', + 'editor_focus', + 'hint_opened', + 'solution_revealed', + 'next_chapter_clicked', + 'exercise_run' + )), + exercise_name TEXT, + result TEXT CHECK(result IS NULL OR result IN ( + 'passed', + 'test_failed', + 'compile_failed', + 'upstream_failed' + )), + tests_passed INTEGER, + tests_total INTEGER, + duration_ms INTEGER CHECK(duration_ms IS NULL OR duration_ms >= 0), + diagnostic_code TEXT, + course_version TEXT NOT NULL, + git_hash TEXT NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (participant_id) REFERENCES participants(id) ON DELETE CASCADE +); + +CREATE INDEX idx_course_events_created_at ON course_events(created_at); +CREATE INDEX idx_course_events_participant ON course_events(participant_id, created_at); +CREATE INDEX idx_course_events_exercise ON course_events(exercise_name, event_type, created_at); +CREATE INDEX idx_course_events_session ON course_events(session_id, created_at); +CREATE UNIQUE INDEX idx_course_events_ui_once_per_session + ON course_events(session_id, event_type, exercise_name) + WHERE event_type != 'exercise_run'; + +-- Removed or renamed exercises deliberately left behind by migration 010. +-- They no longer exist in the current catalog and otherwise pollute progress +-- analysis. Keep the cleanup explicit so no valid current key can be removed by +-- a broad numeric-prefix match. +DELETE FROM submissions WHERE exercise_name IN ( + '00_integers/3_number_to_string', + '00_integers/4_calculate_total_with_tax', + '03_functions/4_countdown', + '06_vectors/2_count_items', + '09_option/2_fallback', + '16_word_frequencies/4_frequent_words', + '17_password_validator/4_char_classes', + '17_password_validator/6_advisor', + '19_modules_and_visibility/4_settings', + '19_modules_and_visibility/4_status', + '20_environment_file_parser/2_parse_line', + '20_environment_file_parser/3_parse_file', + '20_environment_file_parser/4_get_var', + '20_environment_file_parser/5_validate', + '21_csv_parser/3_simple_line', + '21_csv_parser/4_quoted_line', + '21_csv_parser/5_parse_file', + '21_csv_parser/6_records', + '3_display_name.md' +); diff --git a/src/bin/server.rs b/src/bin/server.rs index 467c42e..e133551 100644 --- a/src/bin/server.rs +++ b/src/bin/server.rs @@ -88,6 +88,47 @@ struct AppState { exercises: Arc>, } +/// A privacy-conscious analytics row. Deliberately excludes source code, +/// participant names, URLs, user agents, and free-form client metadata. +struct CourseEvent<'a> { + participant_id: Option<&'a str>, + session_id: &'a str, + event_type: &'a str, + exercise_name: Option<&'a str>, + result: Option<&'a str>, + tests_passed: Option, + tests_total: Option, + duration_ms: Option, + diagnostic_code: Option<&'a str>, +} + +async fn store_course_event(pool: &SqlitePool, event: CourseEvent<'_>) -> Result<(), sqlx::Error> { + sqlx::query( + r" + INSERT OR IGNORE INTO course_events ( + id, participant_id, session_id, event_type, exercise_name, result, + tests_passed, tests_total, duration_ms, diagnostic_code, + course_version, git_hash + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ", + ) + .bind(Ulid::new().to_string()) + .bind(event.participant_id) + .bind(event.session_id) + .bind(event.event_type) + .bind(event.exercise_name) + .bind(event.result) + .bind(event.tests_passed) + .bind(event.tests_total) + .bind(event.duration_ms) + .bind(event.diagnostic_code) + .bind(COURSE_VERSION) + .bind(git_hash()) + .execute(pool) + .await?; + Ok(()) +} + /// Database model for participants. Only the fields we actually read /// in Rust live here; the SQL queries below select exactly these columns /// so `sqlx::FromRow` stays in lockstep. @@ -893,6 +934,7 @@ async fn main() -> Result<()> { .route("/register", post(api_register)) .route("/submit", post(api_submit)) .route("/status/{ulid}", get(api_status)) + .route("/events", post(api_course_event)) .route("/run", post(api_run)) .route("/format", post(api_format)) .with_state(app_state.clone()); @@ -2693,6 +2735,90 @@ async fn api_status( } } +#[derive(Deserialize)] +struct CourseEventRequest { + #[serde(default)] + participant_id: Option, + session_id: String, + event_type: String, + #[serde(default)] + exercise_name: Option, +} + +const UI_EVENT_TYPES: [&str; 5] = [ + "chapter_view", + "editor_focus", + "hint_opened", + "solution_revealed", + "next_chapter_clicked", +]; + +async fn api_course_event( + State(state): State, + Json(request): Json, +) -> StatusCode { + if !UI_EVENT_TYPES.contains(&request.event_type.as_str()) + || !valid_analytics_identifier(&request.session_id, 64) + || request + .exercise_name + .as_deref() + .is_some_and(|name| !valid_exercise_name(name)) + { + return StatusCode::BAD_REQUEST; + } + + if let Some(participant_id) = request.participant_id.as_deref() { + match participant_exists(&state.pool, participant_id).await { + Ok(true) => {} + Ok(false) => return StatusCode::UNAUTHORIZED, + Err(error) => { + error!("Failed to validate analytics participant: {error}"); + return StatusCode::INTERNAL_SERVER_ERROR; + } + } + } + + let event = CourseEvent { + participant_id: request.participant_id.as_deref(), + session_id: &request.session_id, + event_type: &request.event_type, + exercise_name: request.exercise_name.as_deref(), + result: None, + tests_passed: None, + tests_total: None, + duration_ms: None, + diagnostic_code: None, + }; + if let Err(error) = store_course_event(&state.pool, event).await { + error!("Failed to store course event: {error}"); + return StatusCode::INTERNAL_SERVER_ERROR; + } + StatusCode::NO_CONTENT +} + +fn valid_analytics_identifier(value: &str, max_length: usize) -> bool { + !value.is_empty() + && value.len() <= max_length + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) +} + +fn valid_exercise_name(value: &str) -> bool { + value.len() <= 128 + && !value.is_empty() + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'/')) +} + +async fn participant_exists(pool: &SqlitePool, participant_id: &str) -> Result { + sqlx::query_scalar::<_, bool>("SELECT EXISTS(SELECT 1 FROM participants WHERE id = ?)") + .bind(participant_id) + .fetch_one(pool) + .await +} + /// Request body for `/api/run`. We accept any source code; the slug is /// optional and only used for logging. `tests` defaults to `true` so /// exercise editors continue to compile with `--tests` (which surfaces @@ -2705,6 +2831,10 @@ struct RunRequest { code: String, #[serde(default)] slug: Option, + #[serde(default)] + participant_id: Option, + #[serde(default)] + session_id: Option, #[serde(default = "default_tests")] tests: bool, } @@ -2740,7 +2870,11 @@ struct PlaygroundResp { stderr: String, } -async fn api_run(Json(req): Json) -> Result, StatusCode> { +async fn api_run( + State(state): State, + Json(req): Json, +) -> Result, StatusCode> { + let started_at = std::time::Instant::now(); let slug = req.slug.as_deref().unwrap_or(""); info!("/api/run: forwarding {} bytes for {slug}", req.code.len()); @@ -2797,6 +2931,8 @@ async fn api_run(Json(req): Json) -> Result, Statu parsed.success ); + record_run_event(&state.pool, &req, &parsed, &test_results, started_at).await; + Ok(Json(RunResponse { success: parsed.success, stdout: parsed.stdout, @@ -2805,6 +2941,73 @@ async fn api_run(Json(req): Json) -> Result, Statu })) } +async fn record_run_event( + pool: &SqlitePool, + request: &RunRequest, + response: &PlaygroundResp, + test_results: &[TestResult], + started_at: std::time::Instant, +) { + let Some(session_id) = request + .session_id + .as_deref() + .filter(|id| valid_analytics_identifier(id, 64)) + else { + return; + }; + let participant_id = if let Some(id) = request.participant_id.as_deref() { + match participant_exists(pool, id).await { + Ok(true) => Some(id), + Ok(false) => None, + Err(error) => { + warn!("Could not validate run analytics participant: {error}"); + None + } + } + } else { + None + }; + let tests_passed_count = + i64::try_from(test_results.iter().filter(|test| test.passed).count()).unwrap_or(i64::MAX); + let tests_total = i64::try_from(test_results.len()).unwrap_or(i64::MAX); + let result = if tests_total > 0 && tests_passed_count < tests_total { + "test_failed" + } else if !response.success { + "compile_failed" + } else { + "passed" + }; + let diagnostic = first_rust_error_code(&response.stderr); + let duration_ms = i64::try_from(started_at.elapsed().as_millis()).unwrap_or(i64::MAX); + let event = CourseEvent { + participant_id, + session_id, + event_type: "exercise_run", + exercise_name: request + .slug + .as_deref() + .filter(|name| valid_exercise_name(name)), + result: Some(result), + tests_passed: Some(tests_passed_count), + tests_total: Some(tests_total), + duration_ms: Some(duration_ms), + diagnostic_code: diagnostic.as_deref(), + }; + if let Err(error) = store_course_event(pool, event).await { + // Analytics is best-effort and must never prevent a learner from + // receiving their compiler/test result. + warn!("Failed to store run analytics: {error}"); + } +} + +fn first_rust_error_code(stderr: &str) -> Option { + let marker = "error[E"; + let start = stderr.find(marker)? + "error[".len(); + let code = stderr.get(start..start + 5)?; + (code.starts_with('E') && code[1..].bytes().all(|byte| byte.is_ascii_digit())) + .then(|| code.to_string()) +} + /// Request body for `/api/format`. Same request as `/api/run` minus the /// fields the formatter doesn't care about. #[derive(Deserialize)] @@ -3181,6 +3384,31 @@ mod tests { assert!(bucket_participants_by_team(Vec::new()).is_empty()); } + #[test] + fn analytics_identifiers_are_narrowly_validated() { + assert!(valid_analytics_identifier("session-123_ABC", 64)); + assert!(!valid_analytics_identifier("", 64)); + assert!(!valid_analytics_identifier("session with spaces", 64)); + assert!(!valid_analytics_identifier("../session", 64)); + assert!(!valid_analytics_identifier(&"x".repeat(65), 64)); + + assert!(valid_exercise_name("11_option/4_find_user")); + assert!(valid_exercise_name("tour")); + assert!(!valid_exercise_name("")); + assert!(!valid_exercise_name("chapter?participant=secret")); + assert!(!valid_exercise_name("../chapter")); + } + + #[test] + fn extracts_only_structured_rust_error_codes() { + assert_eq!( + first_rust_error_code("error[E0308]: mismatched types"), + Some("E0308".to_string()) + ); + assert_eq!(first_rust_error_code("error: expected expression"), None); + assert_eq!(first_rust_error_code("all tests passed"), None); + } + #[test] fn usable_filters_out_uninformative_git_values() { // Real values pass through, trimmed. diff --git a/static/js/analytics.js b/static/js/analytics.js new file mode 100644 index 0000000..f194404 --- /dev/null +++ b/static/js/analytics.js @@ -0,0 +1,83 @@ +// Small, privacy-conscious course analytics client. Events contain only a +// generated browser-tab session ID, the participant ID already present in the +// course URL (when signed in), an allowlisted event name, and an exercise key. +// Source code, names, URLs, user agents, and arbitrary metadata are never sent. + +const SESSION_KEY = "corrode:analytics-session"; + +export function analyticsSessionId() { + try { + let id = sessionStorage.getItem(SESSION_KEY); + if (!id) { + id = crypto.randomUUID(); + sessionStorage.setItem(SESSION_KEY, id); + } + return id; + } catch (_) { + return crypto.randomUUID(); + } +} + +export function participantId(root = document) { + return root.querySelector("[data-participant-id]")?.dataset.participantId || null; +} + +export function trackCourseEvent(eventType, exerciseName = null) { + const body = JSON.stringify({ + participant_id: participantId(), + session_id: analyticsSessionId(), + event_type: eventType, + exercise_name: exerciseName, + }); + + // UI analytics must never delay navigation or interrupt the course. sendBeacon + // is reliable during page unload; fetch is the fallback for older browsers. + if (navigator.sendBeacon) { + const blob = new Blob([body], { type: "application/json" }); + if (navigator.sendBeacon("/api/events", blob)) return; + } + fetch("/api/events", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body, + keepalive: true, + }).catch(() => {}); +} + +export function bindCourseAnalytics(root = document) { + const container = root.querySelector("[data-course-chapter]"); + if (!container) return; + + const chapter = container.dataset.courseChapter; + trackCourseEvent("chapter_view", chapter); + + root.querySelectorAll(".exercise-section[data-exercise-key]").forEach((section) => { + let focused = false; + section.addEventListener("focusin", () => { + if (focused) return; + focused = true; + trackCourseEvent("editor_focus", section.dataset.exerciseKey); + }); + }); + + root.querySelectorAll(".hints-disclosure").forEach((details) => { + let tracked = false; + details.addEventListener("toggle", () => { + if (!details.open || tracked) return; + tracked = true; + const key = details.dataset.exerciseKey || chapter; + trackCourseEvent( + details.classList.contains("solution-disclosure") + ? "solution_revealed" + : "hint_opened", + key, + ); + }); + }); + + root.querySelectorAll(".next-chapter-cta a, a.next-chapter-cta").forEach((link) => { + link.addEventListener("click", () => { + trackCourseEvent("next_chapter_clicked", chapter); + }); + }); +} diff --git a/static/js/inline-editor.js b/static/js/inline-editor.js index 083c137..41a8671 100644 --- a/static/js/inline-editor.js +++ b/static/js/inline-editor.js @@ -876,6 +876,10 @@ export async function mountInlineEditor(section, opts = {}) { clearActionStatus(); try { const payload = { code, slug: exerciseKey }; + if (features.analytics) { + payload.participant_id = features.analytics.participantId; + payload.session_id = features.analytics.sessionId; + } if (runWithoutTests) payload.tests = false; const resp = await fetch("/api/run", { method: "POST", diff --git a/templates/dashboard.html b/templates/dashboard.html index aaca904..42c280b 100644 --- a/templates/dashboard.html +++ b/templates/dashboard.html @@ -1,6 +1,10 @@ {% extends "base.html" %} {% block title %}A Beginner's Guide To Rust | corrode{% endblock %} {% block topbar %}{% endblock %} {% block content %} -
+
{% match ulid %} {% when Some with (u) %}