feat(audio): analyse the mixed envelope, not the raw source

volume, volume_keyframes and the fades are what comes out of the speakers, and
since #182 the studio plays exactly that — a waveform drawing the raw file's
envelope while a fade takes the sound down contradicts what the viewer hears.

The gain comes from the encoder's own envelope, factored out of the mixer loop
as track_gain_at(track, t_in_track, audible) and expressed in seconds so the
mixer (OUTPUT_SAMPLE_RATE, resampled) and the analysis (the file's own rate,
decoded source) share one implementation. A second copy would drift, and the
whole point is that the picture matches the sound.

The cache fingerprint now hashes the serialised track rather than start/end
alone: two scenarios can name the same file with different mixes, and adding a
field to AudioTrack must not silently leave it out of the key.

Not mergeable yet — see #201. The components read scene-local time against a
scenario-time analysis, which is harmless while the envelope is flat and turns
into a flat trace as soon as it is not.

Author: LeadcodeDev

crates/rustmotion/src/encode/audio.rs

@@ -221,9 +221,6 @@ pub fn mix_audio_tracks_segment(
             .unwrap_or(scenario_samples)
             .min(scenario_samples);
 
-        let fade_in_samples = track.fade_in.unwrap_or(0.0) * TARGET_SAMPLE_RATE as f64;
-        let fade_out_samples = track.fade_out.unwrap_or(0.0) * TARGET_SAMPLE_RATE as f64;
-
         // How much of the track is ever audible in the scenario, regardless
         // of which segment we are materializing right now. Fades are
         // computed against this, not against the segment's own bounds.
@@ -251,25 +248,12 @@ pub fn mix_audio_tracks_segment(
             }
 
             let frame = i / TARGET_CHANNELS as usize;
-            let current_time = track.start + (frame as f64 / TARGET_SAMPLE_RATE as f64);
-            let vol = if !track.volume_keyframes.is_empty() {
-                interpolate_volume_keyframes(&track.volume_keyframes, current_time)
-            } else {
-                track.volume
-            };
-            let mut sample = src_sample * vol;
-
-            // Apply fade in
-            if fade_in_samples > 0.0 && (frame as f64) < fade_in_samples {
-                sample *= frame as f32 / fade_in_samples as f32;
-            }
-
-            // Apply fade out — against the track's own total audible
-            // frames in the scenario, not this segment's length.
-            let frames_from_end = total_frames - frame;
-            if fade_out_samples > 0.0 && (frames_from_end as f64) < fade_out_samples {
-                sample *= frames_from_end as f32 / fade_out_samples as f32;
-            }
+            let sample = src_sample
+                * track_gain_at(
+                    track,
+                    frame as f64 / TARGET_SAMPLE_RATE as f64,
+                    total_frames as f64 / TARGET_SAMPLE_RATE as f64,
+                );
 
             mix_buffer[dst_idx] += sample;
         }
@@ -286,6 +270,40 @@ pub fn mix_audio_tracks_segment(
     Ok(Some(pcm_bytes))
 }
 
+/// The gain applied to a track `t_in_track` seconds after its own first sample,
+/// given that `audible` seconds of it are ever heard.
+///
+/// Expressed in seconds rather than sample indices so the mixer (which works at
+/// `OUTPUT_SAMPLE_RATE` on resampled audio) and the analysis (which works at the
+/// file's own rate on the decoded source) can share it. They must: a waveform
+/// that draws an envelope the mix does not produce is the component lying about
+/// the very track it claims to react to.
+pub(crate) fn track_gain_at(
+    track: &crate::schema::AudioTrack,
+    t_in_track: f64,
+    audible: f64,
+) -> f32 {
+    let mut gain = if track.volume_keyframes.is_empty() {
+        track.volume
+    } else {
+        // Keyframe times are on the *scenario* timeline, not the track's.
+        interpolate_volume_keyframes(&track.volume_keyframes, track.start + t_in_track)
+    };
+
+    if let Some(fade_in) = track.fade_in {
+        if fade_in > 0.0 && t_in_track < fade_in {
+            gain *= (t_in_track / fade_in) as f32;
+        }
+    }
+    if let Some(fade_out) = track.fade_out {
+        let remaining = audible - t_in_track;
+        if fade_out > 0.0 && remaining < fade_out {
+            gain *= (remaining.max(0.0) / fade_out) as f32;
+        }
+    }
+    gain
+}
+
 /// Interpolate volume at a given time using volume keyframes with easing
 fn interpolate_volume_keyframes(keyframes: &[crate::schema::VolumeKeyframe], time: f64) -> f32 {
     if keyframes.is_empty() {

crates/rustmotion/src/encode/audio_analysis.rs

@@ -31,10 +31,12 @@ impl std::fmt::Display for AudioAnalysisFailure {
 /// up that way), so without this a track whose *content* changed under a stable
 /// path — the normal case when someone re-exports a mix while the studio is
 /// open — would keep serving the old envelope forever.
-/// Also carries the track's placement: the analysis content depends only
-/// on the file, but the *lookup* now applies `start`/`end`, so an entry
-/// computed for one placement must not be reused for another.
-type SourceFingerprint = (u64, u128, u32, u64, u64);
+/// File identity (length, mtime), the fps it was bucketed at, and a hash of
+/// everything about the *track* that changes the result: `start`/`end` move the
+/// lookup, and `volume`/`volume_keyframes`/the fades are baked into the
+/// amplitudes. An entry computed for one of those must never be served for
+/// another — two scenarios can name the same file with different mixes.
+type SourceFingerprint = (u64, u128, u32, u64);
 
 static FINGERPRINTS: OnceLock<Mutex<HashMap<String, SourceFingerprint>>> = OnceLock::new();
 
@@ -48,8 +50,7 @@ fn fingerprints() -> &'static Mutex<HashMap<String, SourceFingerprint>> {
 fn source_fingerprint(
     src: &str,
     fps: u32,
-    start: f64,
-    end: Option<f64>,
+    track: &rustmotion_core::schema::AudioTrack,
 ) -> Option<SourceFingerprint> {
     let meta = std::fs::metadata(src).ok()?;
     let mtime = meta
@@ -58,13 +59,19 @@ fn source_fingerprint(
         .duration_since(std::time::UNIX_EPOCH)
         .ok()?
         .as_nanos();
-    Some((
-        meta.len(),
-        mtime,
-        fps,
-        start.to_bits(),
-        end.unwrap_or(f64::INFINITY).to_bits(),
-    ))
+    Some((meta.len(), mtime, fps, track_hash(track)))
+}
+
+/// Hash the track's placement and volume envelope. Serialised rather than
+/// hashed field by field so adding a field to `AudioTrack` cannot silently
+/// leave it out of the key.
+fn track_hash(track: &rustmotion_core::schema::AudioTrack) -> u64 {
+    use std::hash::{Hash, Hasher};
+    let mut hasher = std::collections::hash_map::DefaultHasher::new();
+    serde_json::to_string(track)
+        .unwrap_or_default()
+        .hash(&mut hasher);
+    hasher.finish()
 }
 
 /// Build the 16 log-spaced band frequency boundaries (Hz) from 20..16000.
@@ -107,7 +114,7 @@ pub fn analyze_scenario_audio(scenario: &ResolvedScenario) -> Vec<AudioAnalysisF
 
     for track in tracks {
         let src = &track.src;
-        let fingerprint = source_fingerprint(src, fps, track.start, track.end);
+        let fingerprint = source_fingerprint(src, fps, track);
         let cached_and_current = cache.contains_key(src)
             && fingerprint.is_some()
             && fps_of
@@ -141,6 +148,28 @@ pub fn analyze_scenario_audio(scenario: &ResolvedScenario) -> Vec<AudioAnalysisF
                 .collect(),
         };
 
+        // Follow the *mix*, not the source. `volume`, `volume_keyframes` and the
+        // fades are what comes out of the speakers, and since #182 the studio
+        // plays exactly that — a waveform drawing the raw file's envelope while
+        // a fade takes the sound down contradicts what the viewer hears. The
+        // gain comes from the encoder's own `track_gain_at`, so the picture
+        // cannot drift from the audio.
+        let audible = {
+            let file_seconds = mono.len() as f64 / sample_rate as f64;
+            match track.end {
+                Some(end) => file_seconds.min((end - track.start).max(0.0)),
+                None => file_seconds,
+            }
+        };
+        let mono: Vec<f32> = mono
+            .into_iter()
+            .enumerate()
+            .map(|(i, s)| {
+                let t = i as f64 / sample_rate as f64;
+                s * crate::encode::audio::track_gain_at(track, t, audible)
+            })
+            .collect();
+
         let samples_per_frame = (sample_rate as f64 / fps as f64).ceil() as usize;
         let num_frames = (mono.len() as f64 / samples_per_frame as f64).ceil() as usize;
 

crates/rustmotion/src/tests.rs

@@ -1984,6 +1984,98 @@ mod audio_tests {
         count
     }
 
+    /// The analysis must describe the **mix**, not the source.
+    ///
+    /// `volume`, `volume_keyframes` and the fades are what comes out of the
+    /// speakers, and since #182 the studio plays exactly that. A waveform
+    /// drawing the raw file's envelope while a keyframe takes the sound to
+    /// zero contradicts what the viewer hears.
+    #[test]
+    fn the_analysis_follows_the_mixed_envelope_not_the_raw_file() {
+        let sample_rate = 44100u32;
+        // 2 s of unbroken sine: any variation in the analysis comes from the
+        // envelope, never from the source.
+        let wav_path = std::env::temp_dir().join(format!("rustmotion_test_mix_{}.wav", nanos()));
+        std::fs::write(
+            &wav_path,
+            make_sine_wav(sample_rate * 2, sample_rate * 2, 440.0, sample_rate),
+        )
+        .expect("write fixture");
+        let wav_str = wav_path.to_str().unwrap().to_string();
+
+        // Full for the first second, silent for the second.
+        let json = serde_json::json!({
+            "video": {"width": 32, "height": 32, "fps": 30},
+            "audio": [{
+                "src": wav_str,
+                "volume_keyframes": [
+                    {"time": 0.0, "volume": 1.0},
+                    {"time": 1.0, "volume": 1.0},
+                    {"time": 1.05, "volume": 0.0},
+                    {"time": 2.0, "volume": 0.0}
+                ]
+            }],
+            "scenes": [{"duration": 2.0, "children": []}]
+        })
+        .to_string();
+        let scenario =
+            crate::loader::load_scenario_from_source(None, Some(&json)).expect("load scenario");
+        assert!(crate::encode::audio_analysis::analyze_scenario_audio(&scenario).is_empty());
+
+        let analysis = audio_analysis_cache().get(&wav_str).unwrap().clone();
+        std::fs::remove_file(&wav_path).ok();
+
+        assert!(
+            analysis.amplitude_at(0.5) > 0.5,
+            "inside the audible half the envelope is open"
+        );
+        assert!(
+            analysis.amplitude_at(1.5) < 0.05,
+            "the keyframes take the track to silence — the visualisation must \
+             follow it, not keep drawing the sine underneath"
+        );
+    }
+
+    /// The envelope is part of the cache key: two scenarios naming the same
+    /// file with different mixes must not share an analysis.
+    #[test]
+    fn changing_the_envelope_re_analyses_the_same_file() {
+        let sample_rate = 44100u32;
+        let wav_path = std::env::temp_dir().join(format!("rustmotion_test_env_{}.wav", nanos()));
+        std::fs::write(
+            &wav_path,
+            make_sine_wav(sample_rate, sample_rate, 440.0, sample_rate),
+        )
+        .expect("write fixture");
+        let wav_str = wav_path.to_str().unwrap().to_string();
+
+        let with_volume = |v: f32| {
+            let json = serde_json::json!({
+                "video": {"width": 32, "height": 32, "fps": 30},
+                "audio": [{"src": wav_str, "volume": v}],
+                "scenes": [{"duration": 1.0, "children": []}]
+            })
+            .to_string();
+            let scenario =
+                crate::loader::load_scenario_from_source(None, Some(&json)).expect("load");
+            crate::encode::audio_analysis::analyze_scenario_audio(&scenario);
+            audio_analysis_cache().get(&wav_str).unwrap().amplitude[10]
+        };
+
+        // Amplitudes are normalised per analysis, so a flat gain change cannot
+        // be read off the values — assert the *entry* was replaced instead.
+        let loud = with_volume(1.0);
+        let quiet = with_volume(0.0);
+        std::fs::remove_file(&wav_path).ok();
+
+        assert!(loud > 0.5, "the full-volume take is audible, got {loud}");
+        assert_eq!(
+            quiet, 0.0,
+            "at volume 0 the analysis must be silent — a stale entry would \
+             still report {loud}"
+        );
+    }
+
     /// The failure the whole rewrite exists for: a scenario naming an asset
     /// beside itself must load identically whatever directory the process
     /// runs from. Authoring from the scenario's folder worked; the studio,