diff --git a/crates/compositor/src/audio.rs b/crates/compositor/src/audio.rs index 1b51a7a9..adfeb090 100644 --- a/crates/compositor/src/audio.rs +++ b/crates/compositor/src/audio.rs @@ -4,6 +4,7 @@ use crate::ffi::*; use crate::regions::SpeedSegment; +use crate::scene::SceneAudio; use anyhow::{bail, Result}; use std::f32::consts::PI; use std::ffi::CString; @@ -26,6 +27,114 @@ const PASSTHROUGH_EPSILON: f64 = 1e-3; pub type PlanarPcm = Vec>; +/// Apply the editor's signed sync offset and fast voice-oriented mastering. +/// +/// The automatic path is deliberately deterministic and dependency-free: an 80 Hz high-pass +/// removes desk/room rumble, a linked-stereo 3:1 compressor controls speech peaks, and a final +/// RMS/peak pass raises intelligibility without clipping. Linking the envelope preserves the +/// stereo image. The result stays the same length so video and following clips cannot drift. +pub fn finish_audio(mut pcm: PlanarPcm, settings: SceneAudio) -> PlanarPcm { + let samples = pcm.first().map(Vec::len).unwrap_or(0); + if samples == 0 { + return pcm; + } + for channel in pcm.iter_mut() { + channel.resize(samples, 0.0); + } + + if settings.auto_master { + let cutoff_hz = 80.0f32; + let dt = 1.0 / AUDIO_OUTPUT_SAMPLE_RATE as f32; + let rc = 1.0 / (2.0 * PI * cutoff_hz); + let alpha = rc / (rc + dt); + for channel in pcm.iter_mut() { + let mut previous_input = 0.0f32; + let mut previous_output = 0.0f32; + for sample in channel.iter_mut() { + let input = *sample; + let output = alpha * (previous_output + input - previous_input); + *sample = output; + previous_input = input; + previous_output = output; + } + } + + let threshold_db = -18.0f32; + let ratio = 3.0f32; + let attack = (-1.0 / (0.010 * AUDIO_OUTPUT_SAMPLE_RATE as f32)).exp(); + let release = (-1.0 / (0.120 * AUDIO_OUTPUT_SAMPLE_RATE as f32)).exp(); + let mut envelope = 0.0f32; + for index in 0..samples { + let peak = pcm + .iter() + .map(|channel| channel[index].abs()) + .fold(0.0f32, f32::max); + let coefficient = if peak > envelope { attack } else { release }; + envelope = coefficient * envelope + (1.0 - coefficient) * peak; + let level_db = 20.0 * envelope.max(1e-9).log10(); + let reduction_db = if level_db > threshold_db { + (level_db - threshold_db) * (1.0 - 1.0 / ratio) + } else { + 0.0 + }; + let reduction = 10.0f32.powf(-reduction_db / 20.0); + for channel in pcm.iter_mut() { + channel[index] *= reduction; + } + } + + let mut sum_squares = 0.0f64; + let mut count = 0usize; + let mut peak = 0.0f32; + for &sample in pcm.iter().flatten() { + peak = peak.max(sample.abs()); + if sample.abs() > 1e-5 { + sum_squares += (sample as f64) * (sample as f64); + count += 1; + } + } + if count > 0 && peak > 0.0 { + let rms = (sum_squares / count as f64).sqrt() as f32; + let target_rms = 10.0f32.powf(-16.0 / 20.0); + let peak_ceiling = 10.0f32.powf(-1.0 / 20.0); + let makeup = (target_rms / rms.max(1e-9)) + .min(peak_ceiling / peak) + .clamp(0.1, 4.0); + for sample in pcm.iter_mut().flatten() { + *sample *= makeup; + } + } + } + + let trim = 10.0f32.powf(settings.gain_db.clamp(-24.0, 18.0) / 20.0); + for sample in pcm.iter_mut().flatten() { + *sample = (*sample * trim).clamp(-1.0, 1.0); + } + + let shift = ((settings.offset_ms.clamp(-2_000.0, 2_000.0) / 1_000.0) + * AUDIO_OUTPUT_SAMPLE_RATE as f64) + .round() as i64; + if shift == 0 { + return pcm; + } + if shift > 0 { + let destination = (shift as usize).min(samples); + let count = samples - destination; + for channel in pcm.iter_mut() { + channel.copy_within(..count, destination); + channel[..destination].fill(0.0); + } + } else { + let source = ((-shift) as usize).min(samples); + let count = samples - source; + for channel in pcm.iter_mut() { + channel.copy_within(source.., 0); + channel[count..].fill(0.0); + } + } + pcm +} + extern "C" { fn sn_fmt_stream(s: *mut AVFormatContext, i: i32) -> *mut AVStream; // bindgen rend `AVFormatContext` opaque (atteinte seulement par pointeur), d'où l'accesseur @@ -1063,4 +1172,65 @@ mod tests { let mixed = mix_aligned_tracks(&[(-0.001, &early)], 0.0, 8); assert_eq!(mixed[0], vec![0.5; 8]); } + + #[test] + fn signed_audio_offset_is_length_preserving() { + let settings = SceneAudio { + offset_ms: 1000.0 / AUDIO_OUTPUT_SAMPLE_RATE as f64, + gain_db: 0.0, + auto_master: false, + }; + let delayed = finish_audio(planar(&[1.0, 2.0, 3.0]), settings); + assert_eq!(delayed[0], vec![0.0, 1.0, 1.0]); + + let advanced = finish_audio( + planar(&[1.0, 0.5, 0.25]), + SceneAudio { + offset_ms: -1000.0 / AUDIO_OUTPUT_SAMPLE_RATE as f64, + ..settings + }, + ); + assert_eq!(advanced[0], vec![0.5, 0.25, 0.0]); + } + + #[test] + fn manual_gain_is_applied_when_auto_master_is_off() { + let result = finish_audio( + planar(&[0.25, -0.25]), + SceneAudio { + offset_ms: 0.0, + gain_db: 6.0206, + auto_master: false, + }, + ); + assert!((result[0][0] - 0.5).abs() < 1e-4); + assert!((result[0][1] + 0.5).abs() < 1e-4); + } + + #[test] + fn auto_master_removes_dc_and_respects_peak_ceiling() { + let mut input = vec![0.2f32; AUDIO_OUTPUT_SAMPLE_RATE as usize / 2]; + for index in (0..input.len()).step_by(400) { + input[index] = 1.0; + } + let result = finish_audio( + planar(&input), + SceneAudio { + offset_ms: 0.0, + gain_db: 0.0, + auto_master: true, + }, + ); + let peak = result + .iter() + .flatten() + .fold(0.0f32, |value, sample| value.max(sample.abs())); + let tail_mean = result[0][result[0].len() / 2..] + .iter() + .copied() + .sum::() + / (result[0].len() / 2) as f32; + assert!(peak <= 10.0f32.powf(-1.0 / 20.0) + 1e-5); + assert!(tail_mean.abs() < 0.01); + } } diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs index fc32ecaf..aa6db8de 100644 --- a/crates/compositor/src/compositor_linux.rs +++ b/crates/compositor/src/compositor_linux.rs @@ -1230,9 +1230,12 @@ impl Compositor { // `cover_crop_uv` est la primitive partagee que macOS et Windows // utilisent ; elle rend le rect inchange quand il a deja le bon // ratio, donc aucun placement correct ne bouge. - let (cu0, cv0, cu1, cv1) = crate::frame_geometry::cover_crop_uv( + let [cu0, cv0, cu1, cv1] = crate::frame_geometry::webcam_source_rect( [wcw, wch], [wtw as f32, wth as f32], + scene_ref + .as_ref() + .and_then(|scene| scene.layout.webcam_crop), g.w_px[0] / g.w_px[1].max(0.0001), ); // MIROIR : on inverse l'intervalle u. Le VS interpole `src` diff --git a/crates/compositor/src/compositor_macos.rs b/crates/compositor/src/compositor_macos.rs index 4ea09a42..a9cdaedc 100644 --- a/crates/compositor/src/compositor_macos.rs +++ b/crates/compositor/src/compositor_macos.rs @@ -1601,9 +1601,10 @@ impl Compositor { // --- caméra : ombre PiP puis vidéo --- let enc = self.begin_pass(cmd_buf, &self.rt, None, &self.pipeline_main)?; if let (true, Some((wy, wuv))) = (lp.has_webcam, webcam_tex.as_ref()) { - let (cu0, cv0, cu1, cv1) = crate::frame_geometry::cover_crop_uv( + let [cu0, cv0, cu1, cv1] = crate::frame_geometry::webcam_source_rect( [wcw, wch], [wtw as f32, wth as f32], + scene_ref.as_ref().and_then(|scene| scene.layout.webcam_crop), g.w_px[0] / g.w_px[1].max(0.0001), ); let (u0, u1) = if lp.webcam_mirror { (cu1, cu0) } else { (cu0, cu1) }; diff --git a/crates/compositor/src/compositor_windows.rs b/crates/compositor/src/compositor_windows.rs index b4f19b9b..f4cf9a5b 100644 --- a/crates/compositor/src/compositor_windows.rs +++ b/crates/compositor/src/compositor_windows.rs @@ -1517,9 +1517,12 @@ impl Compositor { // // Le center-crop carré de square/circle en est un cas particulier (boîte 1:1) — il n'a // plus besoin d'être traité à part. - let (su0, sv0, su1, sv1) = cover_crop_uv( + let [su0, sv0, su1, sv1] = crate::frame_geometry::webcam_source_rect( [wcw, wch], [wtw as f32, wth as f32], + scene_ref + .as_ref() + .and_then(|scene| scene.layout.webcam_crop), w_px[0] / w_px[1].max(0.0001), ); // miroir = échanger les bornes u du rect source (flip horizontal). diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs index 738a14f8..6cfe290a 100644 --- a/crates/compositor/src/frame_geometry.rs +++ b/crates/compositor/src/frame_geometry.rs @@ -284,6 +284,23 @@ pub(crate) fn cover_crop_uv(visible: [f32; 2], tex: [f32; 2], box_ar: f32) -> (f let [u0, v0, u1, v1] = cover_uv_rect(full, tex, box_ar); (u0, v0, u1, v1) } + +/// Camera equivalent of the screen crop pipeline: apply the user crop first, then a centred +/// cover-crop inside that authored window so arbitrary layout slots never stretch the image. +pub(crate) fn webcam_source_rect( + visible: [f32; 2], + tex: [f32; 2], + crop: Option, + box_ar: f32, +) -> [f32; 4] { + let u_max = visible[0].max(1.0) / tex[0].max(1.0); + let v_max = visible[1].max(1.0) / tex[1].max(1.0); + cover_uv_rect( + screen_source_rect(u_max, v_max, crop, 1.0, [0.5, 0.5]), + tex, + box_ar, + ) +} /// Rétrécit un rect SOURCE déjà exprimé en UV (`[u0, v0, u1, v1]`) autour de son /// centre pour qu'il porte le ratio `box_ar` une fois rapporté aux pixels de la /// texture. C'est la forme générale de `object-fit: cover`, et LA primitive qui @@ -1750,4 +1767,22 @@ mod tests { assert!((su0 - (960.0 - 720.0) * 0.5 / tex[0]).abs() < 1e-6); assert!((su1 - (960.0 + 720.0) * 0.5 / tex[0]).abs() < 1e-6); } + + #[test] + fn webcam_crop_identity_keeps_the_full_visible_frame() { + let uv = webcam_source_rect([1280.0, 720.0], [2048.0, 1024.0], None, 16.0 / 9.0); + assert_rect(uv, [0.0, 0.0, 1280.0 / 2048.0, 720.0 / 1024.0]); + } + + #[test] + fn webcam_crop_applies_authored_zoom_and_pan_before_layout_cover() { + let crop = SceneCrop { + x: 0.25, + y: 0.20, + width: 0.50, + height: 0.60, + }; + let uv = webcam_source_rect([100.0, 100.0], [100.0, 100.0], Some(crop), 0.50 / 0.60); + assert_rect(uv, [0.25, 0.20, 0.75, 0.80]); + } } diff --git a/crates/compositor/src/pipeline_linux.rs b/crates/compositor/src/pipeline_linux.rs index 0d0d9804..910738fc 100644 --- a/crates/compositor/src/pipeline_linux.rs +++ b/crates/compositor/src/pipeline_linux.rs @@ -21,7 +21,7 @@ use std::ffi::CString; use std::ptr; use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, + assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, finish_audio, stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, }; use crate::config::Cfg; @@ -457,6 +457,7 @@ pub fn run_composited_multi( let mut clip_frame_counts: Vec = vec![0; clips.len()]; let scene = comp.scene_snapshot(); + let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default(); // Ring de staging a 2 : l'export ne veut que du debit, une frame de latence // ne se voit pas dans un fichier. Voir `Compositor::set_readback_depth` pour // la raison pour laquelle la preview, elle, reste a 1. @@ -533,7 +534,10 @@ pub fn run_composited_multi( // raccourci voit son audio raccourci d'autant), puis un seul encode AAC. let declared_audio: Vec = clips.iter().map(|c| c.has_audio).collect(); let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64); - audio_encoder.encode(&assemble_concatenated_pcm(&clip_pcm, &plan), octx)?; + audio_encoder.encode( + &finish_audio(assemble_concatenated_pcm(&clip_pcm, &plan), audio_settings), + octx, + )?; crate::ffi::averr(crate::ffi::av_write_trailer(octx), "write_trailer")?; crate::ffi::avio_closep(&mut pb); crate::ffi::avformat_free_context(octx); diff --git a/crates/compositor/src/pipeline_macos.rs b/crates/compositor/src/pipeline_macos.rs index 2ad12b66..10de3fac 100644 --- a/crates/compositor/src/pipeline_macos.rs +++ b/crates/compositor/src/pipeline_macos.rs @@ -30,7 +30,7 @@ //! décodeurs, symétrique. use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, + assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, finish_audio, stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, }; use crate::compositor::Compositor; @@ -1076,6 +1076,7 @@ pub fn run_composited_multi( // exactement le bug de troncature en slow-motion que la doc de `walk_composited_timeline` // raconte avoir déjà coûté une fois. let scene = comp.scene_snapshot(); + let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default(); frames = unsafe { crate::timeline_walk::walk_composited_timeline( clips, @@ -1130,7 +1131,10 @@ pub fn run_composited_multi( // d'autant, sinon la piste dérive pour tous les suivants. let declared_audio: Vec = clips.iter().map(|clip| clip.has_audio).collect(); let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64); - audio_encoder.encode(&assemble_concatenated_pcm(&clip_pcm, &plan), octx)?; + audio_encoder.encode( + &finish_audio(assemble_concatenated_pcm(&clip_pcm, &plan), audio_settings), + octx, + )?; crate::ffi::averr( crate::ffi::av_write_trailer(octx), @@ -1188,4 +1192,4 @@ pub fn probe_frame_count(_path: &str) -> Result { // cfg-re-export `crate::compositor::Compositor`, et cette fonction helper garantit // que le type reste référencé. #[allow(dead_code)] -fn _typecheck_compositor(_c: &Compositor, _g: &Gpu) {} \ No newline at end of file +fn _typecheck_compositor(_c: &Compositor, _g: &Gpu) {} diff --git a/crates/compositor/src/pipeline_windows.rs b/crates/compositor/src/pipeline_windows.rs index f5b02ac2..11738bcd 100644 --- a/crates/compositor/src/pipeline_windows.rs +++ b/crates/compositor/src/pipeline_windows.rs @@ -3,7 +3,7 @@ //! tout le run, deux lectures seulement. Rien dans la boucle ne peut fausser le fps. use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, + assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, finish_audio, stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, }; use crate::compositor::{Compositor, OUT_H, OUT_W}; @@ -1342,6 +1342,7 @@ unsafe fn run_multi_inner( // La scène (déjà posée par l'appelant via comp.set_scene) pilote le curseur et le // fenêtrage par clip ; `walk_composited_timeline` s'en charge. let scene = comp.scene_snapshot(); + let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default(); // ---- encodeur (choisi à l'exécution, cf. ExportCodec::candidates) + mux ---- // Backend CPU : pas de pool D3D11 du tout. `av_hwdevice_ctx_init(D3D11VA)` échoue sur @@ -1465,7 +1466,10 @@ unsafe fn run_multi_inner( &declared_audio, out_fps as f64, ); - let assembled_audio = assemble_concatenated_pcm(&clip_pcm, &audio_plan); + let assembled_audio = finish_audio( + assemble_concatenated_pcm(&clip_pcm, &audio_plan), + audio_settings, + ); audio_encoder.encode(&assembled_audio, octx)?; averr(av_write_trailer(octx), "write_trailer")?; diff --git a/crates/compositor/src/scene.rs b/crates/compositor/src/scene.rs index 2de56951..2043887c 100644 --- a/crates/compositor/src/scene.rs +++ b/crates/compositor/src/scene.rs @@ -45,6 +45,9 @@ pub struct SceneLayout { pub webcam_position: Option, /// la webcam rétrécit pendant un zoom actif. pub webcam_reactive_zoom: bool, + /// User-authored source crop for the camera. Absent keeps the full frame. + #[serde(default)] + pub webcam_crop: Option, /// Rect webcam résolu côté app (0..1 fractions du cadre de sortie), en PARITÉ EXACTE avec /// `computeCompositeLayout` (TS). Permet à TS et Rust de partager la même source de vérité : /// le natif ne dérive PLUS ses propres placements pour PiP/dual-frame/vertical-stack — il @@ -361,6 +364,24 @@ pub struct SceneCrop { pub height: f32, } +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneAudio { + pub offset_ms: f64, + pub gain_db: f32, + pub auto_master: bool, +} + +impl Default for SceneAudio { + fn default() -> Self { + Self { + offset_ms: 0.0, + gain_db: 0.0, + auto_master: false, + } + } +} + #[derive(Debug, Clone, Copy, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SceneOutput { @@ -389,6 +410,9 @@ pub struct Scene { #[serde(default)] pub camera_fullscreen_regions: Vec, pub cursor: SceneCursor, + /// Global audio finishing. Default keeps old scene payloads bit-for-bit compatible. + #[serde(default)] + pub audio: SceneAudio, /// Crop écran par clip, dans le même ordre que `clips` (`cropByClip` côté TS). #[serde(default)] pub crop_by_clip: Vec>, diff --git a/src/components/ai-edition/NewEditorShell.module.css b/src/components/ai-edition/NewEditorShell.module.css index bb96cc6c..cc24ab34 100644 --- a/src/components/ai-edition/NewEditorShell.module.css +++ b/src/components/ai-edition/NewEditorShell.module.css @@ -2026,6 +2026,20 @@ color: var(--danger); } +.secondaryBtn { + margin: 4px var(--sp-4) 12px; + min-height: 32px; + padding: 0 12px; + border: 1px solid var(--border); + border-radius: var(--r-sm); + background: var(--surface-2); + color: var(--fg-2); + font: 500 12px/1 var(--font-body); + cursor: pointer; +} +.secondaryBtn:hover { background: var(--surface-3); color: var(--fg); } +.secondaryBtn:disabled { opacity: 0.45; cursor: not-allowed; } + .authPanel { padding: 14px 16px; border: 1px solid var(--brand); diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx index 208942e2..eb21f786 100644 --- a/src/components/ai-edition/NewEditorShell.tsx +++ b/src/components/ai-edition/NewEditorShell.tsx @@ -338,6 +338,7 @@ export function NewEditorShell() { if (!document) return []; return document.assets.map((asset) => ({ id: asset.id, + filePath: /^(https?|blob|data):/.test(asset.originalPath) ? undefined : asset.originalPath, // Real Electron assets are filesystem paths and go through toFileUrl. // In the browser preview an asset can already point at an http(s)/ // blob/data URL served by Vite; toFileUrl would mangle those into a @@ -1219,7 +1220,7 @@ export function NewEditorShell() { /> ) : mode === "media" ? ( - + ) : ( void handleNewRecording()} diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx index 5ac70471..3c6adf23 100644 --- a/src/components/ai-edition/RightPanes.tsx +++ b/src/components/ai-edition/RightPanes.tsx @@ -6,6 +6,7 @@ // self-sufficient). import { + AudioLines, FileText, HelpCircle, Layout as LayoutIcon, @@ -1520,6 +1521,23 @@ export function LayoutPane() { ? hasAnyClipWithCamera(document.assets, document.timeline.clips) : false; const layoutControlsDisabled = !hasDocument || !hasAnyCamera; + const webcamCrop = settings.webcamCropRegion; + const cropZoomPct = Math.round(100 / webcamCrop.width); + const cropPanX = webcamCrop.width >= 0.999 ? 50 : (webcamCrop.x / (1 - webcamCrop.width)) * 100; + const cropPanY = webcamCrop.height >= 0.999 ? 50 : (webcamCrop.y / (1 - webcamCrop.height)) * 100; + const setCropZoom = (zoomPct: number) => { + const size = 100 / Math.max(100, zoomPct); + const centerX = webcamCrop.x + webcamCrop.width / 2; + const centerY = webcamCrop.y + webcamCrop.height / 2; + setLive({ + webcamCropRegion: { + x: Math.min(1 - size, Math.max(0, centerX - size / 2)), + y: Math.min(1 - size, Math.max(0, centerY - size / 2)), + width: size, + height: size, + }, + }); + }; return ( } helpText={ts("layout.help")}>
{ts("layout.preset")}
@@ -1652,6 +1670,99 @@ export function LayoutPane() { ) : null} +
{ts("layout.webcamFraming")}
+
+ void commit()} + /> + = 0.999} + onChange={(value) => + setLive({ + webcamCropRegion: { ...webcamCrop, x: (value / 100) * (1 - webcamCrop.width) }, + }) + } + onCommit={() => void commit()} + /> + = 0.999} + onChange={(value) => + setLive({ + webcamCropRegion: { ...webcamCrop, y: (value / 100) * (1 - webcamCrop.height) }, + }) + } + onCommit={() => void commit()} + /> +
+ + ); +} + +// ─── Audio ──────────────────────────────────────────────────────── + +export function AudioPane() { + const ts = useScopedT("settings"); + const { settings, set, setLive, commit, hasDocument } = useEditorSettings(); + return ( + } helpText={ts("audio.help")}> +
+ {ts("audio.autoMaster")} + void set({ audioAutoMaster: value })} + /> +
+
+ setLive({ audioOffsetMs: value })} + onCommit={() => void commit()} + /> + setLive({ audioGainDb: value })} + onCommit={() => void commit()} + /> +
+ ); } diff --git a/src/components/ai-edition/VirtualPreview.audio.test.ts b/src/components/ai-edition/VirtualPreview.audio.test.ts new file mode 100644 index 00000000..a565aebc --- /dev/null +++ b/src/components/ai-edition/VirtualPreview.audio.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { resolveAudioPreviewTime } from "./VirtualPreview"; + +describe("resolveAudioPreviewTime", () => { + it("delays audio for a positive offset", () => { + expect(resolveAudioPreviewTime(0.1, 160, 10)).toEqual({ targetTimeSec: 0, shouldPlay: false }); + expect(resolveAudioPreviewTime(1, 160, 10)).toEqual({ targetTimeSec: 0.84, shouldPlay: true }); + }); + + it("advances audio for a negative offset", () => { + expect(resolveAudioPreviewTime(1, -160, 10)).toEqual({ targetTimeSec: 1.16, shouldPlay: true }); + }); + + it("stops instead of seeking past the track", () => { + expect(resolveAudioPreviewTime(9.9, -160, 10)).toEqual({ + targetTimeSec: 10, + shouldPlay: false, + }); + }); + + it("plays while the duration is still unknown", () => { + expect(resolveAudioPreviewTime(1, 0, Number.NaN)).toEqual({ + targetTimeSec: 1, + shouldPlay: true, + }); + }); +}); diff --git a/src/components/ai-edition/VirtualPreview.tsx b/src/components/ai-edition/VirtualPreview.tsx index c3eb6cbb..18cf00a6 100644 --- a/src/components/ai-edition/VirtualPreview.tsx +++ b/src/components/ai-edition/VirtualPreview.tsx @@ -28,9 +28,56 @@ import styles from "./VirtualPreview.module.css"; export interface VideoSource { id: string; src: string; + /** Original filesystem path, used by the main process to expose the second audio track. */ + filePath?: string; label: string; } +export function resolveAudioPreviewTime( + videoTimeSec: number, + offsetMs: number, + durationSec = Number.POSITIVE_INFINITY, +) { + const raw = videoTimeSec - offsetMs / 1000; + const finiteDuration = Number.isFinite(durationSec) && durationSec > 0 ? durationSec : Infinity; + return { + targetTimeSec: Math.min(Math.max(0, raw), finiteDuration), + shouldPlay: raw >= 0 && raw < finiteDuration, + }; +} + +interface PreviewAudioGraph { + context: AudioContext; + highpasses: BiquadFilterNode[]; + compressor: DynamicsCompressorNode; + gain: GainNode; +} + +function applyPreviewAudioSettings( + graph: PreviewAudioGraph | null, + elements: Array, + autoMaster: boolean, + gainDb: number, +): void { + const outputGain = 10 ** (gainDb / 20); + if (!graph) { + for (const element of elements) { + if (element) element.volume = Math.min(1, outputGain); + } + return; + } + for (const filter of graph.highpasses) { + filter.frequency.value = autoMaster ? 80 : 20; + filter.Q.value = 0.707; + } + graph.compressor.threshold.value = autoMaster ? -18 : 0; + graph.compressor.knee.value = autoMaster ? 12 : 0; + graph.compressor.ratio.value = autoMaster ? 3 : 1; + graph.compressor.attack.value = 0.01; + graph.compressor.release.value = 0.12; + graph.gain.gain.value = outputGain; +} + /** First clip (by timeline order) starting strictly after `afterTimelineStartSec` — * independent of the `clips` array's own order, which is never guaranteed to match * timeline order (a clip can be inserted/reordered at any array index; only @@ -93,6 +140,8 @@ export function VirtualPreview({ clockRef, }: VirtualPreviewProps) { const { settings } = useEditorSettings(); + const audioOffsetMsRef = useRef(settings.audioOffsetMs); + audioOffsetMsRef.current = settings.audioOffsetMs; // ponytail: an oversized, offset video inside .videoFrame's overflow:hidden // box — the same "scale + negative-position the full frame, let the // container clip the rest" technique the export renderer uses via a Pixi @@ -114,6 +163,24 @@ export function VirtualPreview({ top: `${(-region.y * 100) / region.height}%`, }; const videoRef = useRef(null); + const primaryAudioRef = useRef(null); + const supplementalAudioRef = useRef(null); + const [primaryAudioEl, setPrimaryAudioEl] = useState(null); + const [supplementalAudioEl, setSupplementalAudioEl] = useState(null); + const [supplementalAudioSrc, setSupplementalAudioSrc] = useState(null); + const [audioProbeComplete, setAudioProbeComplete] = useState(false); + const audioSettingsRef = useRef({ + autoMaster: settings.audioAutoMaster, + gainDb: settings.audioGainDb, + }); + audioSettingsRef.current = { + autoMaster: settings.audioAutoMaster, + gainDb: settings.audioGainDb, + }; + const audioContextRef = useRef(null); + const audioContextCloseTimerRef = useRef | null>(null); + const audioSourceNodesRef = useRef(new WeakMap()); + const audioGraphRef = useRef(null); const videoFrameRef = useRef(null); const isProgrammaticSeekRef = useRef(false); @@ -133,6 +200,137 @@ export function VirtualPreview({ const virtualDurationSec = useMemo(() => totalVirtualDuration(clips), [clips]); const activeSource = videoSources[sourceIndex] ?? null; + useEffect(() => { + let cancelled = false; + setSupplementalAudioSrc(null); + setAudioProbeComplete(false); + if (!activeSource?.filePath || !window.electronAPI?.preparePreviewAudioTrack) { + setAudioProbeComplete(true); + return () => { + cancelled = true; + }; + } + void window.electronAPI.preparePreviewAudioTrack(activeSource.filePath).then( + (result) => { + if (cancelled) return; + setSupplementalAudioSrc(result.success ? (result.path ?? null) : null); + setAudioProbeComplete(true); + }, + () => { + if (cancelled) return; + setSupplementalAudioSrc(null); + setAudioProbeComplete(true); + }, + ); + return () => { + cancelled = true; + }; + }, [activeSource?.filePath]); + + // Route preview audio through the same fast voice-oriented controls the native export uses. + // The primary media element carries track 1; on macOS the existing IPC helper extracts track 2 + // (normally the microphone) so both are audible instead of Chromium silently choosing one. + useEffect(() => { + if (!primaryAudioEl || !audioProbeComplete) return; + if (supplementalAudioSrc && !supplementalAudioEl) return; + const connectedSources: MediaElementAudioSourceNode[] = []; + const highpasses: BiquadFilterNode[] = []; + let compressor: DynamicsCompressorNode | null = null; + let gain: GainNode | null = null; + try { + let context = audioContextRef.current; + if (!context || context.state === "closed") { + context = new AudioContext(); + audioContextRef.current = context; + audioSourceNodesRef.current = new WeakMap(); + } + compressor = context.createDynamicsCompressor(); + gain = context.createGain(); + compressor.connect(gain).connect(context.destination); + for (const element of [primaryAudioEl, supplementalAudioEl].filter( + (value): value is HTMLAudioElement => Boolean(value), + )) { + let source = audioSourceNodesRef.current.get(element); + if (!source) { + source = context.createMediaElementSource(element); + audioSourceNodesRef.current.set(element, source); + } + source.disconnect(); + const highpass = context.createBiquadFilter(); + highpass.type = "highpass"; + source.connect(highpass).connect(compressor); + connectedSources.push(source); + highpasses.push(highpass); + } + const graph = { context, highpasses, compressor, gain }; + audioGraphRef.current = graph; + applyPreviewAudioSettings( + graph, + [primaryAudioEl, supplementalAudioEl], + audioSettingsRef.current.autoMaster, + audioSettingsRef.current.gainDb, + ); + } catch { + // WebAudio can be unavailable in unit tests or under a denied audio policy. The media + // elements remain usable; the sync loop below still applies offset and playback state. + for (const source of connectedSources) source.disconnect(); + for (const highpass of highpasses) highpass.disconnect(); + compressor?.disconnect(); + gain?.disconnect(); + applyPreviewAudioSettings( + null, + [primaryAudioEl, supplementalAudioEl], + audioSettingsRef.current.autoMaster, + audioSettingsRef.current.gainDb, + ); + } + return () => { + audioGraphRef.current = null; + for (const source of connectedSources) source.disconnect(); + for (const highpass of highpasses) highpass.disconnect(); + compressor?.disconnect(); + gain?.disconnect(); + }; + }, [primaryAudioEl, supplementalAudioEl, supplementalAudioSrc, audioProbeComplete]); + + // Keep one AudioContext for the component. Closing and recreating it on an effect rerun + // permanently silences an HTMLAudioElement because createMediaElementSource may only be + // called once for that element. Delay final cleanup by one task so React StrictMode's + // intentional setup → cleanup → setup cycle can cancel the close and reuse the context. + useEffect(() => { + if (audioContextCloseTimerRef.current) { + clearTimeout(audioContextCloseTimerRef.current); + audioContextCloseTimerRef.current = null; + } + return () => { + audioContextCloseTimerRef.current = setTimeout(() => { + audioContextCloseTimerRef.current = null; + const context = audioContextRef.current; + audioContextRef.current = null; + audioSourceNodesRef.current = new WeakMap(); + if (context) void context.close(); + }, 0); + }; + }, []); + + useEffect(() => { + applyPreviewAudioSettings( + audioGraphRef.current, + [primaryAudioRef.current, supplementalAudioRef.current], + settings.audioAutoMaster, + settings.audioGainDb, + ); + }, [settings.audioAutoMaster, settings.audioGainDb]); + + const setPrimaryAudioElement = useCallback((element: HTMLAudioElement | null) => { + primaryAudioRef.current = element; + setPrimaryAudioEl(element); + }, []); + const setSupplementalAudioElement = useCallback((element: HTMLAudioElement | null) => { + supplementalAudioRef.current = element; + setSupplementalAudioEl(element); + }, []); + // ponytail: the cursor overlay wants source-media time (the recorded // cursor samples live on the original mp4 timeline, not the edited // virtual timeline). `setSourceTimeSec` is called from the 60 Hz rAF @@ -209,6 +407,31 @@ export function VirtualPreview({ if (!v || !Number.isFinite(v.currentTime)) { return; } + for (const audio of [primaryAudioRef.current, supplementalAudioRef.current]) { + if (!audio) continue; + const target = resolveAudioPreviewTime( + v.currentTime, + audioOffsetMsRef.current, + audio.duration, + ); + if (audio.playbackRate !== v.playbackRate) audio.playbackRate = v.playbackRate; + if (Math.abs(audio.currentTime - target.targetTimeSec) > 0.025) { + try { + audio.currentTime = target.targetTimeSec; + } catch { + // media metadata not ready yet + } + } + if (!v.paused && target.shouldPlay && audio.paused) { + if (audioGraphRef.current?.context.state === "suspended") { + void audioGraphRef.current.context.resume(); + } + const playback = audio.play(); + if (playback) void playback.catch(() => undefined); + } else if ((v.paused || !target.shouldPlay) && !audio.paused) { + audio.pause(); + } + } // Publish this frame's live position/rate for other media elements // (webcam) to read directly — see playback-clock.ts for why this // bypasses React state entirely. @@ -539,6 +762,7 @@ export function VirtualPreview({ cursor: settings.cursorShow ? "none" : undefined, }} preload="metadata" + muted playsInline onLoadedMetadata={(e) => { setLoadState("ready"); @@ -626,6 +850,24 @@ export function VirtualPreview({ // handles clip-end advancement, so dropping the event // handler here is safe. /> +