From 05f0e34547366b1e1909cc6aab224fac719bcb2c Mon Sep 17 00:00:00 2001 From: christopher5106 Date: Tue, 6 Oct 2026 17:44:36 +0200 Subject: [PATCH] [LTX-2] Add `conditioning_frame_rate` to decouple the positional time base from the playback frame rate LTX-2 places video tokens on a time axis in seconds (`pixel_frame / fps` in the RoPE coordinates), derived from `frame_rate`. Adapters such as the LTX-2.5 Slow-Motion-Control LoRA are conditioned on a different rate (`fps / speed`) than the video plays at. Add an optional `conditioning_frame_rate` to the LTX-2 classic pipelines and modular blocks: it drives the video, keyframe and reference coordinates and the transformer's `fps`, while the audio length, the duration head and the returned video keep following `frame_rate`. `None` resolves to `frame_rate`, leaving current outputs unchanged. --- .../modular_pipelines/ltx2/before_denoise.py | 54 +++++++++++++++++-- .../modular_pipelines/ltx2/denoise.py | 11 +++- .../modular_pipelines/ltx2/encoders.py | 11 +++- .../ltx2/modular_blocks_ltx2.py | 44 +++++++++++++++ .../ltx2/modular_blocks_ltx25.py | 4 ++ src/diffusers/pipelines/ltx2/pipeline_ltx2.py | 21 ++++++-- .../pipelines/ltx2/pipeline_ltx2_condition.py | 23 ++++++-- .../pipelines/ltx2/pipeline_ltx2_ic_lora.py | 23 ++++++-- .../ltx2/pipeline_ltx2_image2video.py | 21 ++++++-- .../ltx2/test_modular_pipeline_ltx25.py | 19 +++++++ tests/pipelines/ltx2/test_ltx2.py | 26 +++++++++ 11 files changed, 233 insertions(+), 24 deletions(-) diff --git a/src/diffusers/modular_pipelines/ltx2/before_denoise.py b/src/diffusers/modular_pipelines/ltx2/before_denoise.py index 1878c01305fb..5f2bb7e6fd8d 100644 --- a/src/diffusers/modular_pipelines/ltx2/before_denoise.py +++ b/src/diffusers/modular_pipelines/ltx2/before_denoise.py @@ -752,6 +752,15 @@ def inputs(self) -> list[InputParam]: InputParam( "frame_rate", type_hint=float, default=24.0, description="Frames per second of the generated video." ), + InputParam( + "conditioning_frame_rate", + type_hint=float, + description=( + "Frame rate the model is conditioned on (the time axis of the positional embeddings). Defaults " + "to `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture " + "rate differs from its playback rate, e.g. slow-motion LoRAs: `frame_rate / speed`." + ), + ), InputParam("audio_num_frames", type_hint=int, required=True), InputParam.template("num_images_per_prompt", name="num_videos_per_prompt"), InputParam( @@ -790,7 +799,12 @@ def __call__(self, components, state: PipelineState) -> tuple[LTX2ModularPipelin latent_num_frames = (block_state.num_frames - 1) // components.vae_temporal_compression_ratio + 1 block_state.video_coords = components.transformer.rope.prepare_video_coords( - batch_size, latent_num_frames, latent_height, latent_width, device, fps=block_state.frame_rate + batch_size, + latent_num_frames, + latent_height, + latent_width, + device, + fps=block_state.conditioning_frame_rate or block_state.frame_rate, ) block_state.audio_coords = components.transformer.audio_rope.prepare_audio_coords( batch_size, block_state.audio_num_frames, device @@ -863,6 +877,15 @@ def inputs(self) -> list[InputParam]: InputParam( "frame_rate", type_hint=float, default=24.0, description="Frames per second of the generated video." ), + InputParam( + "conditioning_frame_rate", + type_hint=float, + description=( + "Frame rate the model is conditioned on (the time axis of the positional embeddings). Defaults " + "to `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture " + "rate differs from its playback rate, e.g. slow-motion LoRAs: `frame_rate / speed`." + ), + ), InputParam( "noise_scale", type_hint=float, @@ -1022,7 +1045,7 @@ def __call__(self, components, state: PipelineState) -> tuple[LTX2ModularPipelin keyframe_latent_width=kf_latent_width, pixel_frame_idx=(latent_idx - 1) * frame_scale_factor + 1, num_pixel_frames=num_pixel_frames, - fps=block_state.frame_rate, + fps=block_state.conditioning_frame_rate or block_state.frame_rate, patch_size=spatial_patch, patch_size_t=temporal_patch, scale_factors=scale_factors, @@ -1159,6 +1182,15 @@ def inputs(self) -> list[InputParam]: InputParam( "frame_rate", type_hint=float, default=24.0, description="Frames per second of the generated video." ), + InputParam( + "conditioning_frame_rate", + type_hint=float, + description=( + "Frame rate the model is conditioned on (the time axis of the positional embeddings). Defaults " + "to `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture " + "rate differs from its playback rate, e.g. slow-motion LoRAs: `frame_rate / speed`." + ), + ), InputParam( "noise_scale", type_hint=float, @@ -1317,7 +1349,7 @@ def __call__(self, components, state: PipelineState) -> tuple[LTX2ModularPipelin keyframe_latent_width=kf_latent_width, pixel_frame_idx=(latent_idx - 1) * frame_scale_factor + 1, num_pixel_frames=num_pixel_frames, - fps=block_state.frame_rate, + fps=block_state.conditioning_frame_rate or block_state.frame_rate, patch_size=spatial_patch, patch_size_t=temporal_patch, scale_factors=scale_factors, @@ -1706,6 +1738,15 @@ def inputs(self) -> list[InputParam]: InputParam( "frame_rate", type_hint=float, default=24.0, description="Frames per second of the generated video." ), + InputParam( + "conditioning_frame_rate", + type_hint=float, + description=( + "Frame rate the model is conditioned on (the time axis of the positional embeddings). Defaults " + "to `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture " + "rate differs from its playback rate, e.g. slow-motion LoRAs: `frame_rate / speed`." + ), + ), InputParam("audio_num_frames", type_hint=int, required=True), InputParam( "appended_coords", @@ -1750,7 +1791,12 @@ def __call__(self, components, state: PipelineState) -> tuple[LTX2ModularPipelin latent_num_frames = (block_state.num_frames - 1) // components.vae_temporal_compression_ratio + 1 video_coords = components.transformer.rope.prepare_video_coords( - batch_size, latent_num_frames, latent_height, latent_width, device, fps=block_state.frame_rate + batch_size, + latent_num_frames, + latent_height, + latent_width, + device, + fps=block_state.conditioning_frame_rate or block_state.frame_rate, ) block_state.video_coords = torch.cat([video_coords, block_state.appended_coords.to(video_coords.dtype)], dim=2) block_state.audio_coords = components.transformer.audio_rope.prepare_audio_coords( diff --git a/src/diffusers/modular_pipelines/ltx2/denoise.py b/src/diffusers/modular_pipelines/ltx2/denoise.py index 2fb4951377a1..4ebcb9585625 100644 --- a/src/diffusers/modular_pipelines/ltx2/denoise.py +++ b/src/diffusers/modular_pipelines/ltx2/denoise.py @@ -319,6 +319,15 @@ def inputs(self) -> list[InputParam]: InputParam( "frame_rate", type_hint=float, default=24.0, description="Frames per second of the generated video." ), + InputParam( + "conditioning_frame_rate", + type_hint=float, + description=( + "Frame rate the model is conditioned on (the time axis of the positional embeddings). Defaults " + "to `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture " + "rate differs from its playback rate, e.g. slow-motion LoRAs: `frame_rate / speed`." + ), + ), InputParam( "use_cross_timestep", type_hint=bool, @@ -361,7 +370,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=block_state.frame_rate, + fps=block_state.conditioning_frame_rate or block_state.frame_rate, use_cross_timestep=block_state.use_cross_timestep, attention_kwargs=block_state.attention_kwargs, perturbation_mask=None, diff --git a/src/diffusers/modular_pipelines/ltx2/encoders.py b/src/diffusers/modular_pipelines/ltx2/encoders.py index ae3d29b5cfe3..47ff0943cc98 100644 --- a/src/diffusers/modular_pipelines/ltx2/encoders.py +++ b/src/diffusers/modular_pipelines/ltx2/encoders.py @@ -1197,6 +1197,15 @@ def inputs(self) -> list[InputParam]: InputParam( "frame_rate", type_hint=float, default=24.0, description="Frames per second of the generated video." ), + InputParam( + "conditioning_frame_rate", + type_hint=float, + description=( + "Frame rate the model is conditioned on (the time axis of the positional embeddings). Defaults " + "to `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture " + "rate differs from its playback rate, e.g. slow-motion LoRAs: `frame_rate / speed`." + ), + ), InputParam.template("generator"), ] @@ -1285,7 +1294,7 @@ def __call__(self, components, state: PipelineState) -> tuple[LTX2ModularPipelin height=ref_latent_height, width=ref_latent_width, device=device, - fps=block_state.frame_rate, + fps=block_state.conditioning_frame_rate or block_state.frame_rate, ) if downscale_factor != 1: ref_coords[:, 1, :, :] = ref_coords[:, 1, :, :] * downscale_factor diff --git a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py index 86428328a5a6..adea71e92baf 100644 --- a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py +++ b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py @@ -322,6 +322,10 @@ class LTX2CoreDenoiseStep(SequentialPipelineBlocks): Frames per second of the generated video. audio_latents (`Tensor`, *optional*): Optional pre-encoded audio latents; random noise is used when not provided. + conditioning_frame_rate (`float`, *optional*): + Frame rate the model is conditioned on (the time axis of the positional embeddings). Defaults to + `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture rate differs + from its playback rate, e.g. slow-motion LoRAs: `frame_rate / speed`. dtype (`dtype`): The dtype the model inputs are cast to. **denoiser_input_fields (`None`, *optional*): @@ -424,6 +428,10 @@ class LTX2Image2VideoCoreDenoiseStep(SequentialPipelineBlocks): Frames per second of the generated video. audio_latents (`Tensor`, *optional*): Optional pre-encoded audio latents; random noise is used when not provided. + conditioning_frame_rate (`float`, *optional*): + Frame rate the model is conditioned on (the time axis of the positional embeddings). Defaults to + `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture rate differs + from its playback rate, e.g. slow-motion LoRAs: `frame_rate / speed`. dtype (`dtype`): The dtype the model inputs are cast to. **denoiser_input_fields (`None`, *optional*): @@ -520,6 +528,10 @@ class LTX2ConditionCoreDenoiseStep(SequentialPipelineBlocks): `LTX2AutoDurationStep`). frame_rate (`float`, *optional*, defaults to 24.0): Frames per second of the generated video. + conditioning_frame_rate (`float`, *optional*): + Frame rate the model is conditioned on (the time axis of the positional embeddings). Defaults to + `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture rate differs + from its playback rate, e.g. slow-motion LoRAs: `frame_rate / speed`. noise_scale (`float`, *optional*): Initial noise level for the un-conditioned tokens. `None` (default) resolves to `sigmas[0]` when custom `sigmas` are supplied, else 1.0. @@ -676,6 +688,10 @@ class LTX2AutoReferenceEncoderStep(ConditionalPipelineBlocks): `LTX2AutoDurationStep`). frame_rate (`float`, *optional*, defaults to 24.0): Frames per second of the generated video. + conditioning_frame_rate (`float`, *optional*): + Frame rate the model is conditioned on (the time axis of the positional embeddings). Defaults to + `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture rate differs + from its playback rate, e.g. slow-motion LoRAs: `frame_rate / speed`. generator (`Generator`, *optional*): Torch generator for deterministic generation. @@ -831,6 +847,10 @@ class LTX2InContextCoreDenoiseStep(SequentialPipelineBlocks): `LTX2AutoDurationStep`). frame_rate (`float`, *optional*, defaults to 24.0): Frames per second of the generated video. + conditioning_frame_rate (`float`, *optional*): + Frame rate the model is conditioned on (the time axis of the positional embeddings). Defaults to + `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture rate differs + from its playback rate, e.g. slow-motion LoRAs: `frame_rate / speed`. noise_scale (`float`, *optional*): Initial noise level for the un-conditioned tokens. `None` (default) resolves to `sigmas[0]` when custom `sigmas` are supplied, else 1.0. @@ -962,6 +982,10 @@ class LTX2AutoCoreDenoiseStep(ConditionalPipelineBlocks): `LTX2AutoDurationStep`). frame_rate (`float`, *optional*, defaults to 24.0): Frames per second of the generated video. + conditioning_frame_rate (`float`, *optional*): + Frame rate the model is conditioned on (the time axis of the positional embeddings). Defaults to + `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture rate differs + from its playback rate, e.g. slow-motion LoRAs: `frame_rate / speed`. noise_scale (`float`, *optional*): Initial noise level for the un-conditioned tokens. `None` (default) resolves to `sigmas[0]` when custom `sigmas` are supplied, else 1.0. @@ -1297,6 +1321,10 @@ class LTX2Blocks(SequentialPipelineBlocks): which keeps the provided latents. audio_latents (`Tensor`, *optional*): Optional pre-encoded audio latents; random noise is used when not provided. + conditioning_frame_rate (`float`, *optional*): + Frame rate the model is conditioned on (the time axis of the positional embeddings). Defaults to + `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture rate differs + from its playback rate, e.g. slow-motion LoRAs: `frame_rate / speed`. **denoiser_input_fields (`None`, *optional*): conditional model inputs for the denoiser: e.g. prompt_embeds, negative_prompt_embeds, etc. use_cross_timestep (`bool`, *optional*, defaults to True): @@ -1412,6 +1440,10 @@ class LTX2ImageToVideoBlocks(SequentialPipelineBlocks): VAE-encoded reference-image latents used for image-to-video conditioning. audio_latents (`Tensor`, *optional*): Optional pre-encoded audio latents; random noise is used when not provided. + conditioning_frame_rate (`float`, *optional*): + Frame rate the model is conditioned on (the time axis of the positional embeddings). Defaults to + `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture rate differs + from its playback rate, e.g. slow-motion LoRAs: `frame_rate / speed`. **denoiser_input_fields (`None`, *optional*): conditional model inputs for the denoiser: e.g. prompt_embeds, negative_prompt_embeds, etc. use_cross_timestep (`bool`, *optional*, defaults to True): @@ -1511,6 +1543,10 @@ class LTX2ConditionBlocks(SequentialPipelineBlocks): The number of images to generate per prompt. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. + conditioning_frame_rate (`float`, *optional*): + Frame rate the model is conditioned on (the time axis of the positional embeddings). Defaults to + `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture rate differs + from its playback rate, e.g. slow-motion LoRAs: `frame_rate / speed`. noise_scale (`float`, *optional*): Initial noise level for the un-conditioned tokens. `None` (default) resolves to `sigmas[0]` when custom `sigmas` are supplied, else 1.0. @@ -1633,6 +1669,10 @@ class LTX2InContextBlocks(SequentialPipelineBlocks): `conditioning_attention_strength`. frame_rate (`float`, *optional*, defaults to 24.0): Frames per second of the generated video. + conditioning_frame_rate (`float`, *optional*): + Frame rate the model is conditioned on (the time axis of the positional embeddings). Defaults to + `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture rate differs + from its playback rate, e.g. slow-motion LoRAs: `frame_rate / speed`. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. reference_latents (`Tensor`, *optional*): @@ -1794,6 +1834,10 @@ class LTX2AutoBlocks(SequentialPipelineBlocks): Optional pixel-space mask of shape (1, 1, F, H, W) with values in [0, 1] giving spatially varying attention strength. Downsampled to the reference's latent grid and multiplied by `conditioning_attention_strength`. + conditioning_frame_rate (`float`, *optional*): + Frame rate the model is conditioned on (the time axis of the positional embeddings). Defaults to + `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture rate differs + from its playback rate, e.g. slow-motion LoRAs: `frame_rate / speed`. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. condition_latents (`list`, *optional*): diff --git a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py index 7c77aba94a74..919935e63942 100644 --- a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py +++ b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py @@ -290,6 +290,10 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): Optional pixel-space mask of shape (1, 1, F, H, W) with values in [0, 1] giving spatially varying attention strength. Downsampled to the reference's latent grid and multiplied by `conditioning_attention_strength`. + conditioning_frame_rate (`float`, *optional*): + Frame rate the model is conditioned on (the time axis of the positional embeddings). Defaults to + `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture rate differs + from its playback rate, e.g. slow-motion LoRAs: `frame_rate / speed`. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. condition_latents (`list`, *optional*): diff --git a/src/diffusers/pipelines/ltx2/pipeline_ltx2.py b/src/diffusers/pipelines/ltx2/pipeline_ltx2.py index 4e5ced0b4ec8..49a91f2efbcc 100644 --- a/src/diffusers/pipelines/ltx2/pipeline_ltx2.py +++ b/src/diffusers/pipelines/ltx2/pipeline_ltx2.py @@ -935,6 +935,7 @@ def __call__( min_seconds: float = 1.0, max_seconds: float = 20.0, frame_rate: float = 24.0, + conditioning_frame_rate: float | None = None, num_inference_steps: int = 30, sigmas: list[float] | None = None, timesteps: list[int] = None, @@ -998,6 +999,11 @@ def __call__( present. Ignored when `num_frames` is set explicitly. Must be strictly greater than `min_seconds`. frame_rate (`float`, *optional*, defaults to `24.0`): The frames per second (FPS) of the generated video. + conditioning_frame_rate (`float`, *optional*): + The frame rate the model is conditioned on, i.e. the time axis of the positional embeddings. Defaults + to `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture rate + differs from its playback rate, e.g. a slow-motion LoRA at `frame_rate / speed`. Audio duration and the + predicted number of frames keep following `frame_rate`. num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. @@ -1150,6 +1156,8 @@ def __call__( audio_modality_scale = audio_modality_scale or modality_scale audio_guidance_rescale = audio_guidance_rescale or guidance_rescale + conditioning_frame_rate = conditioning_frame_rate if conditioning_frame_rate is not None else frame_rate + # 1. Check inputs. Raise error if not correct self.check_inputs( prompt=prompt, @@ -1367,7 +1375,12 @@ def __call__( # 6. Prepare micro-conditions # Pre-compute video and audio positional ids as they will be the same at each step of the denoising loop video_coords = self.transformer.rope.prepare_video_coords( - latents.shape[0], latent_num_frames, latent_height, latent_width, latents.device, fps=frame_rate + latents.shape[0], + latent_num_frames, + latent_height, + latent_width, + latents.device, + fps=conditioning_frame_rate, ) audio_coords = self.transformer.audio_rope.prepare_audio_coords( audio_latents.shape[0], audio_num_frames, audio_latents.device @@ -1408,7 +1421,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_coords, audio_coords=audio_coords, @@ -1480,7 +1493,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_pos_ids, audio_coords=audio_pos_ids, @@ -1520,7 +1533,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_pos_ids, audio_coords=audio_pos_ids, diff --git a/src/diffusers/pipelines/ltx2/pipeline_ltx2_condition.py b/src/diffusers/pipelines/ltx2/pipeline_ltx2_condition.py index e1bff845302c..f27f4b9d39c1 100644 --- a/src/diffusers/pipelines/ltx2/pipeline_ltx2_condition.py +++ b/src/diffusers/pipelines/ltx2/pipeline_ltx2_condition.py @@ -1355,6 +1355,7 @@ def __call__( min_seconds: float = 1.0, max_seconds: float = 20.0, frame_rate: float = 24.0, + conditioning_frame_rate: float | None = None, num_inference_steps: int = 30, sigmas: list[float] | None = None, timesteps: list[float] | None = None, @@ -1420,6 +1421,11 @@ def __call__( present. Ignored when `num_frames` is set explicitly. Must be strictly greater than `min_seconds`. frame_rate (`float`, *optional*, defaults to `24.0`): The frames per second (FPS) of the generated video. + conditioning_frame_rate (`float`, *optional*): + The frame rate the model is conditioned on, i.e. the time axis of the positional embeddings. Defaults + to `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture rate + differs from its playback rate, e.g. a slow-motion LoRA at `frame_rate / speed`. Audio duration and the + predicted number of frames keep following `frame_rate`. num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. @@ -1574,6 +1580,8 @@ def __call__( audio_modality_scale = audio_modality_scale or modality_scale audio_guidance_rescale = audio_guidance_rescale or guidance_rescale + conditioning_frame_rate = conditioning_frame_rate if conditioning_frame_rate is not None else frame_rate + # 1. Check inputs. Raise error if not correct self.check_inputs( prompt=prompt, @@ -1721,7 +1729,7 @@ def __call__( height=height, width=width, num_frames=num_frames, - frame_rate=frame_rate, + frame_rate=conditioning_frame_rate, noise_scale=noise_scale, dtype=torch.float32, device=device, @@ -1792,7 +1800,12 @@ def __call__( # 6. Prepare micro-conditions # Pre-compute video and audio positional ids as they will be the same at each step of the denoising loop video_coords = self.transformer.rope.prepare_video_coords( - latents.shape[0], latent_num_frames, latent_height, latent_width, latents.device, fps=frame_rate + latents.shape[0], + latent_num_frames, + latent_height, + latent_width, + latents.device, + fps=conditioning_frame_rate, ) audio_coords = self.transformer.audio_rope.prepare_audio_coords( audio_latents.shape[0], audio_num_frames, audio_latents.device @@ -1840,7 +1853,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_coords, audio_coords=audio_coords, @@ -1916,7 +1929,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_pos_ids, audio_coords=audio_pos_ids, @@ -1958,7 +1971,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_pos_ids, audio_coords=audio_pos_ids, diff --git a/src/diffusers/pipelines/ltx2/pipeline_ltx2_ic_lora.py b/src/diffusers/pipelines/ltx2/pipeline_ltx2_ic_lora.py index 2924a086c721..905f7d4d910a 100644 --- a/src/diffusers/pipelines/ltx2/pipeline_ltx2_ic_lora.py +++ b/src/diffusers/pipelines/ltx2/pipeline_ltx2_ic_lora.py @@ -1787,6 +1787,7 @@ def __call__( width: int = 768, num_frames: int = 121, frame_rate: float = 24.0, + conditioning_frame_rate: float | None = None, num_inference_steps: int = 30, sigmas: list[float] | None = None, timesteps: list[float] | None = None, @@ -1864,6 +1865,11 @@ def __call__( The number of video frames to generate. Must satisfy `(n - 1) % 8 == 0`. frame_rate (`float`, *optional*, defaults to `24.0`): The frames per second (FPS) of the generated video. + conditioning_frame_rate (`float`, *optional*): + The frame rate the model is conditioned on, i.e. the time axis of the positional embeddings. Defaults + to `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture rate + differs from its playback rate, e.g. a slow-motion LoRA at `frame_rate / speed`. Audio duration and the + predicted number of frames keep following `frame_rate`. num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. sigmas (`List[float]`, *optional*): @@ -1964,6 +1970,8 @@ def __call__( audio_modality_scale = audio_modality_scale or modality_scale audio_guidance_rescale = audio_guidance_rescale or guidance_rescale + conditioning_frame_rate = conditioning_frame_rate if conditioning_frame_rate is not None else frame_rate + # 1. Check inputs self.check_inputs( prompt=prompt, @@ -2101,7 +2109,7 @@ def __call__( height=height, width=width, num_frames=num_frames, - frame_rate=frame_rate, + frame_rate=conditioning_frame_rate, noise_scale=noise_scale, dtype=torch.float32, device=device, @@ -2195,7 +2203,12 @@ def __call__( # 7. Prepare positional coordinates video_coords = self.transformer.rope.prepare_video_coords( - latents.shape[0], latent_num_frames, latent_height, latent_width, latents.device, fps=frame_rate + latents.shape[0], + latent_num_frames, + latent_height, + latent_width, + latents.device, + fps=conditioning_frame_rate, ) if appended_coords is not None: video_coords = torch.cat([video_coords, appended_coords], dim=2) @@ -2258,7 +2271,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_coords, audio_coords=audio_coords, @@ -2347,7 +2360,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_pos_ids, audio_coords=audio_pos_ids, @@ -2396,7 +2409,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_pos_ids, audio_coords=audio_pos_ids, diff --git a/src/diffusers/pipelines/ltx2/pipeline_ltx2_image2video.py b/src/diffusers/pipelines/ltx2/pipeline_ltx2_image2video.py index 36b7effdfd6c..fde5ca960fb1 100644 --- a/src/diffusers/pipelines/ltx2/pipeline_ltx2_image2video.py +++ b/src/diffusers/pipelines/ltx2/pipeline_ltx2_image2video.py @@ -990,6 +990,7 @@ def __call__( min_seconds: float = 1.0, max_seconds: float = 20.0, frame_rate: float = 24.0, + conditioning_frame_rate: float | None = None, num_inference_steps: int = 30, sigmas: list[float] | None = None, timesteps: list[int] | None = None, @@ -1056,6 +1057,11 @@ def __call__( present. Ignored when `num_frames` is set explicitly. Must be strictly greater than `min_seconds`. frame_rate (`float`, *optional*, defaults to `24.0`): The frames per second (FPS) of the generated video. + conditioning_frame_rate (`float`, *optional*): + The frame rate the model is conditioned on, i.e. the time axis of the positional embeddings. Defaults + to `frame_rate`. Set it apart from `frame_rate` for adapters trained on footage whose capture rate + differs from its playback rate, e.g. a slow-motion LoRA at `frame_rate / speed`. Audio duration and the + predicted number of frames keep following `frame_rate`. num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. @@ -1212,6 +1218,8 @@ def __call__( audio_modality_scale = audio_modality_scale or modality_scale audio_guidance_rescale = audio_guidance_rescale or guidance_rescale + conditioning_frame_rate = conditioning_frame_rate if conditioning_frame_rate is not None else frame_rate + # 1. Check inputs. Raise error if not correct self.check_inputs( prompt=prompt, @@ -1442,7 +1450,12 @@ def __call__( # 6. Prepare micro-conditions # Pre-compute video and audio positional ids as they will be the same at each step of the denoising loop video_coords = self.transformer.rope.prepare_video_coords( - latents.shape[0], latent_num_frames, latent_height, latent_width, latents.device, fps=frame_rate + latents.shape[0], + latent_num_frames, + latent_height, + latent_width, + latents.device, + fps=conditioning_frame_rate, ) audio_coords = self.transformer.audio_rope.prepare_audio_coords( audio_latents.shape[0], audio_num_frames, audio_latents.device @@ -1484,7 +1497,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_coords, audio_coords=audio_coords, @@ -1558,7 +1571,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_pos_ids, audio_coords=audio_pos_ids, @@ -1599,7 +1612,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_pos_ids, audio_coords=audio_pos_ids, diff --git a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py index 69080095c627..86afd81ffa49 100644 --- a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py +++ b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py @@ -22,6 +22,7 @@ from diffusers.pipelines.ltx2.pipeline_ltx2_condition import LTX2VideoCondition from diffusers.pipelines.ltx2.pipeline_ltx2_ic_lora import LTX2ReferenceCondition +from ...testing_utils import assert_tensors_close from ..testing_utils import ( BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, @@ -182,6 +183,24 @@ def test_auto_duration_predicts_a_grid_valid_frame_count(self): assert (num_frames - 1) % pipe.vae_temporal_compression_ratio == 0 assert 0 < num_frames <= round(2.0 * inputs["frame_rate"]) + def test_conditioning_frame_rate_rescales_only_the_video_time_axis(self): + pipe = self.get_pipeline().to("cpu") + frame_rate = self.get_dummy_inputs()["frame_rate"] + outputs = ["videos", "audio", "video_coords", "audio_coords"] + + default = pipe(**self.get_dummy_inputs(), output=outputs) + same = pipe(**self.get_dummy_inputs(), conditioning_frame_rate=frame_rate, output=outputs) + slow = pipe(**self.get_dummy_inputs(), conditioning_frame_rate=2 * frame_rate, output=outputs) + + assert_tensors_close(same["videos"], default["videos"], atol=1e-6, rtol=0) + # A doubled conditioning rate halves the time axis of the video positions and nothing else. + assert_tensors_close(slow["video_coords"][:, 0], default["video_coords"][:, 0] / 2, atol=1e-6, rtol=1e-6) + assert torch.equal(slow["video_coords"][:, 1:], default["video_coords"][:, 1:]) + assert torch.equal(slow["audio_coords"], default["audio_coords"]) + assert slow["videos"].shape == default["videos"].shape + assert slow["audio"].shape == default["audio"].shape + assert not torch.allclose(slow["videos"], default["videos"]) + class TestLTX25Text2VideoModularPipelineLoading(LTX25Text2VideoModularPipelineTesterConfig, ModularLoadingTesterMixin): pass diff --git a/tests/pipelines/ltx2/test_ltx2.py b/tests/pipelines/ltx2/test_ltx2.py index 0ad9da2b46a3..0b7c0101bf92 100644 --- a/tests/pipelines/ltx2/test_ltx2.py +++ b/tests/pipelines/ltx2/test_ltx2.py @@ -244,6 +244,32 @@ def test_invalid_duration_bounds_raise(self): with pytest.raises(ValueError, match="min_seconds"): pipe(**inputs) + def test_conditioning_frame_rate_changes_the_video_only(self): + pipe = self.get_pipeline() + frame_rate = self.get_dummy_inputs()["frame_rate"] + + default = pipe(**self.get_dummy_inputs()) + same = pipe(**{**self.get_dummy_inputs(), "conditioning_frame_rate": frame_rate}) + slow = pipe(**{**self.get_dummy_inputs(), "conditioning_frame_rate": 2 * frame_rate}) + + # Defaulting to `frame_rate` leaves the pipeline unchanged. + assert_tensors_close(same.frames, default.frames, atol=1e-6, rtol=0) + assert_tensors_close(same.audio, default.audio, atol=1e-6, rtol=0) + # A different conditioning rate changes the video, not the output shapes or the audio length. + assert slow.frames.shape == default.frames.shape + assert slow.audio.shape == default.audio.shape + assert not torch.allclose(slow.frames, default.frames) + + def test_duration_head_follows_frame_rate_not_conditioning_frame_rate(self): + pipe = self.get_pipeline_with_duration_head() + inputs = self.get_dummy_inputs() + inputs.pop("num_frames") + inputs.update(min_seconds=1.0, max_seconds=2.0) + + default = pipe(**inputs).frames + slow = pipe(**{**inputs, "conditioning_frame_rate": 5 * inputs["frame_rate"]}).frames + assert slow.shape[1] == default.shape[1] + class TestLTX2PipelineMemory(LTX2PipelineTesterConfig, LTX2MemoryTesterMixin): """Memory optimization tests (CPU offload, group offload, layerwise casting) for the LTX2 pipeline."""